Merge bitcoin/bitcoin#32480: [28.x] 28.2rc1

186e3f1fb6 doc: update manual pages for 28.2rc1 (fanquake)
bbfb994fc1 build: bump version to 28.2rc1 (fanquake)
fdc629fa73 doc: update release notes for 28.2rc1 (fanquake)

Pull request description:

  Final changes for `v28.2rc1`.

ACKs for top commit:
  willcl-ark:
    ACK 186e3f1fb6

Tree-SHA512: 56c192c9bfb7b2d69b7b039d8e024c18ba39dffed42dbd120f15644cac134e008333ec73ac6ef501db2f2a0f3b125b3003650c4209fa26d0ecb8b566d417374e

.cirrus.yml

@@ -3,30 +3,80 @@ env:  # Global defaults
   PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y"
   MAKEJOBS: "-j10"
   TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
-  CI_FAILFAST_TEST_LEAVE_DANGLING: "1"  # Cirrus CI does not care about dangling process and setting this variable avoids killing the CI script itself on error
-  CCACHE_SIZE: "200M"
+  CI_FAILFAST_TEST_LEAVE_DANGLING: "1"  # Cirrus CI does not care about dangling processes and setting this variable avoids killing the CI script itself on error
+  CCACHE_MAXSIZE: "200M"
   CCACHE_DIR: "/tmp/ccache_dir"
   CCACHE_NOHASHDIR: "1"  # Debug info might contain a stale path if the build dir changes, but this is fine
 
-cirrus_ephemeral_worker_template_env: &CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-  DANGER_RUN_CI_ON_HOST: "1"  # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system
-
-persistent_worker_template_env: &PERSISTENT_WORKER_TEMPLATE_ENV
-  RESTART_CI_DOCKER_BEFORE_RUN: "1"
-
-persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE
-  persistent_worker: {}  # https://cirrus-ci.org/guide/persistent-workers/
+# A self-hosted machine(s) can be used via Cirrus CI. It can be configured with
+# multiple users to run tasks in parallel. No sudo permission is required.
+#
+# https://cirrus-ci.org/guide/persistent-workers/
+#
+# Generally, a persistent worker must run Ubuntu 23.04+ or Debian 12+.
+#
+# The following specific types should exist, with the following requirements:
+# - small: For an x86_64 machine, recommended to have 2 CPUs and 8 GB of memory.
+# - medium: For an x86_64 machine, recommended to have 4 CPUs and 16 GB of memory.
+# - arm64: For an aarch64 machine, recommended to have 2 CPUs and 8 GB of memory.
+#
+# CI jobs for the latter configuration can be run on x86_64 hardware
+# by installing qemu-user-static, which works out of the box with
+# podman or docker. Background: https://stackoverflow.com/a/72890225/313633
+#
+# The above machine types are matched to each task by their label. Refer to the
+# Cirrus CI docs for more details.
+#
+# When a contributor maintains a fork of the repo, any pull request they make
+# to their own fork, or to the main repository, will trigger two CI runs:
+# one for the branch push and one for the pull request.
+# This can be avoided by setting SKIP_BRANCH_PUSH=true as a custom env variable
+# in Cirrus repository settings, accessible from
+# https://cirrus-ci.com/github/my-organization/my-repository
+#
+# On machines that are persisted between CI jobs, RESTART_CI_DOCKER_BEFORE_RUN=1
+# ensures that previous containers and artifacts are cleared before each run.
+# This requires installing Podman instead of Docker.
+#
+# Futhermore:
+# - apt-get is required due to PACKAGE_MANAGER_INSTALL
+# - podman-docker-4.1+ is required due to the bugfix in 4.1
+#   (https://github.com/bitcoin/bitcoin/pull/21652#issuecomment-1657098200)
+# - The ./ci/ dependencies (with cirrus-cli) should be installed. One-liner example
+#   for a single user setup with sudo permission:
+#
+#   ```
+#   apt update && apt install git screen python3 bash podman-docker curl -y && curl -L -o cirrus "https://github.com/cirruslabs/cirrus-cli/releases/latest/download/cirrus-linux-$(dpkg --print-architecture)" && mv cirrus /usr/local/bin/cirrus && chmod +x /usr/local/bin/cirrus
+#   ```
+#
+# - There are no strict requirements on the hardware. Having fewer CPU threads
+#   than recommended merely causes the CI script to run slower.
+#   To avoid rare and intermittent OOM due to short memory usage spikes,
+#   it is recommended to add (and persist) swap:
+#
+#   ```
+#   fallocate -l 16G /swapfile_ci && chmod 600 /swapfile_ci && mkswap /swapfile_ci && swapon /swapfile_ci && ( echo '/swapfile_ci none swap sw 0 0' | tee -a /etc/fstab )
+#   ```
+#
+# - To register the persistent worker, open a `screen` session and run:
+#
+#   ```
+#   RESTART_CI_DOCKER_BEFORE_RUN=1 screen cirrus worker run --labels type=todo_fill_in_type --token todo_fill_in_token
+#   ```
 
 # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks
 filter_template: &FILTER_TEMPLATE
-  skip: $CIRRUS_REPO_FULL_NAME == "bitcoin-core/gui" && $CIRRUS_PR == ""  # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution
+  # Allow forks to specify SKIP_BRANCH_PUSH=true and skip CI runs when a branch is pushed,
+  # but still run CI when a PR is created.
+  # https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution
+  skip: $SKIP_BRANCH_PUSH == "true" && $CIRRUS_PR == ""
   stateful: false  # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks
 
 base_template: &BASE_TEMPLATE
   << : *FILTER_TEMPLATE
   merge_base_script:
     # Unconditionally install git (used in fingerprint_script).
-    - bash -c "$PACKAGE_MANAGER_INSTALL git"
+    - git --version || bash -c "$PACKAGE_MANAGER_INSTALL git"
     - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
     - git fetch --depth=1 $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge"
     - git checkout FETCH_HEAD  # Use merged changes to detect silent merge conflicts
@@ -34,26 +84,11 @@ base_template: &BASE_TEMPLATE
 
 main_template: &MAIN_TEMPLATE
   timeout_in: 120m  # https://cirrus-ci.org/faq/#instance-timed-out
-  ccache_cache:
-    folder: "/tmp/ccache_dir"
   ci_script:
     - ./ci/test_run_all.sh
 
-container_depends_template: &CONTAINER_DEPENDS_TEMPLATE
-  << : *BASE_TEMPLATE
-  container:
-    # https://cirrus-ci.org/faq/#are-there-any-limits
-    # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel
-    cpu: 2
-    greedy: true
-    memory: 8G  # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers
-    dockerfile: ci/test_imagefile  # https://cirrus-ci.org/guide/docker-builder-vm/#dockerfile-as-a-ci-environment
-  depends_built_cache:
-    folder: "depends/built"
-    fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:depends)
-
 global_task_template: &GLOBAL_TASK_TEMPLATE
-  << : *CONTAINER_DEPENDS_TEMPLATE
+  << : *BASE_TEMPLATE
   << : *MAIN_TEMPLATE
 
 compute_credits_template: &CREDITS_TEMPLATE
@@ -62,7 +97,7 @@ compute_credits_template: &CREDITS_TEMPLATE
   use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != ""
 
 task:
-  name: 'lint [bookworm]'
+  name: 'lint'
   << : *BASE_TEMPLATE
   container:
     image: debian:bookworm
@@ -70,296 +105,113 @@ task:
     memory: 1G
   # For faster CI feedback, immediately schedule the linters
   << : *CREDITS_TEMPLATE
+  test_runner_cache:
+    folder: "/lint_test_runner"
+    fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:test/lint/test_runner)
   python_cache:
-    folder: "/tmp/python"
+    folder: "/python_build"
     fingerprint_script: cat .python-version /etc/os-release
   unshallow_script:
     - git fetch --unshallow --no-tags
   lint_script:
     - ./ci/lint_run_all.sh
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
 
 task:
-  name: 'tidy [lunar]'
+  name: 'tidy'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
-    cpu: 2
-    memory: 5G
-    docker_arguments:
-      CI_IMAGE_NAME_TAG: ubuntu:lunar
-      FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh"
-  # For faster CI feedback, immediately schedule the linters
-  << : *CREDITS_TEMPLATE
+  persistent_worker:
+    labels:
+      type: medium
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+    FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh"
 
 task:
-  name: "Win64 native [vs2022]"
-  << : *FILTER_TEMPLATE
-  windows_container:
-    cpu: 6
-    memory: 12G
-    image: cirrusci/windowsservercore:visualstudio2022
-  timeout_in: 120m
-  env:
-    PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Current\Bin;%PATH%'
-    PYTHONUTF8: 1
-    CI_VCPKG_TAG: '2023.01.09'
-    VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads'
-    VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives'
-    CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache'
-    WRAPPED_CL: 'C:\Users\ContainerAdministrator\AppData\Local\Temp\cirrus-ci-build\ci\test\wrapped-cl.bat'
-    QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip'
-    QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.5.zip'
-    QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.5'
-    QTBASEDIR: 'C:\Qt_static'
-    x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"'
-    QT_CONFIGURE_COMMAND: '..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml'
-    IgnoreWarnIntDirInTempDetected: 'true'
-  merge_script:
-    - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL pull/$env:CIRRUS_PR/merge; git reset --hard FETCH_HEAD; }
-  msvc_qt_built_cache:
-    folder: "%QTBASEDIR%"
-    reupload_on_changes: false
-    fingerprint_script:
-      - echo %QT_DOWNLOAD_URL% %QT_CONFIGURE_COMMAND%
-      - msbuild -version
-    populate_script:
-      - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip
-      - mkdir C:\jom
-      - tar -xf C:\jom.zip -C C:\jom
-      - curl -L -o %QT_LOCAL_PATH% %QT_DOWNLOAD_URL%
-      - tar -xf %QT_LOCAL_PATH% -C C:\
-      - '%x64_NATIVE_TOOLS%'
-      - cd %QT_SOURCE_DIR%
-      - mkdir build
-      - cd build
-      - '%QT_CONFIGURE_COMMAND% -prefix %QTBASEDIR%'
-      - jom
-      - jom install
-  vcpkg_tools_cache:
-    folder: '%VCPKG_DOWNLOADS%\tools'
-    reupload_on_changes: false
-    fingerprint_script:
-      - echo %CI_VCPKG_TAG%
-      - msbuild -version
-  vcpkg_binary_cache:
-    folder: '%VCPKG_DEFAULT_BINARY_CACHE%'
-    reupload_on_changes: true
-    fingerprint_script:
-      - echo %CI_VCPKG_TAG%
-      - type build_msvc\vcpkg.json
-      - msbuild -version
-    populate_script:
-      - mkdir %VCPKG_DEFAULT_BINARY_CACHE%
-  ccache_cache:
-    folder: '%CCACHE_DIR%'
-  install_tools_script:
-    - choco install --yes --no-progress ccache --version=4.7.4
-    - choco install --yes --no-progress python3 --version=3.9.6
-    - pip install zmq
-    - ccache --version
-    - python -VV
-  install_vcpkg_script:
-    - cd ..
-    - git clone --quiet https://github.com/microsoft/vcpkg.git
-    - cd vcpkg
-    - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG%
-    - .\bootstrap-vcpkg -disableMetrics
-    - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake
-    - .\vcpkg integrate install
-    - .\vcpkg version
-  build_script:
-    - '%x64_NATIVE_TOOLS%'
-    - cd %CIRRUS_WORKING_DIR%
-    - ccache --zero-stats --max-size=%CCACHE_SIZE%
-    - python build_msvc\msvc-autogen.py
-    - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL%;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo
-    - ccache --show-stats
-  check_script:
-    - src\test_bitcoin.exe -l test_suite
-    - src\bench_bitcoin.exe --sanity-check
-    - python test\util\test_runner.py
-    - python test\util\rpcauth-test.py
-  functional_tests_script:
-    # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted".
-    # See: https://learn.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance
-    - netsh int ipv4 set dynamicport tcp start=1025 num=64511
-    - netsh int ipv6 set dynamicport tcp start=1025 num=64511
-    # Exclude feature_dbcrash for now due to timeout
-    - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=99999999 --jobs=6 --timeout-factor=8 --extended --exclude feature_dbcrash
-
-task:
-  name: 'ARM [unit tests, no functional tests] [bullseye]'
+  name: 'ARM, unit tests, no functional tests'
   << : *GLOBAL_TASK_TEMPLATE
-  arm_container:
-    image: debian:bullseye
-    cpu: 2
-    memory: 8G
-    # docker_arguments:  # Can use dockerfile after https://github.com/cirruslabs/cirrus-ci-docs/issues/1154
+  persistent_worker:
+    labels:
+      type: arm64  # Use arm64 worker to sidestep qemu and avoid a slow CI: https://github.com/bitcoin/bitcoin/pull/28087#issuecomment-1649399453
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_arm.sh"
-    QEMU_USER_CMD: ""  # Disable qemu and run the test natively
 
 task:
-  name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [jammy]'
+  name: 'Win64, unit tests, no gui tests, no functional tests'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
-    docker_arguments:
-      CI_IMAGE_NAME_TAG: ubuntu:jammy
-      FILE_ENV: "./ci/test/00_setup_env_win64.sh"
-  << : *CREDITS_TEMPLATE
+  persistent_worker:
+    labels:
+      type: small
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+    FILE_ENV: "./ci/test/00_setup_env_win64.sh"
 
 task:
-  name: '32-bit + dash [gui] [CentOS 8]'
+  name: '32-bit CentOS, dash, gui'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
-    docker_arguments:
-      CI_IMAGE_NAME_TAG: quay.io/centos/centos:stream8
-      FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
-  # For faster CI feedback, immediately schedule one task that runs all tests
-  << : *CREDITS_TEMPLATE
+  persistent_worker:
+    labels:
+      type: small
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    PACKAGE_MANAGER_INSTALL: "yum install -y"
+    FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
 
 task:
-  name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [buster]'
-  previous_releases_cache:
-    folder: "releases"
+  name: 'previous releases, depends DEBUG'
   << : *GLOBAL_TASK_TEMPLATE
-  << : *PERSISTENT_WORKER_TEMPLATE
+  persistent_worker:
+    labels:
+      type: small
   env:
-    << : *PERSISTENT_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh"
+    FILE_ENV: "./ci/test/00_setup_env_native_previous_releases.sh"
 
 task:
-  name: '[TSan, depends, gui] [lunar]'
+  name: 'TSan, depends, gui'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
-    cpu: 6  # Increase CPU and Memory to avoid timeout
-    memory: 24G
-    docker_arguments:
-      CI_IMAGE_NAME_TAG: ubuntu:lunar
-      FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
+  persistent_worker:
+    labels:
+      type: medium
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+    FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
 
 task:
-  name: '[MSan, depends] [lunar]'
+  name: 'MSan, depends'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
-    docker_arguments:
-      CI_IMAGE_NAME_TAG: ubuntu:lunar
-      FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
+  persistent_worker:
+    labels:
+      type: small
+  timeout_in: 300m  # Use longer timeout for the *rare* case where a full build (llvm + msan + depends + ...) needs to be done.
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    MAKEJOBS: "-j4"  # Avoid excessive memory use due to MSan
+    FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
 
 task:
-  name: '[ASan + LSan + UBSan + integer, no depends, USDT] [jammy]'
+  name: 'fuzzer,address,undefined,integer, no depends'
   << : *GLOBAL_TASK_TEMPLATE
-  # We can't use a 'container' for the USDT interface tests as the CirrusCI
-  # containers don't have privileges to hook into bitcoind. CirrusCI uses
-  # Google Compute Engine instances: https://cirrus-ci.org/guide/custom-vms/
-  # Images can be found here: https://cloud.google.com/compute/docs/images/os-details
-  compute_engine_instance:
-    image_project: ubuntu-os-cloud
-    image: family/ubuntu-2204-lts # when upgrading, check if we can drop "ADD_UNTRUSTED_BPFCC_PPA"
-    cpu: 4
-    memory: 12G
+  persistent_worker:
+    labels:
+      type: medium
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    HOME: /root/  # Only needed for compute_engine_instance
-    FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
-    MAKEJOBS: "-j4"  # Avoid excessive memory use
+    FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
 
 task:
-  name: '[fuzzer,address,undefined,integer, no depends] [lunar]'
+  name: 'multiprocess, i686, DEBUG'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
-    cpu: 4  # Increase CPU and memory to avoid timeout
-    memory: 16G
-    docker_arguments:
-      CI_IMAGE_NAME_TAG: ubuntu:lunar
-      FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
+  persistent_worker:
+    labels:
+      type: medium
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+    FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh"
 
 task:
-  name: '[multiprocess, i686, DEBUG] [focal]'
+  name: 'no wallet, libbitcoinkernel'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
-    cpu: 4
-    memory: 16G  # The default memory is sometimes just a bit too small, so double everything
-    docker_arguments:
-      CI_IMAGE_NAME_TAG: ubuntu:focal
-      FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh"
+  persistent_worker:
+    labels:
+      type: small
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+    FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh"
 
 task:
-  name: '[no wallet, libbitcoinkernel] [focal]'
+  name: 'macOS-cross, gui, no tests'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
-    docker_arguments:
-      CI_IMAGE_NAME_TAG: ubuntu:focal
-      FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh"
-  << : *CREDITS_TEMPLATE
+  persistent_worker:
+    labels:
+      type: small
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-
-task:
-  name: 'macOS 10.15 [gui, no tests] [focal]'
-  << : *CONTAINER_DEPENDS_TEMPLATE
-  container:
-    docker_arguments:
-      CI_IMAGE_NAME_TAG: ubuntu:focal
-      FILE_ENV: "./ci/test/00_setup_env_mac.sh"
-  << : *CREDITS_TEMPLATE
-  macos_sdk_cache:
-    folder: "depends/SDKs/$MACOS_SDK"
-    fingerprint_key: "$MACOS_SDK"
-  << : *MAIN_TEMPLATE
-  env:
-    MACOS_SDK: "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers"
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-
-task:
-  name: 'macOS 13 native arm64 [gui, sqlite only] [no depends]'
-  macos_instance:
-    # Use latest image, but hardcode version to avoid silent upgrades (and breaks)
-    image: ghcr.io/cirruslabs/macos-ventura-xcode:14.1  # https://cirrus-ci.org/guide/macOS
-  << : *BASE_TEMPLATE
-  check_clang_script:
-    - clang --version
-  brew_install_script:
-    - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt
-  << : *MAIN_TEMPLATE
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    CI_USE_APT_INSTALL: "no"
-    PACKAGE_MANAGER_INSTALL: "echo"  # Nothing to do
-    FILE_ENV: "./ci/test/00_setup_env_mac_native_arm64.sh"
-
-task:
-  name: 'ARM64 Android APK [jammy]'
-  << : *CONTAINER_DEPENDS_TEMPLATE
-  container:
-    docker_arguments:
-      CI_IMAGE_NAME_TAG: ubuntu:jammy
-      FILE_ENV: "./ci/test/00_setup_env_android.sh"
-  << : *CREDITS_TEMPLATE
-  android_sdk_cache:
-    folder: "depends/SDKs/android"
-    fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.2.8568313"
-  depends_sources_cache:
-    folder: "depends/sources"
-    fingerprint_script: git rev-parse HEAD:depends/packages
-  << : *MAIN_TEMPLATE
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+    FILE_ENV: "./ci/test/00_setup_env_mac_cross.sh"

.github/ISSUE_TEMPLATE/good_first_issue.yml

@@ -29,8 +29,10 @@ body:
     attributes:
       label: Useful Skills
       description: For example, “`std::thread`”, “Qt5 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”.
-    validations:
-      required: false
+      value: |
+        * Compiling Bitcoin Core from source
+        * Running the C++ unit tests and the Python functional tests
+        * ...
   - type: textarea
     attributes:
       label: Guidance for new contributors

.github/workflows/ci.yml

@@ -0,0 +1,355 @@
+# Copyright (c) 2023 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+name: CI
+on:
+  # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request.
+  pull_request:
+  # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push.
+  push:
+    branches:
+      - '**'
+    tags-ignore:
+      - '**'
+
+concurrency:
+  group: ${{ github.event_name != 'pull_request' && github.run_id || github.ref }}
+  cancel-in-progress: true
+
+env:
+  CI_FAILFAST_TEST_LEAVE_DANGLING: 1  # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error
+  MAKEJOBS: '-j10'
+
+jobs:
+  test-each-commit:
+    name: 'test each commit'
+    runs-on: ubuntu-24.04
+    if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1
+    timeout-minutes: 360  # Use maximum time, see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. Assuming a worst case time of 1 hour per commit, this leads to a --max-count=6 below.
+    env:
+      MAX_COUNT: 6
+    steps:
+      - name: Determine fetch depth
+        run: echo "FETCH_DEPTH=$((${{ github.event.pull_request.commits }} + 2))" >> "$GITHUB_ENV"
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: ${{ env.FETCH_DEPTH }}
+      - name: Determine commit range
+        run: |
+          # Checkout HEAD~ and find the test base commit
+          # Checkout HEAD~ because it would be wasteful to rerun tests on the PR
+          # head commit that are already run by other jobs.
+          git checkout HEAD~
+          # Figure out test base commit by listing ancestors of HEAD, excluding
+          # ancestors of the most recent merge commit, limiting the list to the
+          # newest MAX_COUNT ancestors, ordering it from oldest to newest, and
+          # taking the first one.
+          #
+          # If the branch contains up to MAX_COUNT ancestor commits after the
+          # most recent merge commit, all of those commits will be tested. If it
+          # contains more, only the most recent MAX_COUNT commits will be
+          # tested.
+          #
+          # In the command below, the ^@ suffix is used to refer to all parents
+          # of the merge commit as described in:
+          # https://git-scm.com/docs/git-rev-parse#_other_rev_parent_shorthand_notations
+          # and the ^ prefix is used to exclude these parents and all their
+          # ancestors from the rev-list output as described in:
+          # https://git-scm.com/docs/git-rev-list
+          MERGE_BASE=$(git rev-list -n1 --merges HEAD)
+          EXCLUDE_MERGE_BASE_ANCESTORS=
+          # MERGE_BASE can be empty due to limited fetch-depth
+          if test -n "$MERGE_BASE"; then
+            EXCLUDE_MERGE_BASE_ANCESTORS=^${MERGE_BASE}^@
+          fi
+          echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD $EXCLUDE_MERGE_BASE_ANCESTORS | head -1)" >> "$GITHUB_ENV"
+      - run: |
+          sudo apt-get update
+          sudo apt-get install clang ccache build-essential libtool autotools-dev automake pkg-config bsdmainutils python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libminiupnpc-dev libnatpmp-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y
+      - name: Compile and run tests
+        run: |
+          # Run tests on commits after the last merge commit and before the PR head commit
+          # Use clang++, because it is a bit faster and uses less memory than g++
+          git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && ./autogen.sh && CC=clang CXX=clang++ ./configure --with-incompatible-bdb && make clean && make -j $(nproc) check && ./test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.TEST_BASE }}
+
+  macos-native-x86_64:
+    name: 'macOS 13 native, x86_64, no depends, sqlite only, gui'
+    # Use latest image, but hardcode version to avoid silent upgrades (and breaks).
+    # See: https://github.com/actions/runner-images#available-images.
+    runs-on: macos-13
+
+    # No need to run on the read-only mirror, unless it is a PR.
+    if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request'
+
+    timeout-minutes: 120
+
+    env:
+      DANGER_RUN_CI_ON_HOST: 1
+      FILE_ENV: './ci/test/00_setup_env_mac_native.sh'
+      BASE_ROOT_DIR: ${{ github.workspace }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Clang version
+        run: |
+          sudo xcode-select --switch /Applications/Xcode_15.0.app
+          clang --version
+
+      - name: Install Homebrew packages
+        env:
+          HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
+        run: |
+          # A workaround for "The `brew link` step did not complete successfully" error.
+          brew install --quiet python@3 || brew link --overwrite python@3
+          brew install --quiet automake libtool pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq qt@5 qrencode
+
+      - name: Set Ccache directory
+        run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV"
+
+      - name: Restore Ccache cache
+        id: ccache-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ${{ github.job }}-ccache-${{ github.run_id }}
+          restore-keys: ${{ github.job }}-ccache-
+
+      - name: CI script
+        run: ./ci/test_run_all.sh
+
+      - name: Save Ccache cache
+        uses: actions/cache/save@v4
+        if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true'
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
+          key: ${{ github.job }}-ccache-${{ github.run_id }}
+
+  win64-native:
+    name: 'Win64 native, VS 2022'
+    # Use latest image, but hardcode version to avoid silent upgrades (and breaks).
+    # See: https://github.com/actions/runner-images#available-images.
+    runs-on: windows-2022
+
+    # No need to run on the read-only mirror, unless it is a PR.
+    if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request'
+
+    env:
+      CCACHE_MAXSIZE: '200M'
+      CI_CCACHE_VERSION: '4.7.5'
+      CI_QT_CONF: '-release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml'
+      CI_QT_DIR: 'qt-everywhere-src-5.15.11'
+      CI_QT_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.11/single/qt-everywhere-opensource-src-5.15.11.zip'
+      PYTHONUTF8: 1
+      TEST_RUNNER_TIMEOUT_FACTOR: 40
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Configure Developer Command Prompt for Microsoft Visual C++
+        # Using microsoft/setup-msbuild is not enough.
+        uses: ilammy/msvc-dev-cmd@v1
+        with:
+          arch: x64
+
+      - name: Get tool information
+        run: |
+          msbuild -version | Tee-Object -FilePath "msbuild_version"
+          $env:VCToolsVersion | Tee-Object -FilePath "toolset_version"
+          $env:CI_QT_URL | Out-File -FilePath "qt_url"
+          $env:CI_QT_CONF | Out-File -FilePath "qt_conf"
+          py -3 --version
+          Write-Host "PowerShell version $($PSVersionTable.PSVersion.ToString())"
+
+      - name: Restore static Qt cache
+        id: static-qt-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: C:\Qt_static
+          key: ${{ github.job }}-static-qt-${{ hashFiles('msbuild_version', 'qt_url', 'qt_conf') }}
+
+      - name: Build static Qt. Download
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        shell: cmd
+        run: |
+          curl --location --output C:\qt-src.zip %CI_QT_URL%
+          choco install --yes --no-progress jom
+
+      - name: Build static Qt. Expand source archive
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        shell: cmd
+        run: tar -xf C:\qt-src.zip -C C:\
+
+      - name: Build static Qt. Create build directory
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        run: |
+          Rename-Item -Path "C:\$env:CI_QT_DIR" -NewName "C:\qt-src"
+          New-Item -ItemType Directory -Path "C:\qt-src\build"
+
+      - name: Build static Qt. Configure
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        working-directory: C:\qt-src\build
+        shell: cmd
+        run: ..\configure %CI_QT_CONF% -prefix C:\Qt_static
+
+      - name: Build static Qt. Build
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        working-directory: C:\qt-src\build
+        shell: cmd
+        run: jom
+
+      - name: Build static Qt. Install
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        working-directory: C:\qt-src\build
+        shell: cmd
+        run: jom install
+
+      - name: Save static Qt cache
+        if: steps.static-qt-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: C:\Qt_static
+          key: ${{ github.job }}-static-qt-${{ hashFiles('msbuild_version', 'qt_url', 'qt_conf') }}
+
+      - name: Ccache installation cache
+        id: ccache-installation-cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            C:\ProgramData\chocolatey\lib\ccache
+            C:\ProgramData\chocolatey\bin\ccache.exe
+            C:\ccache\cl.exe
+          key: ${{ github.job }}-ccache-installation-${{ env.CI_CCACHE_VERSION }}
+
+      - name: Install Ccache
+        if: steps.ccache-installation-cache.outputs.cache-hit != 'true'
+        run: |
+          choco install --yes --no-progress ccache --version=$env:CI_CCACHE_VERSION
+          New-Item -ItemType Directory -Path "C:\ccache"
+          Copy-Item -Path "$env:ChocolateyInstall\lib\ccache\tools\ccache-$env:CI_CCACHE_VERSION-windows-x86_64\ccache.exe" -Destination "C:\ccache\cl.exe"
+
+      - name: Restore Ccache cache
+        id: ccache-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ~/AppData/Local/ccache
+          key: ${{ github.job }}-ccache-${{ github.run_id }}
+          restore-keys: ${{ github.job }}-ccache-
+
+      - name: Using vcpkg with MSBuild
+        run: |
+          Set-Location "$env:VCPKG_INSTALLATION_ROOT"
+          Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)"
+          .\vcpkg.exe --vcpkg-root "$env:VCPKG_INSTALLATION_ROOT" integrate install
+          git rev-parse HEAD | Tee-Object -FilePath "$env:GITHUB_WORKSPACE\vcpkg_commit"
+
+      - name: vcpkg tools cache
+        uses: actions/cache@v4
+        with:
+          path: C:/vcpkg/downloads/tools
+          key: ${{ github.job }}-vcpkg-tools
+
+      - name: vcpkg binary cache
+        uses: actions/cache@v4
+        with:
+          path: ~/AppData/Local/vcpkg/archives
+          key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('vcpkg_commit', 'msbuild_version', 'toolset_version', 'build_msvc/vcpkg.json') }}
+
+      - name: Generate project files
+        run: py -3 build_msvc\msvc-autogen.py
+
+      - name: Build
+        shell: cmd
+        run: |
+          ccache --zero-stats
+          msbuild build_msvc\bitcoin.sln -property:CLToolPath=C:\ccache;CLToolExe=cl.exe;UseMultiToolTask=true;Configuration=Release -maxCpuCount -verbosity:minimal -noLogo
+
+      - name: Ccache stats
+        run: ccache --show-stats
+
+      - name: Save Ccache cache
+        uses: actions/cache/save@v4
+        if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true'
+        with:
+          path: ~/AppData/Local/ccache
+          # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
+          key: ${{ github.job }}-ccache-${{ github.run_id }}
+
+      - name: Run unit tests
+        run: src\test_bitcoin.exe -l test_suite
+
+      - name: Run benchmarks
+        run: src\bench_bitcoin.exe -sanity-check
+
+      - name: Run util tests
+        run: py -3 test\util\test_runner.py
+
+      - name: Run rpcauth test
+        run: py -3 test\util\rpcauth-test.py
+
+      - name: Run functional tests
+        env:
+          TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }}
+        shell: cmd
+        run: py -3 test\functional\test_runner.py --jobs %NUMBER_OF_PROCESSORS% --ci --quiet --tmpdirprefix=%RUNNER_TEMP% --combinedlogslen=99999999 --timeout-factor=%TEST_RUNNER_TIMEOUT_FACTOR% %TEST_RUNNER_EXTRA%
+
+      - name: Clone fuzz corpus
+        run: |
+          git clone --depth=1 https://github.com/bitcoin-core/qa-assets "$env:RUNNER_TEMP\qa-assets"
+          Set-Location "$env:RUNNER_TEMP\qa-assets"
+          Write-Host "Using qa-assets repo from commit ..."
+          git log -1
+
+      - name: Run fuzz binaries
+        env:
+          BITCOINFUZZ: "${{ github.workspace}}\\src\\fuzz.exe"
+        shell: cmd
+        run: py -3 test\fuzz\test_runner.py --par %NUMBER_OF_PROCESSORS% --loglevel DEBUG %RUNNER_TEMP%\qa-assets\fuzz_seed_corpus
+
+  asan-lsan-ubsan-integer-no-depends-usdt:
+    name: 'ASan + LSan + UBSan + integer, no depends, USDT'
+    runs-on: ubuntu-24.04 # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools
+    # No need to run on the read-only mirror, unless it is a PR.
+    if: github.repository != 'bitcoin-core/gui' || github.event_name == 'pull_request'
+    timeout-minutes: 120
+    env:
+      FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
+      DANGER_CI_ON_HOST_CACHE_FOLDERS: 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set Ccache directory
+        run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV"
+
+      - name: Set base root directory
+        run: echo "BASE_ROOT_DIR=${RUNNER_TEMP}" >> "$GITHUB_ENV"
+
+      - name: Restore Ccache cache
+        id: ccache-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ${{ github.job }}-ccache-${{ github.run_id }}
+          restore-keys: ${{ github.job }}-ccache-
+
+      - name: Enable bpfcc script
+        # In the image build step, no external environment variables are available,
+        # so any settings will need to be written to the settings env file:
+        run: sed -i "s|\${INSTALL_BCC_TRACING_TOOLS}|true|g" ./ci/test/00_setup_env_native_asan.sh
+
+      - name: CI script
+        run: ./ci/test_run_all.sh
+
+      - name: Save Ccache cache
+        uses: actions/cache/save@v4
+        if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true'
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
+          key: ${{ github.job }}-ccache-${{ github.run_id }}

.gitignore

@@ -74,7 +74,7 @@ src/qt/bitcoin-qt.includes
 
 *.log
 *.trs
-*.dmg
+*.zip
 
 *.json.h
 *.raw.h
@@ -130,12 +130,12 @@ win32-build
 test/config.ini
 test/cache/*
 test/.mypy_cache/
+test/lint/test_runner/target/
 
 !src/leveldb*/Makefile
 
 /doc/doxygen/
 
-libbitcoinconsensus.pc
 contrib/devtools/split-debug.sh
 
 # Output from running db4 installation
@@ -144,7 +144,6 @@ db4/
 # clang-check
 *.plist
 
-osx_volname
 dist/
 
 /guix-build-*

.python-version

@@ -1 +1 @@
-3.7.16
+3.9.18

.style.yapf

@@ -107,7 +107,7 @@ each_dict_entry_on_separate_line=True
 i18n_comment=
 
 # The i18n function call names. The presence of this function stops
-# reformattting on that line, because the string it has cannot be moved
+# reformatting on that line, because the string it has cannot be moved
 # away from the i18n comment.
 i18n_function_call=
 

.tx/config

@@ -1,7 +1,7 @@
 [main]
 host = https://www.transifex.com
 
-[o:bitcoin:p:bitcoin:r:qt-translation-025x]
+[o:bitcoin:p:bitcoin:r:qt-translation-028x]
 file_filter = src/qt/locale/bitcoin_<lang>.xlf
 source_file = src/qt/locale/bitcoin_en.xlf
 source_lang = en

CONTRIBUTING.md

@@ -66,9 +66,10 @@ Discussion about codebase improvements happens in GitHub issues and pull
 requests.
 
 The developer
-[mailing list](https://lists.linuxfoundation.org/mailman/listinfo/bitcoin-dev)
+[mailing list](https://groups.google.com/g/bitcoindev)
 should be used to discuss complicated or controversial consensus or P2P protocol changes before working on
 a patch set.
+Archives can be found on [https://gnusha.org/pi/bitcoindev/](https://gnusha.org/pi/bitcoindev/).
 
 
 Contributor Workflow
@@ -148,7 +149,7 @@ the pull request affects. Valid areas as:
   - `net` or `p2p` for changes to the peer-to-peer network code
   - `refactor` for structural changes that do not change behavior
   - `rpc`, `rest` or `zmq` for changes to the RPC, REST or ZMQ APIs
-  - `script` for changes to the scripts and tools
+  - `contrib` or `cli` for changes to the scripts and tools
   - `test`, `qa` or `ci` for changes to the unit tests, QA tests or CI code
   - `util` or `lib` for changes to the utils or libraries
   - `wallet` for changes to the wallet code
@@ -417,11 +418,8 @@ Backporting
 
 Security and bug fixes can be backported from `master` to release
 branches.
-If the backport is non-trivial, it may be appropriate to open an
-additional PR to backport the change, but only after the original PR
-has been merged.
-Otherwise, backports will be done in batches and
-the maintainers will use the proper `Needs backport (...)` labels
+Maintainers will do backports in batches and
+use the proper `Needs backport (...)` labels
 when needed (the original author does not need to worry about it).
 
 A backport should contain the following metadata in the commit body:

COPYING

@@ -1,7 +1,7 @@
 The MIT License (MIT)
 
-Copyright (c) 2009-2023 The Bitcoin Core developers
-Copyright (c) 2009-2023 Bitcoin Developers
+Copyright (c) 2009-2025 The Bitcoin Core developers
+Copyright (c) 2009-2025 Bitcoin Developers
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile.am

@@ -14,13 +14,6 @@ endif
 .PHONY: deploy FORCE
 .INTERMEDIATE: $(COVERAGE_INFO)
 
-export PYTHONPATH
-
-if BUILD_BITCOIN_LIBS
-pkgconfigdir = $(libdir)/pkgconfig
-pkgconfig_DATA = libbitcoinconsensus.pc
-endif
-
 BITCOIND_BIN=$(top_builddir)/src/$(BITCOIN_DAEMON_NAME)$(EXEEXT)
 BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT)
 BITCOIN_TEST_BIN=$(top_builddir)/src/test/$(BITCOIN_TEST_NAME)$(EXEEXT)
@@ -37,7 +30,7 @@ space := $(empty) $(empty)
 
 OSX_APP=Bitcoin-Qt.app
 OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME))
-OSX_DMG = $(OSX_VOLNAME).dmg
+OSX_ZIP = $(OSX_VOLNAME).zip
 OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus
 OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns
 OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed
@@ -120,27 +113,25 @@ OSX_APP_BUILT=$(OSX_APP)/Contents/PkgInfo $(OSX_APP)/Contents/Resources/empty.lp
   $(OSX_APP)/Contents/Resources/bitcoin.icns $(OSX_APP)/Contents/Info.plist \
   $(OSX_APP)/Contents/MacOS/Bitcoin-Qt $(OSX_APP)/Contents/Resources/Base.lproj/InfoPlist.strings
 
-osx_volname:
-	echo $(OSX_VOLNAME) >$@
-
 if BUILD_DARWIN
-$(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING)
-	$(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -dmg
+$(OSX_ZIP): $(OSX_APP_BUILT) $(OSX_PACKAGING)
+	$(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -zip
 
-deploydir: $(OSX_DMG)
+deploydir: $(OSX_ZIP)
 else !BUILD_DARWIN
 APP_DIST_DIR=$(top_builddir)/dist
 
-$(OSX_DMG): deploydir
-	$(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ $(APP_DIST_DIR) -- $(if $(SOURCE_DATE_EPOCH),-volume_date all_file_dates =$(SOURCE_DATE_EPOCH))
+$(OSX_ZIP): deploydir
+	if [ -n "$(SOURCE_DATE_EPOCH)" ]; then find $(APP_DIST_DIR) -exec touch -d @$(SOURCE_DATE_EPOCH) {} +; fi
+	cd $(APP_DIST_DIR) && find . | sort | $(ZIP) -X@ $@
 
 $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING)
-	INSTALL_NAME_TOOL=$(INSTALL_NAME_TOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR)
+	OBJDUMP=$(OBJDUMP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR)
 
 deploydir: $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt
 endif !BUILD_DARWIN
 
-deploy: $(OSX_DMG)
+deploy: $(OSX_ZIP)
 endif
 
 $(BITCOIN_QT_BIN): FORCE
@@ -188,14 +179,14 @@ $(COV_TOOL_WRAPPER):
 	@chmod +x $(COV_TOOL_WRAPPER)
 
 baseline.info: $(COV_TOOL_WRAPPER)
-	$(LCOV) -c -i -d $(abs_builddir)/src -o $@
+	$(LCOV) $(LCOV_OPTS) -c -i -d $(abs_builddir)/src -o $@
 
 baseline_filtered.info: baseline.info
 	$(abs_builddir)/contrib/filter-lcov.py $(LCOV_FILTER_PATTERN) $< $@
 	$(LCOV) -a $@ $(LCOV_OPTS) -o $@
 
 fuzz.info: baseline_filtered.info
-	@TIMEOUT=15 test/fuzz/test_runner.py $(DIR_FUZZ_SEED_CORPUS) -l DEBUG
+	@test/fuzz/test_runner.py $(DIR_FUZZ_SEED_CORPUS) -l DEBUG
 	$(LCOV) -c $(LCOV_OPTS) -d $(abs_builddir)/src --t fuzz-tests -o $@
 	$(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src
 
@@ -213,7 +204,7 @@ test_bitcoin_filtered.info: test_bitcoin.info
 	$(LCOV) -a $@ $(LCOV_OPTS) -o $@
 
 functional_test.info: test_bitcoin_filtered.info
-	@TIMEOUT=15 test/functional/test_runner.py $(EXTENDED_FUNCTIONAL_TESTS)
+	@test/functional/test_runner.py $(EXTENDED_FUNCTIONAL_TESTS)
 	$(LCOV) -c $(LCOV_OPTS) -d $(abs_builddir)/src --t functional-tests -o $@
 	$(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src
 
@@ -222,13 +213,13 @@ functional_test_filtered.info: functional_test.info
 	$(LCOV) -a $@ $(LCOV_OPTS) -o $@
 
 fuzz_coverage.info: fuzz_filtered.info
-	$(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a fuzz_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt
+	$(LCOV) $(LCOV_OPTS) -a baseline_filtered.info -a fuzz_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt
 
 test_bitcoin_coverage.info: baseline_filtered.info test_bitcoin_filtered.info
-	$(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a test_bitcoin_filtered.info -o $@
+	$(LCOV) $(LCOV_OPTS) -a baseline_filtered.info -a test_bitcoin_filtered.info -o $@
 
 total_coverage.info: test_bitcoin_filtered.info functional_test_filtered.info
-	$(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a test_bitcoin_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt
+	$(LCOV) $(LCOV_OPTS) -a baseline_filtered.info -a test_bitcoin_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt
 
 fuzz.coverage/.dirstamp: fuzz_coverage.info
 	$(GENHTML) -s $(LCOV_OPTS) $< -o $(@D)
@@ -311,9 +302,13 @@ EXTRA_DIST += \
     test/util/data/txcreatesignv1.hex \
     test/util/data/txcreatesignv1.json \
     test/util/data/txcreatesignv2.hex \
+    test/util/data/txreplace1.hex \
+    test/util/data/txreplacenoinputs.hex \
+    test/util/data/txreplaceomittedn.hex \
+    test/util/data/txreplacesingleinput.hex \
     test/util/rpcauth-test.py
 
-CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER)
+CLEANFILES = $(OSX_ZIP) $(BITCOIN_WIN_INSTALLER)
 
 DISTCHECK_CONFIGURE_FLAGS = --enable-man
 
@@ -335,18 +330,18 @@ clean-docs:
 clean-local: clean-docs
 	rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ fuzz.coverage/ test/tmp/ cache/ $(OSX_APP)
 	rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__
-	rm -rf osx_volname dist/
+	rm -rf dist/ test/lint/test_runner/target/ test/lint/__pycache__
 
 test-security-check:
 if TARGET_DARWIN
-	$(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO
-	$(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO
+	$(AM_V_at) CXX='$(CXX)' CXXFLAGS='$(CXXFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO
+	$(AM_V_at) CXX='$(CXX)' CXXFLAGS='$(CXXFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO
 endif
 if TARGET_WINDOWS
-	$(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE
-	$(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE
+	$(AM_V_at) CXX='$(CXX)' CXXFLAGS='$(CXXFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE
+	$(AM_V_at) CXX='$(CXX)' CXXFLAGS='$(CXXFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE
 endif
 if TARGET_LINUX
-	$(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF
-	$(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF
+	$(AM_V_at) CXX='$(CXX)' CXXFLAGS='$(CXXFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF
+	$(AM_V_at) CXX='$(CXX)' CXXFLAGS='$(CXXFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF
 endif

SECURITY.md

@@ -15,6 +15,6 @@ The following keys may be used to communicate sensitive information to developer
 |------|-------------|
 | Pieter Wuille | 133E AC17 9436 F14A 5CF1  B794 860F EB80 4E66 9320 |
 | Michael Ford | E777 299F C265 DD04 7930  70EB 944D 35F9 AC3D B76A |
-| Andrew Chow | 1528 1230 0785 C964 44D3  334D 1756 5732 E08E 5E41 |
+| Ava Chow | 1528 1230 0785 C964 44D3  334D 1756 5732 E08E 5E41 |
 
 You can import a key by running the following command with that individual’s fingerprint: `gpg --keyserver hkps://keys.openpgp.org --recv-keys "<fingerprint>"` Ensure that you put quotes around fingerprints containing spaces.

autogen.sh

@@ -14,3 +14,16 @@ fi
 command -v autoreconf >/dev/null || \
   (echo "configuration failed, please install autoconf first" && exit 1)
 autoreconf --install --force --warnings=all
+
+if expr "'$(build-aux/config.guess --timestamp)" \< "'$(depends/config.guess --timestamp)" > /dev/null; then
+  chmod ug+w build-aux/config.guess
+  chmod ug+w src/secp256k1/build-aux/config.guess
+  cp depends/config.guess build-aux
+  cp depends/config.guess src/secp256k1/build-aux
+fi
+if expr "'$(build-aux/config.sub --timestamp)" \< "'$(depends/config.sub --timestamp)" > /dev/null; then
+  chmod ug+w build-aux/config.sub
+  chmod ug+w src/secp256k1/build-aux/config.sub
+  cp depends/config.sub build-aux
+  cp depends/config.sub src/secp256k1/build-aux
+fi

build-aux/m4/bitcoin_qt.m4

@@ -70,8 +70,6 @@ AC_DEFUN([BITCOIN_QT_INIT],[
                  [qt_lib_suffix= ]); bitcoin_qt_want_version=qt5],
         [qt_lib_suffix= ])
 
-  AS_CASE([$host], [*android*], [qt_lib_suffix=_$ANDROID_ARCH])
-
   AC_ARG_WITH([qt-incdir],[AS_HELP_STRING([--with-qt-incdir=INC_DIR],[specify qt include path (overridden by pkgconfig)])], [qt_include_path=$withval], [])
   AC_ARG_WITH([qt-libdir],[AS_HELP_STRING([--with-qt-libdir=LIB_DIR],[specify qt lib path (overridden by pkgconfig)])], [qt_lib_path=$withval], [])
   AC_ARG_WITH([qt-plugindir],[AS_HELP_STRING([--with-qt-plugindir=PLUGIN_DIR],[specify qt plugin path (overridden by pkgconfig)])], [qt_plugin_path=$withval], [])
@@ -80,19 +78,10 @@ AC_DEFUN([BITCOIN_QT_INIT],[
 
   AC_ARG_WITH([qtdbus],
     [AS_HELP_STRING([--with-qtdbus],
-    [enable DBus support (default is yes if qt is enabled and QtDBus is found, except on Android)])],
+    [enable DBus support (default is yes if qt is enabled and QtDBus is found)])],
     [use_dbus=$withval],
     [use_dbus=auto])
 
-  dnl Android doesn't support D-Bus and certainly doesn't use it for notifications
-  case $host in
-    *android*)
-      if test "$use_dbus" != "yes"; then
-        use_dbus=no
-      fi
-    ;;
-  esac
-
   AC_SUBST(QT_TRANSLATION_DIR,$qt_translation_path)
 ])
 
@@ -132,16 +121,10 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[
       if test -d "$qt_plugin_path/accessible"; then
         QT_LIBS="$QT_LIBS -L$qt_plugin_path/accessible"
       fi
-      if test -d "$qt_plugin_path/platforms/android"; then
-        QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms/android -lqtfreetype -lEGL"
-      fi
     fi
 
-    AC_DEFINE([QT_STATICPLUGIN], [1], [Define this symbol if qt plugins are static])
-    if test "$TARGET_OS" != "android"; then
-      _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMinimalIntegrationPlugin], [-lqminimal])
-      AC_DEFINE([QT_QPA_PLATFORM_MINIMAL], [1], [Define this symbol if the minimal qt platform exists])
-    fi
+    _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMinimalIntegrationPlugin], [-lqminimal])
+    AC_DEFINE([QT_QPA_PLATFORM_MINIMAL], [1], [Define this symbol if the minimal qt platform exists])
     if test "$TARGET_OS" = "windows"; then
       dnl Linking against wtsapi32 is required. See #17749 and
       dnl https://bugreports.qt.io/browse/QTBUG-27097.
@@ -160,9 +143,6 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[
       _BITCOIN_QT_CHECK_STATIC_PLUGIN([QCocoaIntegrationPlugin], [-lqcocoa])
       _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMacStylePlugin], [-lqmacstyle])
       AC_DEFINE([QT_QPA_PLATFORM_COCOA], [1], [Define this symbol if the qt platform is cocoa])
-    elif test "$TARGET_OS" = "android"; then
-      QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lplugins_platforms_qtforandroid${qt_lib_suffix} -ljnigraphics -landroid -lqtfreetype${qt_lib_suffix} $QT_LIBS"
-      AC_DEFINE([QT_QPA_PLATFORM_ANDROID], [1], [Define this symbol if the qt platform is android])
     fi
   fi
   CPPFLAGS=$TEMP_CPPFLAGS
@@ -227,7 +207,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[
   BITCOIN_QT_PATH_PROGS([LUPDATE], [lupdate-qt5 lupdate5 lupdate],$qt_bin_path, yes)
   BITCOIN_QT_PATH_PROGS([LCONVERT], [lconvert-qt5 lconvert5 lconvert], $qt_bin_path, yes)
 
-  MOC_DEFS='-DHAVE_CONFIG_H -I$(srcdir)'
+  MOC_DEFS='-I$(srcdir)'
   case $host in
     *darwin*)
      BITCOIN_QT_CHECK([
@@ -357,9 +337,6 @@ AC_DEFUN([_BITCOIN_QT_CHECK_STATIC_LIBS], [
     PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport${qt_lib_suffix}], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"])
   elif test "$TARGET_OS" = "windows"; then
     PKG_CHECK_MODULES([QT_WINDOWSUIAUTOMATION], [${qt_lib_prefix}WindowsUIAutomationSupport${qt_lib_suffix}], [QT_LIBS="$QT_WINDOWSUIAUTOMATION_LIBS $QT_LIBS"])
-  elif test "$TARGET_OS" = "android"; then
-    PKG_CHECK_MODULES([QT_EGL], [${qt_lib_prefix}EglSupport${qt_lib_suffix}], [QT_LIBS="$QT_EGL_LIBS $QT_LIBS"])
-    PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport${qt_lib_suffix}], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"])
   fi
 ])
 

build-aux/m4/bitcoin_runtime_lib.m4

@@ -1,42 +0,0 @@
-# On some platforms clang builtin implementations
-# require compiler-rt as a runtime library to use.
-#
-# See:
-# - https://bugs.llvm.org/show_bug.cgi?id=28629
-
-m4_define([_CHECK_RUNTIME_testbody], [[
-  bool f(long long x, long long y, long long* p)
-  {
-    return __builtin_mul_overflow(x, y, p);
-  }
-  int main() { return 0; }
-]])
-
-AC_DEFUN([CHECK_RUNTIME_LIB], [
-
-  AC_LANG_PUSH([C++])
-
-  AC_MSG_CHECKING([for __builtin_mul_overflow])
-  AC_LINK_IFELSE(
-    [AC_LANG_SOURCE([_CHECK_RUNTIME_testbody])],
-    [
-      AC_MSG_RESULT([yes])
-      AC_DEFINE([HAVE_BUILTIN_MUL_OVERFLOW], [1], [Define if you have a working __builtin_mul_overflow])
-    ],
-    [
-      ax_check_save_flags="$LDFLAGS"
-      LDFLAGS="$LDFLAGS --rtlib=compiler-rt -lgcc_s"
-      AC_LINK_IFELSE(
-        [AC_LANG_SOURCE([_CHECK_RUNTIME_testbody])],
-        [
-          AC_MSG_RESULT([yes, with additional linker flags])
-          RUNTIME_LDFLAGS="--rtlib=compiler-rt -lgcc_s"
-          AC_DEFINE([HAVE_BUILTIN_MUL_OVERFLOW], [1], [Define if you have a working __builtin_mul_overflow])
-        ],
-        [AC_MSG_RESULT([no])])
-      LDFLAGS="$ax_check_save_flags"
-    ])
-
-  AC_LANG_POP
-  AC_SUBST([RUNTIME_LDFLAGS])
-])

build-aux/m4/l_atomic.m4

@@ -4,8 +4,10 @@ dnl permitted in any medium without royalty provided the copyright notice
 dnl and this notice are preserved. This file is offered as-is, without any
 dnl warranty.
 
-# Some versions of gcc/libstdc++ require linking with -latomic if
-# using the C++ atomic library.
+# Clang, when building for 32-bit,
+# and linking against libstdc++, requires linking with
+# -latomic if using the C++ atomic library.
+# Can be tested with: clang++ -std=c++20 test.cpp -m32
 #
 # Sourced from http://bugs.debian.org/797228
 
@@ -22,9 +24,14 @@ m4_define([_CHECK_ATOMIC_testbody], [[
 
     std::atomic<std::chrono::seconds> t{0s};
     t.store(2s);
+    auto t1 = t.load();
+    t.compare_exchange_strong(t1, 3s);
+
+    std::atomic<double> d{};
+    d.store(3.14);
+    auto d1 = d.load();
 
     std::atomic<int64_t> a{};
-
     int64_t v = 5;
     int64_t r = a.fetch_add(v);
     return static_cast<int>(r);

build-aux/m4/l_filesystem.m4

@@ -1,47 +0,0 @@
-dnl Copyright (c) 2022 The Bitcoin Core developers
-dnl Distributed under the MIT software license, see the accompanying
-dnl file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-# GCC 8.1 and earlier requires -lstdc++fs
-# Clang 8.0.0 (libc++) and earlier requires -lc++fs
-
-m4_define([_CHECK_FILESYSTEM_testbody], [[
-  #include <filesystem>
-
-  namespace fs = std::filesystem;
-
-  int main() {
-    (void)fs::current_path().root_name();
-    return 0;
-  }
-]])
-
-AC_DEFUN([CHECK_FILESYSTEM], [
-
-  AC_LANG_PUSH(C++)
-
-  AC_MSG_CHECKING([whether std::filesystem can be used without link library])
-
-  AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[
-      AC_MSG_RESULT([yes])
-    ],[
-      AC_MSG_RESULT([no])
-      SAVED_LIBS="$LIBS"
-      LIBS="$SAVED_LIBS -lstdc++fs"
-      AC_MSG_CHECKING([whether std::filesystem needs -lstdc++fs])
-      AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[
-          AC_MSG_RESULT([yes])
-        ],[
-          AC_MSG_RESULT([no])
-          AC_MSG_CHECKING([whether std::filesystem needs -lc++fs])
-          LIBS="$SAVED_LIBS -lc++fs"
-          AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_FILESYSTEM_testbody])],[
-            AC_MSG_RESULT([yes])
-        ],[
-            AC_MSG_FAILURE([cannot figure out how to use std::filesystem])
-          ])
-        ])
-    ])
-
-  AC_LANG_POP
-])

build_msvc/README.md

@@ -32,7 +32,9 @@ Qt
 ---------------------
 To build Bitcoin Core with the GUI, a static build of Qt is required.
 
-1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.5.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`.
+1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.11.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.11/single/qt-everywhere-opensource-src-5.15.11.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`.
+
+> 💡 **Tip:** If you use the default path with "Extract All" for the Qt source code zip file, and end up with something like `C:\dev\qt-everywhere-opensource-src-5.15.11\qt-everywhere-src-5.15.11`, you are likely to encounter a "path too long" error when building. To fix the problem move the source files to a shorter path such as the recommended `C:\dev\qt-source`.
 
 2. Open "x64 Native Tools Command Prompt for VS 2022", and input the following commands:
 ```cmd

build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in

@@ -10,9 +10,15 @@
   </PropertyGroup>
   <ItemGroup>
 @SOURCE_FILES@
+    <ClCompile Include="..\..\src\bench\coin_selection.cpp" />
+    <ClCompile Include="..\..\src\bench\wallet_balance.cpp" />
+    <ClCompile Include="..\..\src\bench\wallet_create.cpp" />
+    <ClCompile Include="..\..\src\bench\wallet_create_tx.cpp" />
+    <ClCompile Include="..\..\src\bench\wallet_ismine.cpp" />
+    <ClCompile Include="..\..\src\bench\wallet_loading.cpp" />
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj">
+    <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj">
       <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project>
     </ProjectReference>
     <ProjectReference Include="..\libbitcoin_common\libbitcoin_common.vcxproj">

build_msvc/bitcoin-qt/bitcoin-qt.vcxproj

@@ -13,7 +13,7 @@
     <ResourceCompile Include="..\..\src\qt\res\bitcoin-qt-res.rc" />
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj">
+    <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj">
       <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project>
     </ProjectReference>
     <ProjectReference Include="..\libbitcoin_cli\libbitcoin_cli.vcxproj">
@@ -58,11 +58,11 @@
     <Link>
       <SubSystem>Windows</SubSystem>
       <AdditionalDependencies>$(QtReleaseLibraries);%(AdditionalDependencies)</AdditionalDependencies>
-      <AdditionalOptions>/ignore:4206 /LTCG:OFF</AdditionalOptions>
+      <AdditionalOptions>/LTCG:OFF</AdditionalOptions>
     </Link>
     <ResourceCompile>
       <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>HAVE_CONFIG_H;_UNICODE;UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_UNICODE;UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ResourceCompile>
   </ItemDefinitionGroup>
 
@@ -72,11 +72,10 @@
     </ClCompile>
     <Link>
       <AdditionalDependencies>$(QtDebugLibraries);%(AdditionalDependencies)</AdditionalDependencies>
-      <AdditionalOptions>/ignore:4206</AdditionalOptions>
     </Link>
     <ResourceCompile>
       <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>HAVE_CONFIG_H;_UNICODE;UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_UNICODE;UNICODE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ResourceCompile>
   </ItemDefinitionGroup>
 

build_msvc/bitcoin-tx/bitcoin-tx.vcxproj

@@ -12,7 +12,7 @@
     <ClCompile Include="..\..\src\bitcoin-tx.cpp" />
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj">
+    <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj">
       <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project>
     </ProjectReference>
     <ProjectReference Include="..\libbitcoin_common\libbitcoin_common.vcxproj">

build_msvc/bitcoin-util/bitcoin-util.vcxproj

@@ -12,7 +12,7 @@
     <ClCompile Include="..\..\src\bitcoin-util.cpp" />
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj">
+    <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj">
       <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project>
     </ProjectReference>
     <ProjectReference Include="..\libbitcoin_common\libbitcoin_common.vcxproj">

build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj

@@ -15,7 +15,7 @@
     </ClCompile>
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj">
+    <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj">
       <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project>
     </ProjectReference>
     <ProjectReference Include="..\libbitcoin_common\libbitcoin_common.vcxproj">

build_msvc/bitcoin.sln

@@ -2,7 +2,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 16
 VisualStudioVersion = 16.0.28803.452
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoinconsensus", "libbitcoinconsensus\libbitcoinconsensus.vcxproj", "{2B384FA8-9EE1-4544-93CB-0D733C25E8CE}"
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_consensus", "libbitcoin_consensus\libbitcoin_consensus.vcxproj", "{2B384FA8-9EE1-4544-93CB-0D733C25E8CE}"
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoind", "bitcoind\bitcoind.vcxproj", "{D4513DDF-6013-44DC-ADCC-12EAF6D1F038}"
 EndProject
@@ -48,7 +48,9 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libtest_util", "libtest_uti
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_bitcoin-qt", "test_bitcoin-qt\test_bitcoin-qt.vcxproj", "{51201D5E-D939-4854-AE9D-008F03FF518E}"
 EndProject
-Project("{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}") = "libminisketch", "libminisketch\libminisketch.vcxproj", "{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}"
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libminisketch", "libminisketch\libminisketch.vcxproj", "{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzz", "fuzz\fuzz.vcxproj", "{AFCEE6C1-89FB-49AB-A694-BA580A59E2D8}"
 EndProject
 Global
     GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -152,6 +154,10 @@ Global
         {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Debug|x64.Build.0 = Debug|x64
         {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Release|x64.ActiveCfg = Release|x64
         {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Release|x64.Build.0 = Release|x64
+        {AFCEE6C1-89FB-49AB-A694-BA580A59E2D8}.Debug|x64.ActiveCfg = Debug|x64
+        {AFCEE6C1-89FB-49AB-A694-BA580A59E2D8}.Debug|x64.Build.0 = Debug|x64
+        {AFCEE6C1-89FB-49AB-A694-BA580A59E2D8}.Release|x64.ActiveCfg = Release|x64
+        {AFCEE6C1-89FB-49AB-A694-BA580A59E2D8}.Release|x64.Build.0 = Release|x64
     EndGlobalSection
     GlobalSection(SolutionProperties) = preSolution
         HideSolutionNode = FALSE

build_msvc/bitcoin_config.h.in

@@ -38,86 +38,17 @@
 /* Define to 1 to enable SQLite wallet */
 #define USE_SQLITE 1
 
-/* Define to 1 to enable ZMQ functions */
+/* Define this symbol to enable ZMQ functions */
 #define ENABLE_ZMQ 1
 
-/* define if external signer support is enabled (requires Boost::Process) */
-#define ENABLE_EXTERNAL_SIGNER /**/
-
-/* Define this symbol if the consensus lib has been built */
-#define HAVE_CONSENSUS_LIB 1
-
-/* Define to 1 if you have the declaration of `be16toh', and to 0 if you
-   don't. */
-#define HAVE_DECL_BE16TOH 0
-
-/* Define to 1 if you have the declaration of `be32toh', and to 0 if you
-   don't. */
-#define HAVE_DECL_BE32TOH 0
-
-/* Define to 1 if you have the declaration of `be64toh', and to 0 if you
-   don't. */
-#define HAVE_DECL_BE64TOH 0
-
-/* Define to 1 if you have the declaration of `bswap_16', and to 0 if you
-   don't. */
-#define HAVE_DECL_BSWAP_16 0
-
-/* Define to 1 if you have the declaration of `bswap_32', and to 0 if you
-   don't. */
-#define HAVE_DECL_BSWAP_32 0
-
-/* Define to 1 if you have the declaration of `bswap_64', and to 0 if you
-   don't. */
-#define HAVE_DECL_BSWAP_64 0
-
 /* Define to 1 if you have the declaration of `fork', and to 0 if you don't.
    */
 #define HAVE_DECL_FORK 0
 
-/* Define to 1 if you have the declaration of `htobe16', and to 0 if you
-   don't. */
-#define HAVE_DECL_HTOBE16 0
-
-/* Define to 1 if you have the declaration of `htobe32', and to 0 if you
-   don't. */
-#define HAVE_DECL_HTOBE32 0
-
-/* Define to 1 if you have the declaration of `htobe64', and to 0 if you
-   don't. */
-#define HAVE_DECL_HTOBE64 0
-
-/* Define to 1 if you have the declaration of `htole16', and to 0 if you
-   don't. */
-#define HAVE_DECL_HTOLE16 0
-
-/* Define to 1 if you have the declaration of `htole32', and to 0 if you
-   don't. */
-#define HAVE_DECL_HTOLE32 0
-
-/* Define to 1 if you have the declaration of `htole64', and to 0 if you
-   don't. */
-#define HAVE_DECL_HTOLE64 0
-
-/* Define to 1 if you have the declaration of `le16toh', and to 0 if you
-   don't. */
-#define HAVE_DECL_LE16TOH 0
-
-/* Define to 1 if you have the declaration of `le32toh', and to 0 if you
-   don't. */
-#define HAVE_DECL_LE32TOH 0
-
-/* Define to 1 if you have the declaration of `le64toh', and to 0 if you
-   don't. */
-#define HAVE_DECL_LE64TOH 0
-
 /* Define to 1 if you have the declaration of `setsid', and to 0 if you don't.
    */
 #define HAVE_DECL_SETSID 0
 
-/* Define if the dllexport attribute is supported. */
-#define HAVE_DLLEXPORT_ATTRIBUTE 1
-
 /* Define to the address where bug reports for this package should be sent. */
 #define PACKAGE_BUGREPORT "https://github.com/bitcoin/bitcoin/issues"
 
@@ -139,9 +70,6 @@
 /* Define this symbol if the qt platform is windows */
 #define QT_QPA_PLATFORM_WINDOWS 1
 
-/* Define this symbol if qt plugins are static */
-#define QT_STATICPLUGIN 1
-
 /* Windows Universal Platform constraints */
 #if !defined(WINAPI_FAMILY) || (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP)
 /* Either a desktop application without API restrictions, or and older system

build_msvc/bitcoind/bitcoind.vcxproj

@@ -15,7 +15,7 @@
     </ClCompile>
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj">
+    <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj">
       <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project>
     </ProjectReference>
     <ProjectReference Include="..\libbitcoin_common\libbitcoin_common.vcxproj">
@@ -73,16 +73,20 @@
                   Replace="@USE_SQLITE_TRUE@" By=""></ReplaceInFile>
     <ReplaceInFile FilePath="$(ConfigIniOut)"
                   Replace="@BUILD_BITCOIN_CLI_TRUE@" By=""></ReplaceInFile>
+    <ReplaceInFile FilePath="$(ConfigIniOut)"
+                  Replace="@BUILD_BITCOIN_UTIL_TRUE@" By=""></ReplaceInFile>
     <ReplaceInFile FilePath="$(ConfigIniOut)"
                   Replace="@BUILD_BITCOIN_WALLET_TRUE@" By=""></ReplaceInFile>
     <ReplaceInFile FilePath="$(ConfigIniOut)"
                   Replace="@BUILD_BITCOIND_TRUE@" By=""></ReplaceInFile>
     <ReplaceInFile FilePath="$(ConfigIniOut)"
-                  Replace="@ENABLE_FUZZ_TRUE@" By=""></ReplaceInFile>
+                  Replace="@ENABLE_FUZZ_BINARY_TRUE@" By=""></ReplaceInFile>
     <ReplaceInFile FilePath="$(ConfigIniOut)"
                   Replace="@ENABLE_ZMQ_TRUE@" By=""></ReplaceInFile>
     <ReplaceInFile FilePath="$(ConfigIniOut)"
-                  Replace="@ENABLE_EXTERNAL_SIGNER_TRUE@" By=""></ReplaceInFile>
+                  Replace="@ENABLE_EXTERNAL_SIGNER_TRUE@" By="#"></ReplaceInFile>
+    <ReplaceInFile FilePath="$(ConfigIniOut)"
+                  Replace="@ENABLE_USDT_TRACEPOINTS_TRUE@" By="#"></ReplaceInFile>
   </Target>
   <Import Project="..\common.vcxproj" />
 </Project>

build_msvc/common.init.vcxproj.in

@@ -57,7 +57,7 @@
 
 <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
     <ClCompile>
-      <Optimization>Disabled</Optimization>
+      <Optimization>MaxSpeed</Optimization>
       <WholeProgramOptimization>false</WholeProgramOptimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
@@ -87,10 +87,10 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <PrecompiledHeader>NotUsing</PrecompiledHeader>
-      <AdditionalOptions>/utf-8 /Zc:__cplusplus /std:c++20 %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalOptions>/utf-8 /Zc:preprocessor /Zc:__cplusplus /std:c++20 %(AdditionalOptions)</AdditionalOptions>
       <DisableSpecificWarnings>4018;4244;4267;4715;4805</DisableSpecificWarnings>
       <TreatWarningAsError>true</TreatWarningAsError>
-      <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;SECP256K1_STATIC;ZMQ_STATIC;NOMINMAX;WIN32;_CRT_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;PROVIDE_FUZZ_MAIN_FUNCTION;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <AdditionalIncludeDirectories>..\..\src;..\..\src\minisketch\include;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
@@ -98,9 +98,6 @@
       <AdditionalDependencies>Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <RandomizedBaseAddress>true</RandomizedBaseAddress>
     </Link>
-    <Lib>
-      <AdditionalOptions>/ignore:4221</AdditionalOptions>
-    </Lib>
   </ItemDefinitionGroup>
   <Import Project="common.init.vcxproj.user" Condition="Exists('common.init.vcxproj.user')" />
 </Project>

build_msvc/common.qt.init.vcxproj

@@ -13,4 +13,10 @@
     <QtDebugLibraries>$(QtPluginsLibraryDir)\platforms\qwindowsd.lib;$(QtPluginsLibraryDir)\platforms\qminimald.lib;$(QtPluginsLibraryDir)\styles\qwindowsvistastyled.lib;$(QtLibraryDir)\*d.lib;Wtsapi32.lib;crypt32.lib;userenv.lib;netapi32.lib;imm32.lib;Dwmapi.lib;version.lib;winmm.lib;UxTheme.lib</QtDebugLibraries>
   </PropertyGroup>
 
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <PreprocessorDefinitions>QT_NO_KEYWORDS;QT_USE_QSTRINGBUILDER;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
+
 </Project>

build_msvc/fuzz/fuzz.vcxproj

@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="..\common.init.vcxproj" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{AFCEE6C1-89FB-49AB-A694-BA580A59E2D8}</ProjectGuid>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
+  </PropertyGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\src\test\fuzz\*.cpp" Exclude="..\..\src\test\fuzz\utxo_snapshot.cpp" />
+    <ClCompile Include="..\..\src\test\fuzz\util\descriptor.cpp">
+      <ObjectFileName>$(IntDir)test_fuzz_util_descriptor.obj</ObjectFileName>
+    </ClCompile>
+    <ClCompile Include="..\..\src\test\fuzz\util\mempool.cpp">
+      <ObjectFileName>$(IntDir)test_fuzz_util_mempool.obj</ObjectFileName>
+    </ClCompile>
+    <ClCompile Include="..\..\src\test\fuzz\util\net.cpp">
+      <ObjectFileName>$(IntDir)test_fuzz_util_net.obj</ObjectFileName>
+    </ClCompile>
+    <ClCompile Include="..\..\src\wallet\test\fuzz\coincontrol.cpp">
+      <ObjectFileName>$(IntDir)wallet_test_fuzz_coincontrol.obj</ObjectFileName>
+    </ClCompile>
+    <ClCompile Include="..\..\src\wallet\test\fuzz\coinselection.cpp">
+      <ObjectFileName>$(IntDir)wallet_test_fuzz_coinselection.obj</ObjectFileName>
+    </ClCompile>
+    <ClCompile Include="..\..\src\wallet\test\fuzz\fees.cpp">
+      <ObjectFileName>$(IntDir)wallet_test_fuzz_fees.obj</ObjectFileName>
+    </ClCompile>
+    <ClCompile Include="..\..\src\wallet\test\fuzz\notifications.cpp">
+      <ObjectFileName>$(IntDir)wallet_test_fuzz_notifications.obj</ObjectFileName>
+    </ClCompile>
+    <ClCompile Include="..\..\src\wallet\test\fuzz\parse_iso8601.cpp">
+      <ObjectFileName>$(IntDir)wallet_test_fuzz_parse_iso8601.obj</ObjectFileName>
+    </ClCompile>
+    <ClCompile Include="..\..\src\wallet\test\fuzz\scriptpubkeyman.cpp">
+      <ObjectFileName>$(IntDir)wallet_test_fuzz_scriptpubkeyman.obj</ObjectFileName>
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\libminisketch\libminisketch.vcxproj">
+      <Project>{542007e3-be0d-4b0d-a6b0-aa8813e2558d}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj">
+      <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\libbitcoin_cli\libbitcoin_cli.vcxproj">
+      <Project>{0667528c-d734-4009-adf9-c0d6c4a5a5a6}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\libbitcoin_common\libbitcoin_common.vcxproj">
+      <Project>{7c87e378-df58-482e-aa2f-1bc129bc19ce}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\libbitcoin_crypto\libbitcoin_crypto.vcxproj">
+      <Project>{6190199c-6cf4-4dad-bfbd-93fa72a760c1}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\libbitcoin_node\libbitcoin_node.vcxproj">
+      <Project>{460fee33-1fe1-483f-b3bf-931ff8e969a5}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\libbitcoin_util\libbitcoin_util.vcxproj">
+      <Project>{b53a5535-ee9d-4c6f-9a26-f79ee3bc3754}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\libbitcoin_wallet\libbitcoin_wallet.vcxproj">
+      <Project>{93b86837-b543-48a5-a89b-7c87abb77df2}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\libbitcoin_zmq\libbitcoin_zmq.vcxproj">
+      <Project>{792d487f-f14c-49fc-a9de-3fc150f31c3f}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\libtest_util\libtest_util.vcxproj">
+      <Project>{1e065f03-3566-47d0-8fa9-daa72b084e7d}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\libunivalue\libunivalue.vcxproj">
+      <Project>{5724ba7d-a09a-4ba8-800b-c4c1561b3d69}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\libsecp256k1\libsecp256k1.vcxproj">
+      <Project>{bb493552-3b8c-4a8c-bf69-a6e7a51d2ea6}</Project>
+    </ProjectReference>
+    <ProjectReference Include="..\libleveldb\libleveldb.vcxproj">
+      <Project>{18430fef-6b61-4c53-b396-718e02850f1b}</Project>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <Import Project="..\common.vcxproj" />
+</Project>

build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in

@@ -8,7 +8,6 @@
     <ConfigurationType>StaticLibrary</ConfigurationType>
   </PropertyGroup>
   <ItemGroup>
-    <ClCompile Include="..\..\src\common\url.cpp" />
 @SOURCE_FILES@
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />

build_msvc/libbitcoin_consensus/libbitcoin_consensus.vcxproj

@@ -11,20 +11,10 @@
     <ClCompile Include="..\..\src\arith_uint256.cpp" />
     <ClCompile Include="..\..\src\consensus\merkle.cpp" />
     <ClCompile Include="..\..\src\consensus\tx_check.cpp" />
-    <ClCompile Include="..\..\src\crypto\aes.cpp" />
-    <ClCompile Include="..\..\src\crypto\chacha20.cpp" />
-    <ClCompile Include="..\..\src\crypto\hmac_sha256.cpp" />
-    <ClCompile Include="..\..\src\crypto\hmac_sha512.cpp" />
-    <ClCompile Include="..\..\src\crypto\ripemd160.cpp" />
-    <ClCompile Include="..\..\src\crypto\sha1.cpp" />
-    <ClCompile Include="..\..\src\crypto\sha256.cpp" />
-    <ClCompile Include="..\..\src\crypto\sha256_sse4.cpp" />
-    <ClCompile Include="..\..\src\crypto\sha512.cpp" />
     <ClCompile Include="..\..\src\hash.cpp" />
     <ClCompile Include="..\..\src\primitives\block.cpp" />
     <ClCompile Include="..\..\src\primitives\transaction.cpp" />
     <ClCompile Include="..\..\src\pubkey.cpp" />
-    <ClCompile Include="..\..\src\script\bitcoinconsensus.cpp" />
     <ClCompile Include="..\..\src\script\interpreter.cpp" />
     <ClCompile Include="..\..\src\script\script.cpp" />
     <ClCompile Include="..\..\src\script\script_error.cpp" />

build_msvc/libleveldb/libleveldb.vcxproj

@@ -51,7 +51,7 @@
   <ItemDefinitionGroup>
      <ClCompile>
        <PreprocessorDefinitions>HAVE_CRC32C=0;HAVE_SNAPPY=0;LEVELDB_IS_BIG_ENDIAN=0;_UNICODE;UNICODE;_CRT_NONSTDC_NO_DEPRECATE;LEVELDB_PLATFORM_WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-       <DisableSpecificWarnings>4244;4267</DisableSpecificWarnings>
+       <DisableSpecificWarnings>4244;4267;4722</DisableSpecificWarnings>
        <AdditionalIncludeDirectories>..\..\src\leveldb;..\..\src\leveldb\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
      </ClCompile>
   </ItemDefinitionGroup>

build_msvc/libminisketch/libminisketch.vcxproj

@@ -28,7 +28,7 @@
   </ItemGroup>
   <ItemDefinitionGroup>
     <ClCompile>
-      <DisableSpecificWarnings>4060;4065;4146;4244;4267;4554</DisableSpecificWarnings>
+      <DisableSpecificWarnings>4060;4065;4146;4244;4267</DisableSpecificWarnings>
       <PreprocessorDefinitions>HAVE_CLMUL;DISABLE_DEFAULT_FIELDS;ENABLE_FIELD_32;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
   </ItemDefinitionGroup>

build_msvc/libsecp256k1/libsecp256k1.vcxproj

@@ -14,9 +14,10 @@
   </ItemGroup>
   <ItemDefinitionGroup>
     <ClCompile>
-      <PreprocessorDefinitions>ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;ENABLE_MODULE_ELLSWIFT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <UndefinePreprocessorDefinitions>USE_ASM_X86_64;%(UndefinePreprocessorDefinitions)</UndefinePreprocessorDefinitions>
       <AdditionalIncludeDirectories>..\..\src\secp256k1;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <DisableSpecificWarnings>4146;4244;4267;4334</DisableSpecificWarnings>
+      <DisableSpecificWarnings>4146;4244;4267</DisableSpecificWarnings>
     </ClCompile>
   </ItemDefinitionGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />

build_msvc/libsecp256k1_config.h

@@ -1,15 +0,0 @@
-/**********************************************************************
- * Copyright (c) 2013, 2014 Pieter Wuille                             *
- * Distributed under the MIT software license, see the accompanying   *
- * file COPYING or http://www.opensource.org/licenses/mit-license.php.*
- **********************************************************************/
-
-#ifndef BITCOIN_LIBSECP256K1_CONFIG_H
-#define BITCOIN_LIBSECP256K1_CONFIG_H
-
-#undef USE_ASM_X86_64
-
-#define ECMULT_GEN_PREC_BITS 4
-#define ECMULT_WINDOW_SIZE 15
-
-#endif // BITCOIN_LIBSECP256K1_CONFIG_H

build_msvc/libtest_util/libtest_util.vcxproj.in

@@ -8,6 +8,7 @@
     <ConfigurationType>StaticLibrary</ConfigurationType>
   </PropertyGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\src\wallet\test\util.cpp" />
 @SOURCE_FILES@
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />

build_msvc/msvc-autogen.py

@@ -111,7 +111,6 @@ def main():
         set_properties(vcxproj_filename, '@SOURCE_FILES@\n', content)
     parse_config_into_btc_config()
     copyfile(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), os.path.join(SOURCE_DIR, 'config/bitcoin-config.h'))
-    copyfile(os.path.join(SOURCE_DIR,'../build_msvc/libsecp256k1_config.h'), os.path.join(SOURCE_DIR, 'secp256k1/src/libsecp256k1-config.h'))
 
 if __name__ == '__main__':
     main()

build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj

@@ -9,7 +9,6 @@
   </PropertyGroup>
   <ItemGroup>
     <ClCompile Include="..\..\src\init\bitcoin-qt.cpp" />
-    <ClCompile Include="..\..\src\test\util\setup_common.cpp" />
     <ClCompile Include="..\..\src\qt\test\addressbooktests.cpp" />
     <ClCompile Include="..\..\src\qt\test\apptests.cpp" />
     <ClCompile Include="..\..\src\qt\test\optiontests.cpp" />
@@ -27,7 +26,7 @@
     <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_wallettests.cpp" />
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj">
+    <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj">
       <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project>
     </ProjectReference>
     <ProjectReference Include="..\libbitcoin_cli\libbitcoin_cli.vcxproj">
@@ -77,7 +76,7 @@
     </ClCompile>
     <Link>
       <AdditionalDependencies>$(QtLibraryDir)\Qt5Test.lib;$(QtReleaseLibraries);%(AdditionalDependencies)</AdditionalDependencies>
-      <AdditionalOptions>/ignore:4206 /LTCG:OFF</AdditionalOptions>
+      <AdditionalOptions>/LTCG:OFF</AdditionalOptions>
     </Link>
   </ItemDefinitionGroup>
 
@@ -87,7 +86,6 @@
     </ClCompile>
     <Link>
       <AdditionalDependencies>$(QtDebugLibraries);%(AdditionalDependencies)</AdditionalDependencies>
-      <AdditionalOptions>/ignore:4206</AdditionalOptions>
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>

build_msvc/test_bitcoin/test_bitcoin.vcxproj

@@ -10,7 +10,7 @@
   </PropertyGroup>
   <ItemGroup>
     <ClCompile Include="..\..\src\test\*_properties.cpp" />
-    <ClCompile Include="..\..\src\test\*_tests.cpp" />
+    <ClCompile Include="..\..\src\test\*_tests.cpp" Exclude="..\..\src\test\ipc_tests.cpp" />
     <ClCompile Include="..\..\src\test\gen\*_gen.cpp" />
     <ClCompile Include="..\..\src\test\main.cpp" />
     <ClCompile Include="..\..\src\test\util\*.cpp" />
@@ -22,7 +22,7 @@
     <ProjectReference Include="..\libminisketch\libminisketch.vcxproj">
       <Project>{542007e3-be0d-4b0d-a6b0-aa8813e2558d}</Project>
     </ProjectReference>
-    <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj">
+    <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj">
       <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project>
     </ProjectReference>
     <ProjectReference Include="..\libbitcoin_cli\libbitcoin_cli.vcxproj">
@@ -68,7 +68,7 @@
       <RawTestFile Include="..\..\src\test\data\*.raw" />
     </ItemGroup>
     <HeaderFromHexdump RawFilePath="%(RawTestFile.FullPath)" HeaderFilePath="%(RawTestFile.FullPath).h" SourceHeader="static unsigned const char %(RawTestFile.Filename)_raw[] = {" SourceFooter="};" />
-    <HeaderFromHexdump RawFilePath="%(JsonTestFile.FullPath)" HeaderFilePath="%(JsonTestFile.FullPath).h" SourceHeader="namespace json_tests{ static unsigned const char %(JsonTestFile.Filename)[] = {" SourceFooter="};}" />
+    <HeaderFromHexdump RawFilePath="%(JsonTestFile.FullPath)" HeaderFilePath="%(JsonTestFile.FullPath).h" SourceHeader="#include &lt;string&gt;&#x0D;&#x0A;namespace json_tests{ static const std::string %(JsonTestFile.Filename){" SourceFooter="};}" />
   </Target>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />

build_msvc/vcpkg.json

@@ -3,18 +3,15 @@
   "version-string": "1",
   "dependencies": [
     "berkeleydb",
+    "boost-date-time",
     "boost-multi-index",
-    "boost-process",
     "boost-signals2",
     "boost-test",
+    "libevent",
     "sqlite3",
-    {
-      "name": "libevent",
-      "features": ["thread"]
-    },
     "zeromq"
   ],
-  "builtin-baseline": "f14984af3738e69f197bf0e647a8dca12de92996",
+  "builtin-baseline": "9edb1b8e590cc086563301d735cae4b6e732d2d2",
   "overrides": [
     {
       "name": "libevent",

ci/README.md

@@ -14,22 +14,17 @@ testing compared to other parts of the codebase. If you want to keep the work tr
 system in a virtual machine with a Linux operating system of your choice.
 
 To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage
-requires `docker` to be installed. To install all requirements on Ubuntu, run
+requires `bash`, `docker`, and `python3` to be installed. To run on different architectures than the host `qemu` is also required. To install all requirements on Ubuntu, run
 
 ```
-sudo apt install docker.io bash
+sudo apt install bash docker.io python3 qemu-user-static
 ```
 
-To run the default test stage,
+It is recommended to run the ci system in a clean env. To run the test stage
+with a specific configuration,
 
 ```
-./ci/test_run_all.sh
-```
-
-To run the test stage with a specific configuration,
-
-```
-FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
+env -i HOME="$HOME" PATH="$PATH" USER="$USER" bash -c 'FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh'
 ```
 
 ### Configurations
@@ -44,14 +39,11 @@ the system package manager to install build dependencies. This guarantees that
 the tester is using the same versions as the release builds, which also use
 `./depends`.
 
-If no `FILE_ENV` has been specified or values are left out, `00_setup_env.sh`
-is used as the default configuration with fallback values.
-
 It is also possible to force a specific configuration without modifying the
 file. For example,
 
 ```
-MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
+env -i HOME="$HOME" PATH="$PATH" USER="$USER" bash -c 'MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh'
 ```
 
 The files starting with `0n` (`n` greater than 0) are the scripts that are run
@@ -60,5 +52,5 @@ in order.
 ### Cache
 
 In order to avoid rebuilding all dependencies for each build, the binaries are
-cached and re-used when possible. Changes in the dependency-generator will
+cached and reused when possible. Changes in the dependency-generator will
 trigger cache-invalidation and rebuilds as necessary.

ci/lint/04_install.sh

@@ -1,45 +1,69 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018-2022 The Bitcoin Core developers
+# Copyright (c) 2018-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C
 
+export CI_RETRY_EXE="/ci_retry --"
+
+pushd "/"
+
 ${CI_RETRY_EXE} apt-get update
 # Lint dependencies:
+# - automake pkg-config libtool (for lint_includes_build_config)
 # - curl/xz-utils (to install shellcheck)
 # - git (used in many lint scripts)
 # - gpg (used by verify-commits)
-${CI_RETRY_EXE} apt-get install -y curl xz-utils git gpg
+${CI_RETRY_EXE} apt-get install -y automake pkg-config libtool curl xz-utils git gpg
 
-if [ -z "${SKIP_PYTHON_INSTALL}" ]; then
-    PYTHON_PATH=/tmp/python
-    if [ ! -d "${PYTHON_PATH}/bin" ]; then
-      (
-        git clone https://github.com/pyenv/pyenv.git
-        cd pyenv/plugins/python-build || exit 1
-        ./install.sh
-      )
-      # For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment
-      ${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \
-        libbz2-dev libreadline-dev libsqlite3-dev curl llvm \
-        libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
-        clang
-      env CC=clang python-build "$(cat "${BASE_ROOT_DIR}/.python-version")" "${PYTHON_PATH}"
-    fi
-    export PATH="${PYTHON_PATH}/bin:${PATH}"
-    command -v python3
-    python3 --version
+PYTHON_PATH="/python_build"
+if [ ! -d "${PYTHON_PATH}/bin" ]; then
+  (
+    ${CI_RETRY_EXE} git clone --depth=1 https://github.com/pyenv/pyenv.git
+    cd pyenv/plugins/python-build || exit 1
+    ./install.sh
+  )
+  # For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment
+  ${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \
+    libbz2-dev libreadline-dev libsqlite3-dev curl llvm \
+    libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
+    clang
+  env CC=clang python-build "$(cat "/.python-version")" "${PYTHON_PATH}"
+fi
+export PATH="${PYTHON_PATH}/bin:${PATH}"
+command -v python3
+python3 --version
+
+export LINT_RUNNER_PATH="/lint_test_runner"
+if [ ! -d "${LINT_RUNNER_PATH}" ]; then
+  ${CI_RETRY_EXE} apt-get install -y cargo
+  (
+    cd "/test/lint/test_runner" || exit 1
+    cargo build
+    mkdir -p "${LINT_RUNNER_PATH}"
+    mv target/debug/test_runner "${LINT_RUNNER_PATH}"
+  )
 fi
 
-${CI_RETRY_EXE} pip3 install codespell==2.2.1
-${CI_RETRY_EXE} pip3 install flake8==5.0.4
-${CI_RETRY_EXE} pip3 install mypy==0.971
-${CI_RETRY_EXE} pip3 install pyzmq==24.0.1
-${CI_RETRY_EXE} pip3 install vulture==2.6
+${CI_RETRY_EXE} pip3 install \
+  codespell==2.2.6 \
+  flake8==6.1.0 \
+  lief==0.13.2 \
+  mypy==1.4.1 \
+  pyzmq==25.1.0 \
+  ruff==0.5.5 \
+  vulture==2.6
 
 SHELLCHECK_VERSION=v0.8.0
 curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | \
     tar --xz -xf - --directory /tmp/
 mv "/tmp/shellcheck-${SHELLCHECK_VERSION}/shellcheck" /usr/bin/
+
+MLC_VERSION=v0.18.0
+MLC_BIN=mlc-x86_64-linux
+curl -sL "https://github.com/becheran/mlc/releases/download/${MLC_VERSION}/${MLC_BIN}" -o "/usr/bin/mlc"
+chmod +x /usr/bin/mlc
+
+popd || exit

ci/lint/06_script.sh

@@ -6,30 +6,27 @@
 
 export LC_ALL=C
 
-if [ -n "$LOCAL_BRANCH" ]; then
-  # To faithfully recreate CI linting locally, specify all commits on the current
-  # branch.
-  COMMIT_RANGE="$(git merge-base HEAD master)..HEAD"
-elif [ -n "$CIRRUS_PR" ]; then
+set -ex
+
+if [ -n "$CIRRUS_PR" ]; then
   COMMIT_RANGE="HEAD~..HEAD"
-  echo
-  git log --no-merges --oneline "$COMMIT_RANGE"
-  echo
-  test/lint/commit-script-check.sh "$COMMIT_RANGE"
+  if [ "$(git rev-list -1 HEAD)" != "$(git rev-list -1 --merges HEAD)" ]; then
+    echo "Error: The top commit must be a merge commit, usually the remote 'pull/${PR_NUMBER}/merge' branch."
+    false
+  fi
 else
-  COMMIT_RANGE="SKIP_EMPTY_NOT_A_PR"
+  # Otherwise, assume that a merge commit exists. This merge commit is assumed
+  # to be the base, after which linting will be done. If the merge commit is
+  # HEAD, the range will be empty.
+  COMMIT_RANGE="$( git rev-list --max-count=1 --merges HEAD )..HEAD"
 fi
 export COMMIT_RANGE
 
-# This only checks that the trees are pure subtrees, it is not doing a full
-# check with -r to not have to fetch all the remotes.
-test/lint/git-subtree-check.sh src/crypto/ctaes
-test/lint/git-subtree-check.sh src/secp256k1
-test/lint/git-subtree-check.sh src/minisketch
-test/lint/git-subtree-check.sh src/leveldb
-test/lint/git-subtree-check.sh src/crc32c
-test/lint/check-doc.py
-test/lint/all-lint.py
+echo
+git log --no-merges --oneline "$COMMIT_RANGE"
+echo
+test/lint/commit-script-check.sh "$COMMIT_RANGE"
+RUST_BACKTRACE=1 "${LINT_RUNNER_PATH}/test_runner"
 
 if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; then
     # Sanity check only the last few commits to get notified of missing sigs,

ci/lint/Dockerfile

@@ -1,29 +0,0 @@
-# See test/lint/README.md for usage.
-#
-# This container basically has to live in this directory in order to pull in the CI
-# install scripts. If it lived in the root directory, it would have to pull in the
-# entire repo as docker context during build; if it lived elsewhere, it wouldn't be
-# able to make back-references to pull in the install scripts. So here it lives.
-
-FROM python:3.7-buster
-
-ENV DEBIAN_FRONTEND=noninteractive
-ENV LC_ALL=C.UTF-8
-
-# This is used by the 04_install.sh script; we can't read the Python version from
-# .python-version for the same reasons as above, and it's more efficient to pull a
-# preexisting Python image than it is to build from source.
-ENV SKIP_PYTHON_INSTALL=1
-
-# Must be built from ./ci/lint/ for these paths to work.
-COPY ./docker-entrypoint.sh /entrypoint.sh
-COPY ./04_install.sh /install.sh
-
-RUN /install.sh && \
-  echo 'alias lint="./ci/lint/06_script.sh"' >> ~/.bashrc && \
-  chmod 755 /entrypoint.sh && \
-  rm -rf /var/lib/apt/lists/*
-
-
-WORKDIR /bitcoin
-ENTRYPOINT ["/entrypoint.sh"]

ci/lint/container-entrypoint.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit/.
+
+export LC_ALL=C
+
+# Fixes permission issues when there is a container UID/GID mismatch with the owner
+# of the mounted bitcoin src dir.
+git config --global --add safe.directory /bitcoin
+
+export PATH="/python_build/bin:${PATH}"
+export LINT_RUNNER_PATH="/lint_test_runner"
+
+if [ -z "$1" ]; then
+  bash -ic "./ci/lint/06_script.sh"
+else
+  exec "$@"
+fi

ci/lint/docker-entrypoint.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-export LC_ALL=C
-
-# Fixes permission issues when there is a container UID/GID mismatch with the owner
-# of the mounted bitcoin src dir.
-git config --global --add safe.directory /bitcoin
-
-if [ -z "$1" ]; then
-  LOCAL_BRANCH=1 bash -ic "./ci/lint/06_script.sh"
-else
-  exec "$@"
-fi

ci/lint_imagefile

@@ -0,0 +1,25 @@
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit/.
+
+# See test/lint/README.md for usage.
+
+FROM docker.io/debian:bookworm
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LC_ALL=C.UTF-8
+
+COPY ./ci/retry/retry /ci_retry
+COPY ./.python-version /.python-version
+COPY ./ci/lint/container-entrypoint.sh /entrypoint.sh
+COPY ./ci/lint/04_install.sh /install.sh
+COPY ./test/lint/test_runner /test/lint/test_runner
+
+RUN /install.sh && \
+  echo 'alias lint="./ci/lint/06_script.sh"' >> ~/.bashrc && \
+  chmod 755 /entrypoint.sh && \
+  rm -rf /var/lib/apt/lists/*
+
+
+WORKDIR /bitcoin
+ENTRYPOINT ["/entrypoint.sh"]

ci/lint_run_all.sh

@@ -1,11 +1,17 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-set -o errexit; source ./ci/test/00_setup_env.sh
+# Only used in .cirrus.yml. Refer to test/lint/README.md on how to run locally.
+
+cp "./ci/retry/retry" "/ci_retry"
+cp "./.python-version" "/.python-version"
+mkdir --parents "/test/lint"
+cp --recursive "./test/lint/test_runner" "/test/lint/"
 set -o errexit; source ./ci/lint/04_install.sh
-set -o errexit; source ./ci/lint/06_script.sh
+set -o errexit
+./ci/lint/06_script.sh

ci/test/00_setup_env.sh

@@ -6,16 +6,25 @@
 
 export LC_ALL=C.UTF-8
 
-# The root dir.
+set -ex
+
+# The source root dir, usually from git, usually read-only.
 # The ci system copies this folder.
-BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
-export BASE_ROOT_DIR
+BASE_READ_ONLY_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
+export BASE_READ_ONLY_DIR
+# The destination root dir inside the container.
+# This folder will also hold any SDKs.
+# This folder only exists on the ci guest and will be a copy of BASE_READ_ONLY_DIR
+export BASE_ROOT_DIR="${BASE_ROOT_DIR:-/ci_container_base}"
 # The depends dir.
 # This folder exists only on the ci guest, and on the ci host as a volume.
 export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends}
-# A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
-# This folder only exists on the ci host.
+# A folder for the ci system to put temporary files (build result, datadirs for tests, ...)
+# This folder only exists on the ci guest.
 export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch}
+# A folder for the ci system to put executables.
+# This folder only exists on the ci guest.
+export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/"
 
 echo "Setting specific values in env"
 if [ -n "${FILE_ENV}" ]; then
@@ -27,48 +36,37 @@ fi
 echo "Fallback to default values in env (if not yet set)"
 # The number of parallel jobs to pass down to make and test_runner.py
 export MAKEJOBS=${MAKEJOBS:--j4}
-# What host to compile for. See also ./depends/README.md
-# Tests that need cross-compilation export the appropriate HOST.
-# Tests that run natively guess the host
-export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")}
 # Whether to prefer BusyBox over GNU utilities
 export USE_BUSY_BOX=${USE_BUSY_BOX:-false}
 
 export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
 export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true}
 export RUN_TIDY=${RUN_TIDY:-false}
-export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false}
 # By how much to scale the test_runner timeouts (option --timeout-factor).
 # This is needed because some ci machines have slow CPU or disk, so sanitizers
 # might be slow or a reindex might be waiting on disk IO.
 export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-40}
-export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-}
 export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false}
 
-export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed}
-export CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG:-ubuntu:20.04}
 # Randomize test order.
 # See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/random.html
 export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1}
 # See man 7 debconf
 export DEBIAN_FRONTEND=noninteractive
-export CCACHE_SIZE=${CCACHE_SIZE:-100M}
+export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:-100M}
 export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
 export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
 # The cache dir.
 # This folder exists only on the ci guest, and on the ci host as a volume.
 export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache}
 # Folder where the build result is put (bin and lib).
-export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST}
+export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out}
 # Folder where the build is done (dist and out-of-tree build).
 export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build}
 # The folder for previous release binaries.
 # This folder exists only on the ci guest, and on the ci host as a volume.
-export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST}
-export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu"
-export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
-export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison}
+export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases}
+export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison e2fsprogs cmake}
 export GOAL=${GOAL:-install}
 export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets}
-export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH
 export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"}

ci/test/00_setup_env_android.sh

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019-2022 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-export HOST=aarch64-linux-android
-export PACKAGES="unzip openjdk-8-jdk gradle"
-export CONTAINER_NAME=ci_android
-export CI_IMAGE_NAME_TAG="ubuntu:jammy"
-
-export RUN_UNIT_TESTS=false
-export RUN_FUNCTIONAL_TESTS=false
-
-export ANDROID_API_LEVEL=28
-export ANDROID_BUILD_TOOLS_VERSION=28.0.3
-export ANDROID_NDK_VERSION=23.2.8568313
-export ANDROID_TOOLS_URL=https://dl.google.com/android/repository/commandlinetools-linux-8512546_latest.zip
-export ANDROID_HOME="${DEPENDS_DIR}/SDKs/android"
-export ANDROID_NDK_HOME="${ANDROID_HOME}/ndk/${ANDROID_NDK_VERSION}"
-export DEP_OPTS="ANDROID_SDK=${ANDROID_HOME} ANDROID_NDK=${ANDROID_NDK_HOME} ANDROID_API_LEVEL=${ANDROID_API_LEVEL} ANDROID_TOOLCHAIN_BIN=${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/"
-
-export BITCOIN_CONFIG="--disable-tests --enable-gui-tests --disable-bench --disable-fuzz-binary --without-utils --without-libs --without-daemon"

ci/test/00_setup_env_arm.sh

@@ -1,28 +1,20 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2021 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
 export HOST=arm-linux-gnueabihf
-# The host arch is unknown, so we run the tests through qemu.
-# If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string.
-if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-arm -L /usr/arm-linux-gnueabihf/"}"; fi
 export DPKG_ADD_ARCH="armhf"
 export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf"
-if [ -n "$QEMU_USER_CMD" ]; then
-  # Likely cross-compiling, so install the needed gcc and qemu-user
-  export PACKAGES="$PACKAGES qemu-user"
-fi
 export CONTAINER_NAME=ci_arm_linux
-# Use debian to avoid 404 apt errors when cross compiling
-export CI_IMAGE_NAME_TAG="debian:bullseye"
+export CI_IMAGE_NAME_TAG="docker.io/arm64v8/debian:bookworm"  # Check that https://packages.debian.org/bookworm/g++-arm-linux-gnueabihf (version 12.2, similar to guix) can cross-compile
 export USE_BUSY_BOX=true
 export RUN_UNIT_TESTS=true
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="install"
 # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1"
 # This could be removed once the ABI change warning does not show up by default
-export BITCOIN_CONFIG="--enable-reduce-exports CXXFLAGS=-Wno-psabi"
+export BITCOIN_CONFIG="--enable-reduce-exports CXXFLAGS='-Wno-psabi -Wno-error=maybe-uninitialized'"

ci/test/00_setup_env_i686_centos.sh

@@ -8,11 +8,10 @@ export LC_ALL=C.UTF-8
 
 export HOST=i686-pc-linux-gnu
 export CONTAINER_NAME=ci_i686_centos
-export CI_IMAGE_NAME_TAG=quay.io/centos/centos:stream8
-export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python38 python38-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison"
+export CI_IMAGE_NAME_TAG="quay.io/centos/amd64:stream9"
+export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison util-linux e2fsprogs cmake"
 export PIP_PACKAGES="pyzmq"
 export GOAL="install"
-export NO_WERROR=1 # GCC 8
+export NO_WERROR=1  # Suppress error: #warning _FORTIFY_SOURCE > 2 is treated like 2 on this platform [-Werror=cpp]
 export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports"
 export CONFIG_SHELL="/bin/dash"
-export TEST_RUNNER_ENV="LC_ALL=en_US.UTF-8"

ci/test/00_setup_env_i686_multiprocess.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2020-2022 The Bitcoin Core developers
+# Copyright (c) 2020-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -8,10 +8,11 @@ export LC_ALL=C.UTF-8
 
 export HOST=i686-pc-linux-gnu
 export CONTAINER_NAME=ci_i686_multiprocess
-export CI_IMAGE_NAME_TAG=ubuntu:20.04
-export PACKAGES="cmake python3 llvm clang g++-multilib"
+export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:24.04"
+export PACKAGES="llvm clang g++-multilib"
 export DEP_OPTS="DEBUG=1 MULTIPROCESS=1"
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' LDFLAGS='--rtlib=compiler-rt -lgcc_s'"
-export TEST_RUNNER_ENV="BITCOIND=bitcoin-node"
-export TEST_RUNNER_EXTRA="--nosandbox"
+export TEST_RUNNER_EXTRA="--v2transport"
+export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' \
+CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='-Wno-error=documentation'"
+export BITCOIND=bitcoin-node  # Used in functional tests

ci/test/00_setup_env_mac_cross.sh

@@ -1,17 +1,19 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2021 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
+export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
+
 export CONTAINER_NAME=ci_macos_cross
-export CI_IMAGE_NAME_TAG=ubuntu:20.04  # Check that Focal can cross-compile to macos
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 export HOST=x86_64-apple-darwin
-export PACKAGES="cmake libz-dev libtinfo5 python3-setuptools xorriso"
-export XCODE_VERSION=12.2
-export XCODE_BUILD_ID=12B45b
+export PACKAGES="clang lld llvm zip"
+export XCODE_VERSION=15.0
+export XCODE_BUILD_ID=15A240d
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="deploy"

ci/test/00_setup_env_mac_native.sh

@@ -1,16 +1,19 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2022 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-export HOST=arm64-apple-darwin
-export PIP_PACKAGES="zmq"
+export HOST=x86_64-apple-darwin
+# Homebrew's python@3.12 is marked as externally managed (PEP 668).
+# Therefore, `--break-system-packages` is needed.
+export PIP_PACKAGES="--break-system-packages zmq"
 export GOAL="install"
 export BITCOIN_CONFIG="--with-gui --with-miniupnpc --with-natpmp --enable-reduce-exports"
 export CI_OS_NAME="macos"
 export NO_DEPENDS=1
 export OSX_SDK=""
-export CCACHE_SIZE=300M
+export CCACHE_MAXSIZE=400M
+export RUN_FUZZ_TESTS=true

ci/test/00_setup_env_native_asan.sh

@@ -1,26 +1,29 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2022 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-# Only install BCC tracing packages in Cirrus CI.
-if [[ "${CIRRUS_CI}" == "true" ]]; then
-  # We install an up-to-date 'bpfcc-tools' package from an untrusted PPA.
-  # This can be dropped with the next Ubuntu or Debian release that includes up-to-date packages.
-  # See the if-then in ci/test/04_install.sh too.
-  export ADD_UNTRUSTED_BPFCC_PPA=true
-  export BPFCC_PACKAGE="bpfcc-tools"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
+
+# Only install BCC tracing packages in CI. Container has to match the host for BCC to work.
+if [[ "${INSTALL_BCC_TRACING_TOOLS}" == "true" ]]; then
+  # Required for USDT functional tests to run
+  BPFCC_PACKAGE="bpfcc-tools linux-headers-$(uname --kernel-release)"
+  export CI_CONTAINER_CAP="--privileged -v /sys/kernel:/sys/kernel:rw"
 else
-  export ADD_UNTRUSTED_BPFCC_PPA=false
-  export BPFCC_PACKAGE=""
+  BPFCC_PACKAGE=""
+  export CI_CONTAINER_CAP="--cap-add SYS_PTRACE"  # If run with (ASan + LSan), the container needs access to ptrace (https://github.com/google/sanitizers/issues/764)
 fi
 
 export CONTAINER_NAME=ci_native_asan
-export PACKAGES="systemtap-sdt-dev clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}"
-export CI_IMAGE_NAME_TAG=ubuntu:22.04
+export PACKAGES="systemtap-sdt-dev clang-18 llvm-18 libclang-rt-18-dev python3-zmq qtbase5-dev qttools5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}"
 export NO_DEPENDS=1
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"
+export BITCOIN_CONFIG="--enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \
+CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \
+--with-sanitizers=address,float-divide-by-zero,integer,undefined \
+CC='clang-18 -ftrivial-auto-var-init=pattern' CXX='clang++-18 -ftrivial-auto-var-init=pattern'"
+export CCACHE_MAXSIZE=300M

ci/test/00_setup_env_native_fuzz.sh

@@ -1,18 +1,21 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2022 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 export CONTAINER_NAME=ci_native_fuzz
-export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev"
+export PACKAGES="clang-18 llvm-18 libclang-rt-18-dev libevent-dev libboost-dev libsqlite3-dev"
 export NO_DEPENDS=1
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export RUN_FUZZ_TESTS=true
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC='clang-16 -ftrivial-auto-var-init=pattern' CXX='clang++-16 -ftrivial-auto-var-init=pattern'"
-export CCACHE_SIZE=200M
+export CI_CONTAINER_CAP="--cap-add SYS_PTRACE"  # If run with (ASan + LSan), the container needs access to ptrace (https://github.com/google/sanitizers/issues/764)
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,float-divide-by-zero,integer \
+CC='clang-18 -ftrivial-auto-var-init=pattern' CXX='clang++-18 -ftrivial-auto-var-init=pattern'"
+export CCACHE_MAXSIZE=200M
+export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-18"

ci/test/00_setup_env_native_fuzz_with_msan.sh

@@ -1,25 +1,26 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2020-2022 The Bitcoin Core developers
+# Copyright (c) 2020-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
-LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
+LIBCXX_DIR="/msan/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
-LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument"
+LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
 export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
 
 export CONTAINER_NAME="ci_native_fuzz_msan"
-export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake"
+export PACKAGES="ninja-build"
 # BDB generates false-positives and will be removed in future
-export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+# _FORTIFY_SOURCE is not compatible with MSAN.
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE -U_FORTIFY_SOURCE'"
 export USE_MEMORY_SANITIZER="true"
 export RUN_UNIT_TESTS="false"
 export RUN_FUNCTIONAL_TESTS="false"
 export RUN_FUZZ_TESTS=true
-export CCACHE_SIZE=250M
+export CCACHE_MAXSIZE=250M

ci/test/00_setup_env_native_fuzz_with_valgrind.sh

@@ -1,20 +1,20 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2022 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="debian:bookworm"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 export CONTAINER_NAME=ci_native_fuzz_valgrind
-export PACKAGES="clang llvm libclang-rt-dev python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev valgrind"
+export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libevent-dev libboost-dev libsqlite3-dev valgrind"
 export NO_DEPENDS=1
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export RUN_FUZZ_TESTS=true
 export FUZZ_TESTS_CONFIG="--valgrind"
 export GOAL="install"
-# Temporarily pin dwarf 4, until using Valgrind 3.20 or later
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'"
-export CCACHE_SIZE=200M
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang-16 CXX=clang++-16"
+export CCACHE_MAXSIZE=200M
+export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-16"

ci/test/00_setup_env_native_msan.sh

@@ -1,23 +1,24 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2020-2022 The Bitcoin Core developers
+# Copyright (c) 2020-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
-LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
+LIBCXX_DIR="/msan/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
-LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument"
+LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
 export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
 
 export CONTAINER_NAME="ci_native_msan"
-export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake"
+export PACKAGES="ninja-build"
 # BDB generates false-positives and will be removed in future
-export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export GOAL="install"
-export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+# _FORTIFY_SOURCE is not compatible with MSAN.
+export BITCOIN_CONFIG="--with-sanitizers=memory CPPFLAGS='-U_FORTIFY_SOURCE'"
 export USE_MEMORY_SANITIZER="true"
 export RUN_FUNCTIONAL_TESTS="false"
-export CCACHE_SIZE=250M
+export CCACHE_MAXSIZE=250M

ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh

@@ -1,16 +1,15 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2022 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel
-export CI_IMAGE_NAME_TAG=ubuntu:focal
-# Use minimum supported python3.7 (or python3.8, as best-effort) and clang-8, see doc/dependencies.md
-export PACKAGES="python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev"
-export DEP_OPTS="NO_WALLET=1 CC=clang-8 CXX='clang++-8 -stdlib=libc++'"
+export CI_IMAGE_NAME_TAG="docker.io/debian:bullseye"
+# Use minimum supported python3.9 and clang-16, see doc/dependencies.md
+export PACKAGES="python3-zmq clang-16 llvm-16 libc++abi-16-dev libc++-16-dev"
+export DEP_OPTS="NO_WALLET=1 CC=clang-16 CXX='clang++-16 -stdlib=libc++'"
 export GOAL="install"
-export NO_WERROR=1
-export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared"
+export BITCOIN_CONFIG="--enable-reduce-exports --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared"

ci/test/00_setup_env_native_previous_releases.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-present The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_native_previous_releases
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
+# Use minimum supported python3.9 (or best effort 3.10) and gcc-11, see doc/dependencies.md
+export PACKAGES="gcc-11 g++-11 python3-zmq"
+export DEP_OPTS="NO_UPNP=1 NO_NATPMP=1 DEBUG=1 CC=gcc-11 CXX=g++-11"
+export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash"  # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash
+export RUN_UNIT_TESTS_SEQUENTIAL="true"
+export RUN_UNIT_TESTS="false"
+export GOAL="install"
+export DOWNLOAD_PREVIOUS_RELEASES="true"
+export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports --enable-debug \
+CFLAGS=\"-g0 -O2 -funsigned-char\" CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS=\"-g0 -O2 -funsigned-char\""

ci/test/00_setup_env_native_qt5.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019-2022 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-export CONTAINER_NAME=ci_native_qt5
-export CI_IMAGE_NAME_TAG=debian:buster
-# Use minimum supported python3.7 and gcc-8, see doc/dependencies.md
-export PACKAGES="gcc-8 g++-8 python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev"
-export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-8 CXX=g++-8"
-export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash"  # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash
-export RUN_UNIT_TESTS_SEQUENTIAL="true"
-export RUN_UNIT_TESTS="false"
-export GOAL="install"
-export NO_WERROR=1
-export DOWNLOAD_PREVIOUS_RELEASES="true"
-export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports \
---enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8"

ci/test/00_setup_env_native_tidy.sh

@@ -1,19 +1,20 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2022 The Bitcoin Core developers
+# Copyright (c) 2023-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="ubuntu:lunar" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 export CONTAINER_NAME=ci_native_tidy
-export PACKAGES="clang-16 libclang-16-dev llvm-16-dev libomp-16-dev clang-tidy-16 bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev"
+export TIDY_LLVM_V="18"
+export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq bear libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev qtbase5-dev qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev"
 export NO_DEPENDS=1
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export RUN_FUZZ_TESTS=false
 export RUN_TIDY=true
 export GOAL="install"
-export BITCOIN_CONFIG="CC=clang-16 CXX=clang++-16 --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0 -I/usr/lib/llvm-16/lib/clang/16/include'"
-export CCACHE_SIZE=200M
+export BITCOIN_CONFIG="CC=clang-${TIDY_LLVM_V} CXX=clang++-${TIDY_LLVM_V} --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0'"
+export CCACHE_MAXSIZE=200M

ci/test/00_setup_env_native_tsan.sh

@@ -1,14 +1,14 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2022 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_tsan
-export CI_IMAGE_NAME_TAG=ubuntu:23.04  # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
-export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libc++abi-16-dev libc++-16-dev python3-zmq"
-export DEP_OPTS="CC=clang-16 CXX='clang++-16 -stdlib=libc++'"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
+export PACKAGES="clang-18 llvm-18 libclang-rt-18-dev libc++abi-18-dev libc++-18-dev python3-zmq"
+export DEP_OPTS="CC=clang-18 CXX='clang++-18 -stdlib=libc++'"
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' CXXFLAGS='-g' --with-sanitizers=thread"
+export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES' --with-sanitizers=thread"

ci/test/00_setup_env_native_valgrind.sh

@@ -1,17 +1,16 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2022 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="debian:bookworm"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 export CONTAINER_NAME=ci_native_valgrind
-export PACKAGES="valgrind clang llvm libclang-rt-dev python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev"
+export PACKAGES="valgrind clang-16 llvm-16 libclang-rt-16-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev"
 export USE_VALGRIND=1
 export NO_DEPENDS=1
-export TEST_RUNNER_EXTRA="--nosandbox --exclude feature_init,rpc_bind,feature_bind_extra"  # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547
+export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra"  # feature_init excluded for now, see https://github.com/bitcoin/bitcoin/issues/30011 ; bind tests excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547
 export GOAL="install"
-# Temporarily pin dwarf 4, until using Valgrind 3.20 or later
-export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'"  # TODO enable GUI
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang-16 CXX=clang++-16" # TODO enable GUI

ci/test/00_setup_env_s390x.sh

@@ -1,26 +1,16 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2022 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
 export HOST=s390x-linux-gnu
-# The host arch is unknown, so we run the tests through qemu.
-# If the host is s390x and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string.
-if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-s390x"}"; fi
 export PACKAGES="python3-zmq"
-if [ -n "$QEMU_USER_CMD" ]; then
-  # Likely cross-compiling, so install the needed gcc and qemu-user
-  export DPKG_ADD_ARCH="s390x"
-  export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x"
-fi
-# Use debian to avoid 404 apt errors
 export CONTAINER_NAME=ci_s390x
-export CI_IMAGE_NAME_TAG="debian:bookworm"
-export TEST_RUNNER_ENV="LC_ALL=C"
-export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra"  # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547
+export CI_IMAGE_NAME_TAG="docker.io/s390x/ubuntu:24.04"
+export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra"  # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547
 export RUN_FUNCTIONAL_TESTS=true
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests"  # GUI tests disabled for now, see https://github.com/bitcoin/bitcoin/issues/23730
+export BITCOIN_CONFIG="--enable-reduce-exports"

ci/test/00_setup_env_win64.sh

@@ -1,16 +1,19 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2022 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_win64
-export CI_IMAGE_NAME_TAG=ubuntu:22.04  # Check that Jammy can cross-compile to win64
+export CI_IMAGE_NAME_TAG="docker.io/amd64/debian:bookworm"  # Check that https://packages.debian.org/bookworm/g++-mingw-w64-x86-64-posix (version 12.2, similar to guix) can cross-compile
 export HOST=x86_64-w64-mingw32
 export DPKG_ADD_ARCH="i386"
-export PACKAGES="python3 nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file"
+export PACKAGES="nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file"
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="deploy"
-export BITCOIN_CONFIG="--enable-reduce-exports --enable-external-signer --disable-gui-tests"
+# Prior to 11.0.0, the mingw-w64 headers were missing noreturn attributes, causing warnings when
+# cross-compiling for Windows. https://sourceforge.net/p/mingw-w64/bugs/306/
+# https://github.com/mingw-w64/mingw-w64/commit/1690994f515910a31b9fb7c7bd3a52d4ba987abe
+export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests CXXFLAGS='-Wno-return-type -Wno-error=maybe-uninitialized -Wno-error=array-bounds'"

ci/test/01_base_install.sh

@@ -1,11 +1,13 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018-2022 The Bitcoin Core developers
+# Copyright (c) 2018-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
+set -ex
+
 CFG_DONE="ci.base-install-done"  # Use a global git setting to remember whether this script ran to avoid running it twice
 
 if [ "$(git config --global ${CFG_DONE})" == "true" ]; then
@@ -18,17 +20,9 @@ if [ -n "$DPKG_ADD_ARCH" ]; then
 fi
 
 if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then
-  ${CI_RETRY_EXE} bash -c "dnf -y install epel-release"
-  ${CI_RETRY_EXE} bash -c "dnf -y --allowerasing install $CI_BASE_PACKAGES $PACKAGES"
-elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
-  if [[ "${ADD_UNTRUSTED_BPFCC_PPA}" == "true" ]]; then
-    # Ubuntu 22.04 LTS and Debian 11 both have an outdated bpfcc-tools packages.
-    # The iovisor PPA is outdated as well. The next Ubuntu and Debian releases will contain updated
-    # packages. Meanwhile, use an untrusted PPA to install an up-to-date version of the bpfcc-tools
-    # package.
-    # TODO: drop this once we can use newer images in GCE
-    add-apt-repository ppa:hadret/bpfcc
-  fi
+  bash -c "dnf -y install epel-release"
+  bash -c "dnf -y --allowerasing install $CI_BASE_PACKAGES $PACKAGES"
+elif [ "$CI_OS_NAME" != "macos" ]; then
   if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then
     echo "${APPEND_APT_SOURCES_LIST}" >> /etc/apt/sources.list
   fi
@@ -37,28 +31,50 @@ elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
 fi
 
 if [ -n "$PIP_PACKAGES" ]; then
-  if [ "$CI_OS_NAME" == "macos" ]; then
-    sudo -H pip3 install --upgrade pip
-    # shellcheck disable=SC2086
-    IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
-  else
-    # shellcheck disable=SC2086
-    ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
-  fi
+  # shellcheck disable=SC2086
+  ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
 fi
 
 if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
-  update-alternatives --install /usr/bin/clang++ clang++ "$(which clang++-16)" 100
-  update-alternatives --install /usr/bin/clang clang "$(which clang-16)" 100
-  git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.1 "${BASE_SCRATCH_DIR}"/msan/llvm-project
-  cmake -B "${BASE_SCRATCH_DIR}"/msan/build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DLIBCXX_ENABLE_DEBUG_MODE=ON -DLIBCXX_ENABLE_ASSERTIONS=ON -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes
-  make -C "${BASE_SCRATCH_DIR}"/msan/build/ "$MAKEJOBS"
+  ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-18.1.3" /msan/llvm-project
+
+  cmake -G Ninja -B /msan/clang_build/ \
+    -DLLVM_ENABLE_PROJECTS="clang" \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DLLVM_TARGETS_TO_BUILD=Native \
+    -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \
+    -S /msan/llvm-project/llvm
+
+  ninja -C /msan/clang_build/ "-j$( nproc )"  # Use nproc, because MAKEJOBS is the default in docker image builds
+  ninja -C /msan/clang_build/ install-runtimes
+
+  update-alternatives --install /usr/bin/clang++ clang++ /msan/clang_build/bin/clang++ 100
+  update-alternatives --install /usr/bin/clang clang /msan/clang_build/bin/clang 100
+  update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /msan/clang_build/bin/llvm-symbolizer 100
+
+  cmake -G Ninja -B /msan/cxx_build/ \
+    -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DLLVM_USE_SANITIZER=MemoryWithOrigins \
+    -DCMAKE_C_COMPILER=clang \
+    -DCMAKE_CXX_COMPILER=clang++ \
+    -DLLVM_TARGETS_TO_BUILD=Native \
+    -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \
+    -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \
+    -DLIBCXX_HARDENING_MODE=debug \
+    -S /msan/llvm-project/runtimes
+
+  ninja -C /msan/cxx_build/ "-j$( nproc )"  # Use nproc, because MAKEJOBS is the default in docker image builds
+
+  # Clear no longer needed source folder
+  du -sh /msan/llvm-project
+  rm -rf /msan/llvm-project
 fi
 
 if [[ "${RUN_TIDY}" == "true" ]]; then
-  git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_16 "${DIR_IWYU}"/include-what-you-use
-  cmake -B "${DIR_IWYU}"/build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-16 -S "${DIR_IWYU}"/include-what-you-use
-  make -C "${DIR_IWYU}"/build/ install "$MAKEJOBS"
+  ${CI_RETRY_EXE} git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_"${TIDY_LLVM_V}" /include-what-you-use
+  cmake -B /iwyu-build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-"${TIDY_LLVM_V}" -S /include-what-you-use
+  make -C /iwyu-build/ install "-j$( nproc )"  # Use nproc, because MAKEJOBS is the default in docker image builds
 fi
 
 mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources"
@@ -69,19 +85,9 @@ if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]
   OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz"
   OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}"
   if [ ! -f "$OSX_SDK_PATH" ]; then
-    curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH"
+    ${CI_RETRY_EXE} curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH"
   fi
   tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH"
 fi
 
-if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then
-  ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip
-  if [ ! -f "$ANDROID_TOOLS_PATH" ]; then
-    curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH"
-  fi
-  mkdir -p "$ANDROID_HOME"
-  unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME"
-  yes | "${ANDROID_HOME}"/cmdline-tools/bin/sdkmanager --sdk_root="${ANDROID_HOME}" --install "build-tools;${ANDROID_BUILD_TOOLS_VERSION}" "platform-tools" "platforms;android-${ANDROID_API_LEVEL}" "ndk;${ANDROID_NDK_VERSION}"
-fi
-
 git config --global ${CFG_DONE} "true"

ci/test/02_run_container.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018-present The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+export CI_IMAGE_LABEL="bitcoin-ci-test"
+
+set -ex
+
+if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
+  # Export all env vars to avoid missing some.
+  # Though, exclude those with newlines to avoid parsing problems.
+  python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value and "HOME" != key and "PATH" != key and "USER" != key]' | tee "/tmp/env-$USER-$CONTAINER_NAME"
+  # System-dependent env vars must be kept as is. So read them from the container.
+  docker run --rm "${CI_IMAGE_NAME_TAG}" bash -c "env | grep --extended-regexp '^(HOME|PATH|USER)='" | tee --append "/tmp/env-$USER-$CONTAINER_NAME"
+  echo "Creating $CI_IMAGE_NAME_TAG container to run in"
+
+  DOCKER_BUILDKIT=1 docker build \
+      --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \
+      --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \
+      --build-arg "FILE_ENV=${FILE_ENV}" \
+      --label="${CI_IMAGE_LABEL}" \
+      --tag="${CONTAINER_NAME}" \
+      "${BASE_READ_ONLY_DIR}"
+
+  docker volume create "${CONTAINER_NAME}_ccache" || true
+  docker volume create "${CONTAINER_NAME}_depends" || true
+  docker volume create "${CONTAINER_NAME}_depends_sources" || true
+  docker volume create "${CONTAINER_NAME}_previous_releases" || true
+
+  CI_CCACHE_MOUNT="type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR"
+  CI_DEPENDS_MOUNT="type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built"
+  CI_DEPENDS_SOURCES_MOUNT="type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources"
+  CI_PREVIOUS_RELEASES_MOUNT="type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR"
+
+  if [ "$DANGER_CI_ON_HOST_CACHE_FOLDERS" ]; then
+    # ensure the directories exist
+    mkdir -p "${CCACHE_DIR}"
+    mkdir -p "${DEPENDS_DIR}/built"
+    mkdir -p "${DEPENDS_DIR}/sources"
+    mkdir -p "${PREVIOUS_RELEASES_DIR}"
+
+    CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=$CCACHE_DIR"
+    CI_DEPENDS_MOUNT="type=bind,src=${DEPENDS_DIR}/built,dst=$DEPENDS_DIR/built"
+    CI_DEPENDS_SOURCES_MOUNT="type=bind,src=${DEPENDS_DIR}/sources,dst=$DEPENDS_DIR/sources"
+    CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR"
+  fi
+
+  docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true
+
+  if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
+    echo "Restart docker before run to stop and clear all containers started with --rm"
+    podman container rm --force --all  # Similar to "systemctl restart docker"
+
+    # Still prune everything in case the filtered pruning doesn't work, or if labels were not set
+    # on a previous run. Belt and suspenders approach, should be fine to remove in the future.
+    # Prune images used by --external containers (e.g. build containers) when
+    # using podman.
+    echo "Prune all dangling images"
+    podman image prune --force --external
+  fi
+  echo "Prune all dangling $CI_IMAGE_LABEL images"
+  # When detecting podman-docker, `--external` should be added.
+  docker image prune --force --filter "label=$CI_IMAGE_LABEL"
+
+  # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME
+  # to allow support starting multiple runs simultaneously by the same user.
+  # shellcheck disable=SC2086
+  CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \
+                  --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \
+                  --mount "${CI_CCACHE_MOUNT}" \
+                  --mount "${CI_DEPENDS_MOUNT}" \
+                  --mount "${CI_DEPENDS_SOURCES_MOUNT}" \
+                  --mount "${CI_PREVIOUS_RELEASES_MOUNT}" \
+                  --env-file /tmp/env-$USER-$CONTAINER_NAME \
+                  --name "$CONTAINER_NAME" \
+                  --network ci-ip6net \
+                  "$CONTAINER_NAME")
+  export CI_CONTAINER_ID
+  export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}"
+else
+  echo "Running on host system without docker wrapper"
+  echo "Create missing folders"
+  mkdir -p "${CCACHE_DIR}"
+  mkdir -p "${PREVIOUS_RELEASES_DIR}"
+fi
+
+if [ "$CI_OS_NAME" == "macos" ]; then
+  IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt"
+  export IN_GETOPT_BIN
+fi
+
+CI_EXEC () {
+  $CI_EXEC_CMD_PREFIX bash -c "export PATH=\"/path_with space:${BINS_SCRATCH_DIR}:${BASE_ROOT_DIR}/ci/retry:\$PATH\" && cd \"${BASE_ROOT_DIR}\" && $*"
+}
+export -f CI_EXEC
+
+# Normalize all folders to BASE_ROOT_DIR
+CI_EXEC rsync --archive --stats --human-readable "${BASE_READ_ONLY_DIR}/" "${BASE_ROOT_DIR}" || echo "Nothing to copy from ${BASE_READ_ONLY_DIR}/"
+CI_EXEC "${BASE_ROOT_DIR}/ci/test/01_base_install.sh"
+
+# Fixes permission issues when there is a container UID/GID mismatch with the owner
+# of the git source code directory.
+CI_EXEC git config --global --add safe.directory \"*\"
+
+CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}"
+
+CI_EXEC "${BASE_ROOT_DIR}/ci/test/03_test_script.sh"
+
+if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
+  echo "Stop and remove CI container by ID"
+  docker container kill "${CI_CONTAINER_ID}"
+fi

ci/test/03_test_script.sh

@@ -0,0 +1,199 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018-present The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+set -ex
+
+export ASAN_OPTIONS="detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1"
+export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan"
+export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1"
+export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
+
+if [ "$CI_OS_NAME" == "macos" ]; then
+  top -l 1 -s 0 | awk ' /PhysMem/ {print}'
+  echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
+else
+  free -m -h
+  echo "Number of CPUs (nproc): $(nproc)"
+  echo "System info: $(uname --kernel-name --kernel-release)"
+  lscpu
+fi
+echo "Free disk space:"
+df -h
+
+# What host to compile for. See also ./depends/README.md
+# Tests that need cross-compilation export the appropriate HOST.
+# Tests that run natively guess the host
+export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")}
+
+(
+  # compact->outputs[i].file_size is uninitialized memory, so reading it is UB.
+  # The statistic bytes_written is only used for logging, which is disabled in
+  # CI, so as a temporary minimal fix to work around UB and CI failures, leave
+  # bytes_written unmodified.
+  # See https://github.com/bitcoin/bitcoin/pull/28359#issuecomment-1698694748
+  # Tee patch to stdout to make it clear CI is testing modified code.
+  tee >(patch -p1) <<'EOF'
+--- a/src/leveldb/db/db_impl.cc
++++ b/src/leveldb/db/db_impl.cc
+@@ -1028,9 +1028,6 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
+       stats.bytes_read += compact->compaction->input(which, i)->file_size;
+     }
+   }
+-  for (size_t i = 0; i < compact->outputs.size(); i++) {
+-    stats.bytes_written += compact->outputs[i].file_size;
+-  }
+
+   mutex_.Lock();
+   stats_[compact->compaction->level() + 1].Add(stats);
+EOF
+)
+
+if [ "$RUN_FUZZ_TESTS" = "true" ]; then
+  export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
+  if [ ! -d "$DIR_FUZZ_IN" ]; then
+    ${CI_RETRY_EXE} git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}"
+  fi
+  (
+    cd "${DIR_QA_ASSETS}"
+    echo "Using qa-assets repo from commit ..."
+    git log -1
+  )
+elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then
+  export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/
+  if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then
+    mkdir -p "$DIR_UNIT_TEST_DATA"
+    ${CI_RETRY_EXE} curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json"
+  fi
+fi
+
+if [ "$USE_BUSY_BOX" = "true" ]; then
+  echo "Setup to use BusyBox utils"
+  # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version)
+  # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed)
+  for util in $(busybox --list | grep -v "^ar$" | grep -v "^tar$" ); do ln -s "$(command -v busybox)" "${BINS_SCRATCH_DIR}/$util"; done
+  # Print BusyBox version
+  patch --help
+fi
+
+# Make sure default datadir does not exist and is never read by creating a dummy file
+if [ "$CI_OS_NAME" == "macos" ]; then
+  echo > "${HOME}/Library/Application Support/Bitcoin"
+else
+  echo > "${HOME}/.bitcoin"
+fi
+
+if [ -z "$NO_DEPENDS" ]; then
+  if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then
+    SHELL_OPTS="CONFIG_SHELL=/bin/dash"
+  else
+    SHELL_OPTS="CONFIG_SHELL="
+  fi
+  bash -c "$SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS LOG=1"
+fi
+if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then
+  test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR"
+fi
+
+BITCOIN_CONFIG_ALL="--disable-dependency-tracking"
+if [ -z "$NO_DEPENDS" ]; then
+  BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site"
+fi
+if [ -z "$NO_WERROR" ]; then
+  BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror"
+fi
+
+ccache --zero-stats
+PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats"
+
+BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR"
+
+if [ -n "$CONFIG_SHELL" ]; then
+  "$CONFIG_SHELL" -c "./autogen.sh"
+else
+  ./autogen.sh
+fi
+
+mkdir -p "${BASE_BUILD_DIR}"
+cd "${BASE_BUILD_DIR}"
+
+bash -c "${BASE_ROOT_DIR}/configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false)
+
+make distdir VERSION="$HOST"
+
+cd "${BASE_BUILD_DIR}/bitcoin-$HOST"
+
+bash -c "./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false)
+
+if [[ "${RUN_TIDY}" == "true" ]]; then
+  MAYBE_BEAR="bear --config src/.bear-tidy-config"
+  MAYBE_TOKEN="--"
+fi
+
+bash -c "${MAYBE_BEAR} ${MAYBE_TOKEN} make $MAKEJOBS $GOAL" || ( echo "Build failure. Verbose build follows." && make "$GOAL" V=1 ; false )
+
+bash -c "${PRINT_CCACHE_STATISTICS}"
+du -sh "${DEPENDS_DIR}"/*/
+du -sh "${PREVIOUS_RELEASES_DIR}"
+
+if [[ $HOST = *-mingw32 ]]; then
+  # Generate all binaries, so that they can be wrapped
+  make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1
+  make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1
+  "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh"
+fi
+
+if [ -n "$USE_VALGRIND" ]; then
+  "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh"
+fi
+
+if [ "$RUN_UNIT_TESTS" = "true" ]; then
+  DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" make "${MAKEJOBS}" check VERBOSE=1
+fi
+
+if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then
+  DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_OUTDIR}"/bin/test_bitcoin --catch_system_errors=no -l test_suite
+fi
+
+if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
+  # shellcheck disable=SC2086
+  LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/functional/test_runner.py --ci "${MAKEJOBS}" --tmpdirprefix "${BASE_SCRATCH_DIR}"/test_runner/ --ansi --combinedlogslen=99999999 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" ${TEST_RUNNER_EXTRA} --quiet --failfast
+fi
+
+if [ "${RUN_TIDY}" = "true" ]; then
+  cmake -B /tidy-build -DLLVM_DIR=/usr/lib/llvm-"${TIDY_LLVM_V}"/cmake -DCMAKE_BUILD_TYPE=Release -S "${BASE_ROOT_DIR}"/contrib/devtools/bitcoin-tidy
+  cmake --build /tidy-build "$MAKEJOBS"
+  cmake --build /tidy-build --target bitcoin-tidy-tests "$MAKEJOBS"
+
+  set -eo pipefail
+  cd "${BASE_BUILD_DIR}/bitcoin-$HOST/src/"
+  if ! ( run-clang-tidy-"${TIDY_LLVM_V}" -quiet -load="/tidy-build/libbitcoin-tidy.so" "${MAKEJOBS}" | tee tmp.tidy-out.txt ); then
+    grep -C5 "error: " tmp.tidy-out.txt
+    echo "^^^ ⚠️ Failure generated from clang-tidy"
+    false
+  fi
+  # Filter out files by regex here, because regex may not be
+  # accepted in src/.bear-tidy-config
+  # Filter out:
+  # * qt qrc and moc generated files
+  jq 'map(select(.file | test("src/qt/qrc_.*\\.cpp$|/moc_.*\\.cpp$") | not))' ../compile_commands.json > tmp.json
+  mv tmp.json ../compile_commands.json
+  cd "${BASE_BUILD_DIR}/bitcoin-$HOST/"
+  python3 "/include-what-you-use/iwyu_tool.py" \
+           -p . "${MAKEJOBS}" \
+           -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" \
+           -Xiwyu --max_line_length=160 \
+           2>&1 | tee /tmp/iwyu_ci.out
+  cd "${BASE_ROOT_DIR}/src"
+  python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out
+  git --no-pager diff
+fi
+
+if [ "$RUN_FUZZ_TESTS" = "true" ]; then
+  # shellcheck disable=SC2086
+  LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} "${MAKEJOBS}" -l DEBUG "${DIR_FUZZ_IN}" --empty_min_time=60
+fi

ci/test/04_install.sh

@@ -1,110 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018-2022 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-if [[ $QEMU_USER_CMD == qemu-s390* ]]; then
-  export LC_ALL=C
-fi
-
-# Create folders that are mounted into the docker
-mkdir -p "${CCACHE_DIR}"
-mkdir -p "${PREVIOUS_RELEASES_DIR}"
-
-export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1"
-export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan"
-export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan"
-export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
-env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env
-if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
-  CI_CONTAINER_CAP="--cap-add SYS_PTRACE"
-fi
-
-export P_CI_DIR="$PWD"
-export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/"
-
-if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
-  echo "Creating $CI_IMAGE_NAME_TAG container to run in"
-  DOCKER_BUILDKIT=1 ${CI_RETRY_EXE} docker build \
-      --file "${BASE_ROOT_DIR}/ci/test_imagefile" \
-      --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \
-      --build-arg "FILE_ENV=${FILE_ENV}" \
-      --tag="${CONTAINER_NAME}" \
-      "${BASE_ROOT_DIR}"
-  docker volume create "${CONTAINER_NAME}_ccache" || true
-  docker volume create "${CONTAINER_NAME}_depends" || true
-  docker volume create "${CONTAINER_NAME}_previous_releases" || true
-
-  if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
-    echo "Restart docker before run to stop and clear all containers started with --rm"
-    systemctl restart docker
-  fi
-
-  # shellcheck disable=SC2086
-  CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \
-                  --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \
-                  --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \
-                  --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR" \
-                  --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \
-                  -w $BASE_ROOT_DIR \
-                  --env-file /tmp/env \
-                  --name $CONTAINER_NAME \
-                  $CONTAINER_NAME)
-  export CI_CONTAINER_ID
-  export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}"
-else
-  echo "Running on host system without docker wrapper"
-fi
-
-CI_EXEC () {
-  $CI_EXEC_CMD_PREFIX bash -c "export PATH=${BINS_SCRATCH_DIR}:\$PATH && cd \"$P_CI_DIR\" && $*"
-}
-export -f CI_EXEC
-
-CI_EXEC rsync --archive --stats --human-readable /ci_base_install/ "${BASE_ROOT_DIR}" || echo "/ci_base_install/ missing"
-CI_EXEC "${BASE_ROOT_DIR}/ci/test/01_base_install.sh"
-CI_EXEC rsync --archive --stats --human-readable /ro_base/ "${BASE_ROOT_DIR}" || echo "Nothing to copy from ro_base"
-# Fixes permission issues when there is a container UID/GID mismatch with the owner
-# of the git source code directory.
-CI_EXEC git config --global --add safe.directory \"*\"
-
-CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}"
-
-if [ "$CI_OS_NAME" == "macos" ]; then
-  top -l 1 -s 0 | awk ' /PhysMem/ {print}'
-  echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
-else
-  CI_EXEC free -m -h
-  CI_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\)
-  CI_EXEC echo "$(lscpu | grep Endian)"
-fi
-CI_EXEC echo "Free disk space:"
-CI_EXEC df -h
-
-if [ "$RUN_FUZZ_TESTS" = "true" ]; then
-  export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
-  if [ ! -d "$DIR_FUZZ_IN" ]; then
-    CI_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}"
-  fi
-elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then
-  export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/
-  if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then
-    CI_EXEC mkdir -p "$DIR_UNIT_TEST_DATA"
-    CI_EXEC curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json"
-  fi
-fi
-
-CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/"
-
-if [ "$USE_BUSY_BOX" = "true" ]; then
-  echo "Setup to use BusyBox utils"
-  # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version)
-  # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed)
-  # shellcheck disable=SC1010
-  CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \)\; do ln -s \$\(command -v busybox\) "${BINS_SCRATCH_DIR}/\$util"\; done
-  # Print BusyBox version
-  CI_EXEC patch --help
-fi

ci/test/05_before_script.sh

@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018-2022 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-# Make sure default datadir does not exist and is never read by creating a dummy file
-if [ "$CI_OS_NAME" == "macos" ]; then
-  echo > "${HOME}/Library/Application Support/Bitcoin"
-else
-  CI_EXEC echo \> \$HOME/.bitcoin
-fi
-
-if [ -z "$NO_DEPENDS" ]; then
-  if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then
-    # CentOS has problems building the depends if the config shell is not explicitly set
-    # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to
-    #  an error as the first command is executed)
-    SHELL_OPTS="LC_ALL=en_US.UTF-8 CONFIG_SHELL=/bin/dash"
-  else
-    SHELL_OPTS="CONFIG_SHELL="
-  fi
-  CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" LOG=1
-fi
-if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then
-  CI_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR"
-fi

ci/test/06_script_a.sh

@@ -1,68 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018-2022 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking"
-if [ -z "$NO_DEPENDS" ]; then
-  BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site"
-fi
-if [ -z "$NO_WERROR" ]; then
-  BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror"
-fi
-
-CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE"
-PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats"
-
-if [ -n "$ANDROID_TOOLS_URL" ]; then
-  CI_EXEC make distclean || true
-  CI_EXEC ./autogen.sh
-  CI_EXEC ./configure "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false)
-  CI_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk"
-  CI_EXEC "${PRINT_CCACHE_STATISTICS}"
-  exit 0
-fi
-
-BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR"
-
-if [ -n "$CONFIG_SHELL" ]; then
-  CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh"
-else
-  CI_EXEC ./autogen.sh
-fi
-
-CI_EXEC mkdir -p "${BASE_BUILD_DIR}"
-export P_CI_DIR="${BASE_BUILD_DIR}"
-
-CI_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false)
-
-CI_EXEC make distdir VERSION="$HOST"
-
-export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST"
-
-CI_EXEC ./configure --cache-file=../config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false)
-
-set -o errtrace
-trap 'CI_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR
-
-if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
-  # MemorySanitizer (MSAN) does not support tracking memory initialization done by
-  # using the Linux getrandom syscall. Avoid using getrandom by undefining
-  # HAVE_SYS_GETRANDOM. See https://github.com/google/sanitizers/issues/852 for
-  # details.
-  CI_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h'
-fi
-
-if [[ "${RUN_TIDY}" == "true" ]]; then
-  MAYBE_BEAR="bear --config src/.bear-tidy-config"
-  MAYBE_TOKEN="--"
-fi
-
-CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false )
-
-CI_EXEC "${PRINT_CCACHE_STATISTICS}"
-CI_EXEC du -sh "${DEPENDS_DIR}"/*/
-CI_EXEC du -sh "${PREVIOUS_RELEASES_DIR}"

ci/test/06_script_b.sh

@@ -1,104 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018-2022 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-if [[ $HOST = *-mingw32 ]]; then
-  # Generate all binaries, so that they can be wrapped
-  CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1
-  CI_EXEC make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1
-  CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh"
-fi
-
-if [ -n "$QEMU_USER_CMD" ]; then
-  # Generate all binaries, so that they can be wrapped
-  CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1
-  CI_EXEC make "$MAKEJOBS" -C src minisketch/test VERBOSE=1
-  CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh"
-fi
-
-if [ -n "$USE_VALGRIND" ]; then
-  CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh"
-fi
-
-if [ "$RUN_UNIT_TESTS" = "true" ]; then
-  CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" make "$MAKEJOBS" check VERBOSE=1
-fi
-
-if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then
-  CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_OUTDIR}/bin/test_bitcoin" --catch_system_errors=no -l test_suite
-fi
-
-if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
-  CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=99999999 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast
-fi
-
-if [ "${RUN_TIDY}" = "true" ]; then
-  set -eo pipefail
-  export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/src/"
-  ( CI_EXEC run-clang-tidy-16 -quiet "${MAKEJOBS}" ) | grep -C5 "error"
-  export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/"
-  CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/iwyu_tool.py"\
-          " src/common/init.cpp"\
-          " src/common/url.cpp"\
-          " src/compat"\
-          " src/dbwrapper.cpp"\
-          " src/init"\
-          " src/kernel"\
-          " src/node/chainstate.cpp"\
-          " src/node/chainstatemanager_args.cpp"\
-          " src/node/mempool_args.cpp"\
-          " src/node/minisketchwrapper.cpp"\
-          " src/node/utxo_snapshot.cpp"\
-          " src/node/validation_cache_args.cpp"\
-          " src/policy/feerate.cpp"\
-          " src/policy/packages.cpp"\
-          " src/policy/settings.cpp"\
-          " src/primitives/transaction.cpp"\
-          " src/random.cpp"\
-          " src/rpc/fees.cpp"\
-          " src/rpc/signmessage.cpp"\
-          " src/test/fuzz/string.cpp"\
-          " src/test/fuzz/txorphan.cpp"\
-          " src/test/fuzz/util/"\
-          " src/test/util/coins.cpp"\
-          " src/uint256.cpp"\
-          " src/util/bip32.cpp"\
-          " src/util/bytevectorhash.cpp"\
-          " src/util/check.cpp"\
-          " src/util/error.cpp"\
-          " src/util/exception.cpp"\
-          " src/util/getuniquepath.cpp"\
-          " src/util/hasher.cpp"\
-          " src/util/message.cpp"\
-          " src/util/moneystr.cpp"\
-          " src/util/serfloat.cpp"\
-          " src/util/spanparsing.cpp"\
-          " src/util/strencodings.cpp"\
-          " src/util/string.cpp"\
-          " src/util/syserror.cpp"\
-          " src/util/threadinterrupt.cpp"\
-          " src/zmq"\
-          " -p . ${MAKEJOBS}"\
-          " -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp"\
-          " |& tee /tmp/iwyu_ci.out"
-  export P_CI_DIR="${BASE_ROOT_DIR}/src"
-  CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/fix_includes.py --nosafe_headers < /tmp/iwyu_ci.out"
-  CI_EXEC "git --no-pager diff"
-fi
-
-if [ "$RUN_SECURITY_TESTS" = "true" ]; then
-  CI_EXEC make test-security-check
-fi
-
-if [ "$RUN_FUZZ_TESTS" = "true" ]; then
-  CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py "${FUZZ_TESTS_CONFIG}" "$MAKEJOBS" -l DEBUG "${DIR_FUZZ_IN}"
-fi
-
-if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
-  echo "Stop and remove CI container by ID"
-  docker container kill "${CI_CONTAINER_ID}"
-fi

ci/test/wrap-qemu.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018-2022 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{test_json,unitester,object}}; do
-    # shellcheck disable=SC2044
-    for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do
-      echo "Wrap $b ..."
-      mv "$b" "${b}_orig"
-      echo '#!/usr/bin/env bash' > "$b"
-      echo "$QEMU_USER_CMD \"${b}_orig\" \"\$@\"" >> "$b"
-      chmod +x "$b"
-    done
-done

ci/test/wrapped-cl.bat

@@ -1 +0,0 @@
-ccache cl %*

ci/test_imagefile

@@ -1,3 +1,9 @@
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit/.
+
+# See ci/README.md for usage.
+
 ARG CI_IMAGE_NAME_TAG
 FROM ${CI_IMAGE_NAME_TAG}
 
@@ -5,6 +11,6 @@ ARG FILE_ENV
 ENV FILE_ENV=${FILE_ENV}
 
 COPY ./ci/retry/retry /usr/bin/retry
-COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_base_install/ci/test/
+COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_container_base/ci/test/
 
-RUN ["bash", "-c", "cd /ci_base_install/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"]
+RUN ["bash", "-c", "cd /ci_container_base/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"]

ci/test_run_all.sh

@@ -1,13 +1,11 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Copyright (c) 2019-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
 set -o errexit; source ./ci/test/00_setup_env.sh
-set -o errexit; source ./ci/test/04_install.sh
-set -o errexit; source ./ci/test/05_before_script.sh
-set -o errexit; source ./ci/test/06_script_a.sh
-set -o errexit; source ./ci/test/06_script_b.sh
+set -o errexit
+"./ci/test/02_run_container.sh"

contrib/README.md

@@ -35,7 +35,7 @@ Test and Verify Tools
 ### [TestGen](/contrib/testgen) ###
 Utilities to generate test vectors for the data-driven Bitcoin tests.
 
-### [Verify Binaries](/contrib/verifybinaries) ###
+### [Verify-Binaries](/contrib/verify-binaries) ###
 This script attempts to download and verify the signature file SHA256SUMS.asc from bitcoin.org.
 
 Command Line Tools

contrib/asmap/README.md

@@ -0,0 +1,12 @@
+# ASMap Tool
+
+Tool for performing various operations on textual and binary asmap files,
+particularly encoding/compressing the raw data to the binary format that can
+be used in Bitcoin Core with the `-asmap` option.
+
+Example usage:
+```
+python3 asmap-tool.py encode /path/to/input.file /path/to/output.file
+python3 asmap-tool.py decode /path/to/input.file /path/to/output.file
+python3 asmap-tool.py diff /path/to/first.file /path/to/second.file
+```

contrib/asmap/asmap-tool.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 Pieter Wuille
+# Distributed under the MIT software license, see the accompanying
+# file LICENSE or http://www.opensource.org/licenses/mit-license.php.
+
+import argparse
+import sys
+import ipaddress
+import json
+import math
+from collections import defaultdict
+
+import asmap
+
+def load_file(input_file):
+    try:
+        contents = input_file.read()
+    except OSError as err:
+        sys.exit(f"Input file '{input_file.name}' cannot be read: {err.strerror}.")
+    try:
+        bin_asmap = asmap.ASMap.from_binary(contents)
+    except ValueError:
+        bin_asmap = None
+    txt_error = None
+    entries = None
+    try:
+        txt_contents = str(contents, encoding="utf-8")
+    except UnicodeError:
+        txt_error = "invalid UTF-8"
+        txt_contents = None
+    if txt_contents is not None:
+        entries = []
+        for line in txt_contents.split("\n"):
+            idx = line.find('#')
+            if idx >= 0:
+                line = line[:idx]
+            line = line.lstrip(' ').rstrip(' \t\r\n')
+            if len(line) == 0:
+                continue
+            fields = line.split(' ')
+            if len(fields) != 2:
+                txt_error = f"unparseable line '{line}'"
+                entries = None
+                break
+            prefix, asn = fields
+            if len(asn) <= 2 or asn[:2] != "AS" or any(c < '0' or c > '9' for c in asn[2:]):
+                txt_error = f"invalid ASN '{asn}'"
+                entries = None
+                break
+            try:
+                net = ipaddress.ip_network(prefix)
+            except ValueError:
+                txt_error = f"invalid network '{prefix}'"
+                entries = None
+                break
+            entries.append((asmap.net_to_prefix(net), int(asn[2:])))
+    if entries is not None and bin_asmap is not None and len(contents) > 0:
+        sys.exit(f"Input file '{input_file.name}' is ambiguous.")
+    if entries is not None:
+        state = asmap.ASMap()
+        state.update_multi(entries)
+        return state
+    if bin_asmap is not None:
+        return bin_asmap
+    sys.exit(f"Input file '{input_file.name}' is neither a valid binary asmap file nor valid text input ({txt_error}).")
+
+
+def save_binary(output_file, state, fill):
+    contents = state.to_binary(fill=fill)
+    try:
+        output_file.write(contents)
+        output_file.close()
+    except OSError as err:
+        sys.exit(f"Output file '{output_file.name}' cannot be written to: {err.strerror}.")
+
+def save_text(output_file, state, fill, overlapping):
+    for prefix, asn in state.to_entries(fill=fill, overlapping=overlapping):
+        net = asmap.prefix_to_net(prefix)
+        try:
+            print(f"{net} AS{asn}", file=output_file)
+        except OSError as err:
+            sys.exit(f"Output file '{output_file.name}' cannot be written to: {err.strerror}.")
+    try:
+        output_file.close()
+    except OSError as err:
+        sys.exit(f"Output file '{output_file.name}' cannot be written to: {err.strerror}.")
+
+def main():
+    parser = argparse.ArgumentParser(description="Tool for performing various operations on textual and binary asmap files.")
+    subparsers = parser.add_subparsers(title="valid subcommands", dest="subcommand")
+
+    parser_encode = subparsers.add_parser("encode", help="convert asmap data to binary format")
+    parser_encode.add_argument('-f', '--fill', dest="fill", default=False, action="store_true",
+                               help="permit reassigning undefined network ranges arbitrarily to reduce size")
+    parser_encode.add_argument('infile', nargs='?', type=argparse.FileType('rb'), default=sys.stdin.buffer,
+                               help="input asmap file (text or binary); default is stdin")
+    parser_encode.add_argument('outfile', nargs='?', type=argparse.FileType('wb'), default=sys.stdout.buffer,
+                               help="output binary asmap file; default is stdout")
+
+    parser_decode = subparsers.add_parser("decode", help="convert asmap data to text format")
+    parser_decode.add_argument('-f', '--fill', dest="fill", default=False, action="store_true",
+                               help="permit reassigning undefined network ranges arbitrarily to reduce length")
+    parser_decode.add_argument('-n', '--nonoverlapping', dest="overlapping", default=True, action="store_false",
+                               help="output strictly non-overall ping network ranges (increases output size)")
+    parser_decode.add_argument('infile', nargs='?', type=argparse.FileType('rb'), default=sys.stdin.buffer,
+                               help="input asmap file (text or binary); default is stdin")
+    parser_decode.add_argument('outfile', nargs='?', type=argparse.FileType('w'), default=sys.stdout,
+                               help="output text file; default is stdout")
+
+    parser_diff = subparsers.add_parser("diff", help="compute the difference between two asmap files")
+    parser_diff.add_argument('-i', '--ignore-unassigned', dest="ignore_unassigned", default=False, action="store_true",
+                             help="ignore unassigned ranges in the first input (useful when second input is filled)")
+    parser_diff.add_argument('infile1', type=argparse.FileType('rb'),
+                             help="first file to compare (text or binary)")
+    parser_diff.add_argument('infile2', type=argparse.FileType('rb'),
+                             help="second file to compare (text or binary)")
+
+    parser_diff_addrs = subparsers.add_parser("diff_addrs",
+                                              help="compute difference between two asmap files for a set of addresses")
+    parser_diff_addrs.add_argument('-s', '--show-addresses', dest="show_addresses", default=False, action="store_true",
+                                   help="include reassigned addresses in the output")
+    parser_diff_addrs.add_argument("infile1", type=argparse.FileType("rb"),
+                                   help="first file to compare (text or binary)")
+    parser_diff_addrs.add_argument("infile2", type=argparse.FileType("rb"),
+                                   help="second file to compare (text or binary)")
+    parser_diff_addrs.add_argument("addrs_file", type=argparse.FileType("r"),
+                                   help="address file containing getnodeaddresses output to use in the comparison "
+                                   "(make sure to set the count parameter to zero to get all node addresses, "
+                                   "e.g. 'bitcoin-cli getnodeaddresses 0 > addrs.json')")
+    args = parser.parse_args()
+    if args.subcommand is None:
+        parser.print_help()
+    elif args.subcommand == "encode":
+        state = load_file(args.infile)
+        save_binary(args.outfile, state, fill=args.fill)
+    elif args.subcommand == "decode":
+        state = load_file(args.infile)
+        save_text(args.outfile, state, fill=args.fill, overlapping=args.overlapping)
+    elif args.subcommand == "diff":
+        state1 = load_file(args.infile1)
+        state2 = load_file(args.infile2)
+        ipv4_changed = 0
+        ipv6_changed = 0
+        for prefix, old_asn, new_asn in state1.diff(state2):
+            if args.ignore_unassigned and old_asn == 0:
+                continue
+            net = asmap.prefix_to_net(prefix)
+            if isinstance(net, ipaddress.IPv4Network):
+                ipv4_changed += 1 << (32 - net.prefixlen)
+            elif isinstance(net, ipaddress.IPv6Network):
+                ipv6_changed += 1 << (128 - net.prefixlen)
+            if new_asn == 0:
+                print(f"# {net} was AS{old_asn}")
+            elif old_asn == 0:
+                print(f"{net} AS{new_asn} # was unassigned")
+            else:
+                print(f"{net} AS{new_asn} # was AS{old_asn}")
+        ipv4_change_str = "" if ipv4_changed == 0 else f" (2^{math.log2(ipv4_changed):.2f})"
+        ipv6_change_str = "" if ipv6_changed == 0 else f" (2^{math.log2(ipv6_changed):.2f})"
+
+        print(
+            f"# {ipv4_changed}{ipv4_change_str} IPv4 addresses changed; "
+            f"{ipv6_changed}{ipv6_change_str} IPv6 addresses changed"
+        )
+    elif args.subcommand == "diff_addrs":
+        state1 = load_file(args.infile1)
+        state2 = load_file(args.infile2)
+        address_info = json.load(args.addrs_file)
+        addrs = {a["address"] for a in address_info if a["network"] in ["ipv4", "ipv6"]}
+        reassignments = defaultdict(list)
+        for addr in addrs:
+            net = ipaddress.ip_network(addr)
+            prefix = asmap.net_to_prefix(net)
+            old_asn = state1.lookup(prefix)
+            new_asn = state2.lookup(prefix)
+            if new_asn != old_asn:
+                reassignments[(old_asn, new_asn)].append(addr)
+        reassignments = sorted(reassignments.items(), key=lambda item: len(item[1]), reverse=True)
+        num_reassignment_type = defaultdict(int)
+        for (old_asn, new_asn), reassigned_addrs in reassignments:
+            num_reassigned = len(reassigned_addrs)
+            num_reassignment_type[(bool(old_asn), bool(new_asn))] += num_reassigned
+            old_asn_str = f"AS{old_asn}" if old_asn else "unassigned"
+            new_asn_str = f"AS{new_asn}" if new_asn else "unassigned"
+            opt = ": " + ", ".join(reassigned_addrs) if args.show_addresses else ""
+            print(f"{num_reassigned} address(es) reassigned from {old_asn_str} to {new_asn_str}{opt}")
+        num_reassignments = sum(len(addrs) for _, addrs in reassignments)
+        share = num_reassignments / len(addrs) if len(addrs) > 0 else 0
+        print(f"Summary: {num_reassignments:,} ({share:.2%}) of {len(addrs):,} addresses were reassigned "
+              f"(migrations={num_reassignment_type[True, True]}, assignments={num_reassignment_type[False, True]}, "
+              f"unassignments={num_reassignment_type[True, False]})")
+    else:
+        parser.print_help()
+        sys.exit("No command provided.")
+
+if __name__ == '__main__':
+    main()

contrib/asmap/asmap.py

@@ -10,11 +10,12 @@ import copy
 import ipaddress
 import random
 import unittest
+from collections.abc import Callable, Iterable
 from enum import Enum
 from functools import total_ordering
-from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union, overload
+from typing import Optional, Union, overload
 
-def net_to_prefix(net: Union[ipaddress.IPv4Network,ipaddress.IPv6Network]) -> List[bool]:
+def net_to_prefix(net: Union[ipaddress.IPv4Network,ipaddress.IPv6Network]) -> list[bool]:
     """
     Convert an IPv4 or IPv6 network to a prefix represented as a list of bits.
 
@@ -32,7 +33,7 @@ def net_to_prefix(net: Union[ipaddress.IPv4Network,ipaddress.IPv6Network]) -> Li
     assert (netrange & ((1 << (128 - num_bits)) - 1)) == 0
     return [((netrange >> (127 - i)) & 1) != 0 for i in range(num_bits)]
 
-def prefix_to_net(prefix: List[bool]) -> Union[ipaddress.IPv4Network,ipaddress.IPv6Network]:
+def prefix_to_net(prefix: list[bool]) -> Union[ipaddress.IPv4Network,ipaddress.IPv6Network]:
     """The reverse operation of net_to_prefix."""
     # Convert to number
     netrange = sum(b << (127 - i) for i, b in enumerate(prefix))
@@ -47,10 +48,10 @@ def prefix_to_net(prefix: List[bool]) -> Union[ipaddress.IPv4Network,ipaddress.I
     return ipaddress.IPv6Network((netrange, num_bits), True)
 
 # Shortcut for (prefix, ASN) entries.
-ASNEntry = Tuple[List[bool], int]
+ASNEntry = tuple[list[bool], int]
 
 # Shortcut for (prefix, old ASN, new ASN) entries.
-ASNDiff = Tuple[List[bool], int, int]
+ASNDiff = tuple[list[bool], int, int]
 
 class _VarLenCoder:
     """
@@ -75,7 +76,7 @@ class _VarLenCoder:
       other classes start one past the last element of the class before it.
     """
 
-    def __init__(self, minval: int, clsbits: List[int]):
+    def __init__(self, minval: int, clsbits: list[int]):
         """Construct a new _VarLenCoder."""
         self._minval = minval
         self._clsbits = clsbits
@@ -85,7 +86,7 @@ class _VarLenCoder:
         """Check whether value val is in the range this coder supports."""
         return self._minval <= val <= self._maxval
 
-    def encode(self, val: int, ret: List[int]) -> None:
+    def encode(self, val: int, ret: list[int]) -> None:
         """Append encoding of val onto integer list ret."""
 
         assert self._minval <= val <= self._maxval
@@ -120,7 +121,7 @@ class _VarLenCoder:
                 break
         return ret + bits
 
-    def decode(self, stream, bitpos) -> Tuple[int,int]:
+    def decode(self, stream, bitpos) -> tuple[int,int]:
         """Decode a number starting at bitpos in stream, returning value and new bitpos."""
         val = self._minval
         bits = 0
@@ -281,11 +282,11 @@ class ASMap:
     -             mappings, represented by new trie nodes.
     """
 
-    def update(self, prefix: List[bool], asn: int) -> None:
+    def update(self, prefix: list[bool], asn: int) -> None:
         """Update this ASMap object to map prefix to the specified asn."""
         assert asn == 0 or _CODER_ASN.can_encode(asn)
 
-        def recurse(node: List, offset: int) -> None:
+        def recurse(node: list, offset: int) -> None:
             if offset == len(prefix):
                 # Reached the end of prefix; overwrite this node.
                 node.clear()
@@ -306,7 +307,7 @@ class ASMap:
                 node.append(oldasn)
         recurse(self._trie, 0)
 
-    def update_multi(self, entries: List[Tuple[List[bool], int]]) -> None:
+    def update_multi(self, entries: list[tuple[list[bool], int]]) -> None:
         """Apply multiple update operations, where longer prefixes take precedence."""
         entries.sort(key=lambda entry: len(entry[0]))
         for prefix, asn in entries:
@@ -314,7 +315,7 @@ class ASMap:
 
     def _set_trie(self, trie) -> None:
         """Set trie directly. Internal use only."""
-        def recurse(node: List) -> None:
+        def recurse(node: list) -> None:
             if len(node) < 2:
                 return
             recurse(node[0])
@@ -342,7 +343,7 @@ class ASMap:
             for prefix, asn in sorted(entries, key=entry_key):
                 self.update(prefix, asn)
 
-    def lookup(self, prefix: List[bool]) -> Optional[int]:
+    def lookup(self, prefix: list[bool]) -> Optional[int]:
         """Look up a prefix. Returns ASN, or 0 if unassigned, or None if indeterminate."""
         node = self._trie
         for bit in prefix:
@@ -353,11 +354,11 @@ class ASMap:
             return node[0]
         return None
 
-    def _to_entries_flat(self, fill: bool = False) -> List[ASNEntry]:
+    def _to_entries_flat(self, fill: bool = False) -> list[ASNEntry]:
         """Convert an ASMap object to a list of non-overlapping (prefix, asn) objects."""
-        prefix : List[bool] = []
+        prefix : list[bool] = []
 
-        def recurse(node: List) -> List[ASNEntry]:
+        def recurse(node: list) -> list[ASNEntry]:
             ret = []
             if len(node) == 1:
                 if node[0] > 0:
@@ -375,24 +376,24 @@ class ASMap:
             return ret
         return recurse(self._trie)
 
-    def _to_entries_minimal(self, fill: bool = False) -> List[ASNEntry]:
+    def _to_entries_minimal(self, fill: bool = False) -> list[ASNEntry]:
         """Convert a trie to a minimal list of ASNEntry objects, exploiting overlap."""
-        prefix : List[bool] = []
+        prefix : list[bool] = []
 
-        def recurse(node: List) -> (Tuple[Dict[Optional[int], List[ASNEntry]], bool]):
+        def recurse(node: list) -> (tuple[dict[Optional[int], list[ASNEntry]], bool]):
             if len(node) == 1 and node[0] == 0:
                 return {None if fill else 0: []}, True
             if len(node) == 1:
                 return {node[0]: [], None: [(list(prefix), node[0])]}, False
-            ret: Dict[Optional[int], List[ASNEntry]] = {}
+            ret: dict[Optional[int], list[ASNEntry]] = {}
             prefix.append(False)
             left, lhole = recurse(node[0])
             prefix[-1] = True
             right, rhole = recurse(node[1])
             prefix.pop()
             hole = not fill and (lhole or rhole)
-            def candidate(ctx: Optional[int], res0: Optional[List[ASNEntry]],
-                    res1: Optional[List[ASNEntry]]):
+            def candidate(ctx: Optional[int], res0: Optional[list[ASNEntry]],
+                    res1: Optional[list[ASNEntry]]):
                 if res0 is not None and res1 is not None:
                     if ctx not in ret or len(res0) + len(res1) < len(ret[ctx]):
                         ret[ctx] = res0 + res1
@@ -417,7 +418,7 @@ class ASMap:
         """Convert this ASMap object to a string containing Python code constructing it."""
         return f"ASMap({self._trie})"
 
-    def to_entries(self, overlapping: bool = True, fill: bool = False) -> List[ASNEntry]:
+    def to_entries(self, overlapping: bool = True, fill: bool = False) -> list[ASNEntry]:
         """
         Convert the mappings in this ASMap object to a list of ASNEntry objects.
 
@@ -448,7 +449,7 @@ class ASMap:
         assert max_asn >= 1 or unassigned_prob == 1
         assert _CODER_ASN.can_encode(max_asn)
         assert 0.0 <= unassigned_prob <= 1.0
-        trie: List = []
+        trie: list = []
         leaves = [trie]
         ret = ASMap()
         for i in range(1, num_leaves):
@@ -472,12 +473,12 @@ class ASMap:
 
     def _to_binnode(self, fill: bool = False) -> _BinNode:
         """Convert a trie to a _BinNode object."""
-        def recurse(node: List) -> Tuple[Dict[Optional[int], _BinNode], bool]:
+        def recurse(node: list) -> tuple[dict[Optional[int], _BinNode], bool]:
             if len(node) == 1 and node[0] == 0:
                 return {(None if fill else 0): _BinNode.make_end()}, True
             if len(node) == 1:
                 return {None: _BinNode.make_leaf(node[0]), node[0]: _BinNode.make_end()}, False
-            ret: Dict[Optional[int], _BinNode] = {}
+            ret: dict[Optional[int], _BinNode] = {}
             left, lhole = recurse(node[0])
             right, rhole = recurse(node[1])
             hole = (lhole or rhole) and not fill
@@ -488,12 +489,14 @@ class ASMap:
                     if ctx not in ret or cand.size < ret[ctx].size:
                         ret[ctx] = cand
 
-            for ctx in set(left) | set(right):
+            union = set(left) | set(right)
+            sorted_union = sorted(union, key=lambda x: (x is None, x))
+            for ctx in sorted_union:
                 candidate(ctx, left.get(ctx), right.get(ctx), _BinNode.make_branch)
                 candidate(ctx, left.get(None), right.get(ctx), _BinNode.make_branch)
                 candidate(ctx, left.get(ctx), right.get(None), _BinNode.make_branch)
             if not hole:
-                for ctx in set(ret) - set([None]):
+                for ctx in sorted(set(ret) - set([None])):
                     candidate(None, ctx, ret[ctx], _BinNode.make_default)
             if None in ret:
                 ret = {ctx:enc for ctx, enc in ret.items()
@@ -507,7 +510,7 @@ class ASMap:
     @staticmethod
     def _from_binnode(binnode: _BinNode) -> "ASMap":
         """Construct an ASMap object from a _BinNode. Internal use only."""
-        def recurse(node: _BinNode, default: int) -> List:
+        def recurse(node: _BinNode, default: int) -> list:
             if node.ins == _Instruction.RETURN:
                 return [node.arg1]
             if node.ins == _Instruction.JUMP:
@@ -542,7 +545,7 @@ class ASMap:
         Returns:
             A bytes object with the encoding of this ASMap object.
         """
-        bits: List[int] = []
+        bits: list[int] = []
 
         def recurse(node: _BinNode) -> None:
             _CODER_INS.encode(node.ins.value, bits)
@@ -582,11 +585,11 @@ class ASMap:
     def from_binary(bindata: bytes) -> Optional["ASMap"]:
         """Decode an ASMap object from the provided binary encoding."""
 
-        bits: List[int] = []
+        bits: list[int] = []
         for byte in bindata:
             bits.extend((byte >> i) & 1 for i in range(8))
 
-        def recurse(bitpos: int) -> Tuple[_BinNode, int]:
+        def recurse(bitpos: int) -> tuple[_BinNode, int]:
             insval, bitpos = _CODER_INS.decode(bits, bitpos)
             ins = _Instruction(insval)
             if ins == _Instruction.RETURN:
@@ -632,7 +635,7 @@ class ASMap:
 
     def extends(self, req: "ASMap") -> bool:
         """Determine whether this matches req for all subranges where req is assigned."""
-        def recurse(actual: List, require: List) -> bool:
+        def recurse(actual: list, require: list) -> bool:
             if len(require) == 1 and require[0] == 0:
                 return True
             if len(require) == 1:
@@ -646,20 +649,20 @@ class ASMap:
         #pylint: disable=protected-access
         return recurse(self._trie, req._trie)
 
-    def diff(self, other: "ASMap") -> List[ASNDiff]:
+    def diff(self, other: "ASMap") -> list[ASNDiff]:
         """Compute the diff from self to other."""
-        prefix: List[bool] = []
-        ret: List[ASNDiff] = []
+        prefix: list[bool] = []
+        ret: list[ASNDiff] = []
 
-        def recurse(old_node: List, new_node: List):
+        def recurse(old_node: list, new_node: list):
             if len(old_node) == 1 and len(new_node) == 1:
                 if old_node[0] != new_node[0]:
                     ret.append((list(prefix), old_node[0], new_node[0]))
             else:
-                old_left: List = old_node if len(old_node) == 1 else old_node[0]
-                old_right: List = old_node if len(old_node) == 1 else old_node[1]
-                new_left: List = new_node if len(new_node) == 1 else new_node[0]
-                new_right: List = new_node if len(new_node) == 1 else new_node[1]
+                old_left: list = old_node if len(old_node) == 1 else old_node[0]
+                old_right: list = old_node if len(old_node) == 1 else old_node[1]
+                new_left: list = new_node if len(new_node) == 1 else new_node[0]
+                new_right: list = new_node if len(new_node) == 1 else new_node[1]
                 prefix.append(False)
                 recurse(old_left, new_left)
                 prefix[-1] = True
@@ -760,7 +763,7 @@ class TestASMap(unittest.TestCase):
                     # It starts off being equal to asmap.
                     patched = copy.copy(asmap)
                     # Keep a list of patches performed.
-                    patches: List[ASNEntry] = []
+                    patches: list[ASNEntry] = []
                     # Initially there cannot be any difference.
                     self.assertEqual(asmap.diff(patched), [])
                     # Make 5 patches, each building on top of the previous ones.

contrib/completions/bash/bitcoin-cli.bash

@@ -9,7 +9,7 @@ _bitcoin_rpc() {
     local rpcargs=()
     for i in ${COMP_LINE}; do
         case "$i" in
-            -conf=*|-datadir=*|-regtest|-rpc*|-testnet)
+            -conf=*|-datadir=*|-regtest|-rpc*|-testnet|-testnet4)
                 rpcargs=( "${rpcargs[@]}" "$i" )
                 ;;
         esac

contrib/debian/copyright

@@ -5,7 +5,7 @@ Upstream-Contact: Satoshi Nakamoto <satoshin@gmx.com>
 Source: https://github.com/bitcoin/bitcoin
 
 Files: *
-Copyright: 2009-2023, Bitcoin Core Developers
+Copyright: 2009-2025, Bitcoin Core Developers
 License: Expat
 Comment: The Bitcoin Core Developers encompasses all contributors to the
          project, listed in the release notes or the git log.

contrib/devtools/README.md

@@ -83,13 +83,23 @@ A small script to automatically create manpages in ../../doc/man by running the
 This requires help2man which can be found at: https://www.gnu.org/software/help2man/
 
 With in-tree builds this tool can be run from any directory within the
-repostitory. To use this tool with out-of-tree builds set `BUILDDIR`. For
+repository. To use this tool with out-of-tree builds set `BUILDDIR`. For
 example:
 
 ```bash
 BUILDDIR=$PWD/build contrib/devtools/gen-manpages.py
 ```
 
+headerssync-params.py
+=====================
+
+A script to generate optimal parameters for the headerssync module (src/headerssync.cpp). It takes no command-line
+options, as all its configuration is set at the top of the file. It runs many times faster inside PyPy. Invocation:
+
+```bash
+pypy3 contrib/devtools/headerssync-params.py
+```
+
 gen-bitcoin-conf.sh
 ===================
 

contrib/devtools/bitcoin-tidy/CMakeLists.txt

@@ -0,0 +1,67 @@
+cmake_minimum_required(VERSION 3.22)
+
+project(bitcoin-tidy
+    VERSION
+    1.0.0
+    DESCRIPTION "clang-tidy checks for Bitcoin Core"
+    LANGUAGES CXX)
+
+include(GNUInstallDirs)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+set(CMAKE_CXX_EXTENSIONS False)
+
+set(CMAKE_DISABLE_FIND_PACKAGE_CURL ON)
+set(CMAKE_DISABLE_FIND_PACKAGE_FFI ON)
+set(CMAKE_DISABLE_FIND_PACKAGE_LibEdit ON)
+set(CMAKE_DISABLE_FIND_PACKAGE_LibXml2 ON)
+set(CMAKE_DISABLE_FIND_PACKAGE_Terminfo ON)
+set(CMAKE_DISABLE_FIND_PACKAGE_ZLIB ON)
+set(CMAKE_DISABLE_FIND_PACKAGE_zstd ON)
+
+find_package(LLVM REQUIRED CONFIG)
+find_program(CLANG_TIDY_EXE NAMES "clang-tidy-${LLVM_VERSION_MAJOR}" "clang-tidy" HINTS ${LLVM_TOOLS_BINARY_DIR})
+message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}")
+message(STATUS "Found clang-tidy: ${CLANG_TIDY_EXE}")
+
+add_library(bitcoin-tidy MODULE bitcoin-tidy.cpp logprintf.cpp nontrivial-threadlocal.cpp)
+target_include_directories(bitcoin-tidy SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
+
+# Disable RTTI and exceptions as necessary
+if (MSVC)
+    target_compile_options(bitcoin-tidy PRIVATE /GR-)
+else()
+    target_compile_options(bitcoin-tidy PRIVATE -fno-rtti)
+    target_compile_options(bitcoin-tidy PRIVATE -fno-exceptions)
+endif()
+
+if(CMAKE_HOST_APPLE)
+    # ld64 expects no undefined symbols by default
+    target_link_options(bitcoin-tidy PRIVATE -Wl,-flat_namespace)
+    target_link_options(bitcoin-tidy PRIVATE -Wl,-undefined -Wl,suppress)
+endif()
+
+# Add warnings
+if (MSVC)
+    target_compile_options(bitcoin-tidy PRIVATE /W4)
+else()
+    target_compile_options(bitcoin-tidy PRIVATE -Wall)
+    target_compile_options(bitcoin-tidy PRIVATE -Wextra)
+endif()
+
+if(CMAKE_VERSION VERSION_LESS 3.27)
+    set(CLANG_TIDY_COMMAND "${CLANG_TIDY_EXE}" "--load=${CMAKE_BINARY_DIR}/${CMAKE_SHARED_MODULE_PREFIX}bitcoin-tidy${CMAKE_SHARED_MODULE_SUFFIX}" "-checks=-*,bitcoin-*")
+else()
+    # CLANG_TIDY_COMMAND supports generator expressions as of 3.27
+    set(CLANG_TIDY_COMMAND "${CLANG_TIDY_EXE}" "--load=$<TARGET_FILE:bitcoin-tidy>" "-checks=-*,bitcoin-*")
+endif()
+
+# Create a dummy library that runs clang-tidy tests as a side-effect of building
+add_library(bitcoin-tidy-tests OBJECT EXCLUDE_FROM_ALL example_logprintf.cpp example_nontrivial-threadlocal.cpp)
+add_dependencies(bitcoin-tidy-tests bitcoin-tidy)
+
+set_target_properties(bitcoin-tidy-tests PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_COMMAND}")
+
+
+install(TARGETS bitcoin-tidy LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})

contrib/devtools/bitcoin-tidy/README.md

@@ -0,0 +1,11 @@
+# Bitcoin Tidy
+
+Example Usage:
+
+```bash
+cmake -S . -B build -DLLVM_DIR=$(llvm-config --cmakedir) -DCMAKE_BUILD_TYPE=Release
+
+cmake --build build -j$(nproc)
+
+cmake --build build --target bitcoin-tidy-tests -j$(nproc)
+```

contrib/devtools/bitcoin-tidy/bitcoin-tidy.cpp

@@ -0,0 +1,24 @@
+// Copyright (c) 2023 Bitcoin Developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "logprintf.h"
+#include "nontrivial-threadlocal.h"
+
+#include <clang-tidy/ClangTidyModule.h>
+#include <clang-tidy/ClangTidyModuleRegistry.h>
+
+class BitcoinModule final : public clang::tidy::ClangTidyModule
+{
+public:
+    void addCheckFactories(clang::tidy::ClangTidyCheckFactories& CheckFactories) override
+    {
+        CheckFactories.registerCheck<bitcoin::LogPrintfCheck>("bitcoin-unterminated-logprintf");
+        CheckFactories.registerCheck<bitcoin::NonTrivialThreadLocal>("bitcoin-nontrivial-threadlocal");
+    }
+};
+
+static clang::tidy::ClangTidyModuleRegistry::Add<BitcoinModule>
+    X("bitcoin-module", "Adds bitcoin checks.");
+
+volatile int BitcoinModuleAnchorSource = 0;

contrib/devtools/bitcoin-tidy/example_logprintf.cpp

@@ -0,0 +1,108 @@
+// Copyright (c) 2023 Bitcoin Developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <string>
+
+// Test for bitcoin-unterminated-logprintf
+
+enum LogFlags {
+    NONE
+};
+
+enum Level {
+    None
+};
+
+template <typename... Args>
+static inline void LogPrintf_(const std::string& logging_function, const std::string& source_file, const int source_line, const LogFlags flag, const Level level, const char* fmt, const Args&... args)
+{
+}
+
+#define LogPrintLevel_(category, level, ...) LogPrintf_(__func__, __FILE__, __LINE__, category, level, __VA_ARGS__)
+#define LogPrintf(...) LogPrintLevel_(LogFlags::NONE, Level::None, __VA_ARGS__)
+
+#define LogPrint(category, ...) \
+    do {                        \
+        LogPrintf(__VA_ARGS__); \
+    } while (0)
+
+
+class CWallet
+{
+    std::string GetDisplayName() const
+    {
+        return "default wallet";
+    }
+
+public:
+    template <typename... Params>
+    void WalletLogPrintf(const char* fmt, Params... parameters) const
+    {
+        LogPrintf(("%s " + std::string{fmt}).c_str(), GetDisplayName(), parameters...);
+    };
+};
+
+struct ScriptPubKeyMan
+{
+    std::string GetDisplayName() const
+    {
+        return "default wallet";
+    }
+
+    template <typename... Params>
+    void WalletLogPrintf(const char* fmt, Params... parameters) const
+    {
+        LogPrintf(("%s " + std::string{fmt}).c_str(), GetDisplayName(), parameters...);
+    };
+};
+
+void good_func()
+{
+    LogPrintf("hello world!\n");
+}
+void good_func2()
+{
+    CWallet wallet;
+    wallet.WalletLogPrintf("hi\n");
+    ScriptPubKeyMan spkm;
+    spkm.WalletLogPrintf("hi\n");
+
+    const CWallet& walletref = wallet;
+    walletref.WalletLogPrintf("hi\n");
+
+    auto* walletptr = new CWallet();
+    walletptr->WalletLogPrintf("hi\n");
+    delete walletptr;
+}
+void bad_func()
+{
+    LogPrintf("hello world!");
+}
+void bad_func2()
+{
+    LogPrintf("");
+}
+void bad_func3()
+{
+    // Ending in "..." has no special meaning.
+    LogPrintf("hello world!...");
+}
+void bad_func4_ignored()
+{
+    LogPrintf("hello world!"); // NOLINT(bitcoin-unterminated-logprintf)
+}
+void bad_func5()
+{
+    CWallet wallet;
+    wallet.WalletLogPrintf("hi");
+    ScriptPubKeyMan spkm;
+    spkm.WalletLogPrintf("hi");
+
+    const CWallet& walletref = wallet;
+    walletref.WalletLogPrintf("hi");
+
+    auto* walletptr = new CWallet();
+    walletptr->WalletLogPrintf("hi");
+    delete walletptr;
+}

contrib/devtools/bitcoin-tidy/example_nontrivial-threadlocal.cpp

@@ -0,0 +1,2 @@
+#include <string>
+thread_local std::string foo;

contrib/devtools/bitcoin-tidy/logprintf.cpp

@@ -0,0 +1,60 @@
+// Copyright (c) 2023 Bitcoin Developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "logprintf.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/ASTMatchers/ASTMatchFinder.h>
+
+
+namespace {
+AST_MATCHER(clang::StringLiteral, unterminated)
+{
+    size_t len = Node.getLength();
+    if (len > 0 && Node.getCodeUnit(len - 1) == '\n') {
+        return false;
+    }
+    return true;
+}
+} // namespace
+
+namespace bitcoin {
+
+void LogPrintfCheck::registerMatchers(clang::ast_matchers::MatchFinder* finder)
+{
+    using namespace clang::ast_matchers;
+
+    /*
+      Logprintf(..., ..., ..., ..., ..., "foo", ...)
+    */
+
+    finder->addMatcher(
+        callExpr(
+            callee(functionDecl(hasName("LogPrintf_"))),
+            hasArgument(5, stringLiteral(unterminated()).bind("logstring"))),
+        this);
+
+    /*
+      auto walletptr = &wallet;
+      wallet.WalletLogPrintf("foo");
+      wallet->WalletLogPrintf("foo");
+    */
+    finder->addMatcher(
+        cxxMemberCallExpr(
+            callee(cxxMethodDecl(hasName("WalletLogPrintf"))),
+            hasArgument(0, stringLiteral(unterminated()).bind("logstring"))),
+        this);
+}
+
+void LogPrintfCheck::check(const clang::ast_matchers::MatchFinder::MatchResult& Result)
+{
+    if (const clang::StringLiteral* lit = Result.Nodes.getNodeAs<clang::StringLiteral>("logstring")) {
+        const clang::ASTContext& ctx = *Result.Context;
+        const auto user_diag = diag(lit->getEndLoc(), "Unterminated format string used with LogPrintf");
+        const auto& loc = lit->getLocationOfByte(lit->getByteLength(), *Result.SourceManager, ctx.getLangOpts(), ctx.getTargetInfo());
+        user_diag << clang::FixItHint::CreateInsertion(loc, "\\n");
+    }
+}
+
+} // namespace bitcoin

contrib/devtools/bitcoin-tidy/logprintf.h

@@ -0,0 +1,29 @@
+// Copyright (c) 2023 Bitcoin Developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef LOGPRINTF_CHECK_H
+#define LOGPRINTF_CHECK_H
+
+#include <clang-tidy/ClangTidyCheck.h>
+
+namespace bitcoin {
+
+// Warn about any use of LogPrintf that does not end with a newline.
+class LogPrintfCheck final : public clang::tidy::ClangTidyCheck
+{
+public:
+    LogPrintfCheck(clang::StringRef Name, clang::tidy::ClangTidyContext* Context)
+        : clang::tidy::ClangTidyCheck(Name, Context) {}
+
+    bool isLanguageVersionSupported(const clang::LangOptions& LangOpts) const override
+    {
+        return LangOpts.CPlusPlus;
+    }
+    void registerMatchers(clang::ast_matchers::MatchFinder* Finder) override;
+    void check(const clang::ast_matchers::MatchFinder::MatchResult& Result) override;
+};
+
+} // namespace bitcoin
+
+#endif // LOGPRINTF_CHECK_H

contrib/devtools/bitcoin-tidy/nontrivial-threadlocal.cpp

@@ -0,0 +1,44 @@
+// Copyright (c) 2023 Bitcoin Developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "nontrivial-threadlocal.h"
+
+#include <clang/AST/ASTContext.h>
+#include <clang/ASTMatchers/ASTMatchFinder.h>
+
+
+// Copied from clang-tidy's UnusedRaiiCheck
+namespace {
+AST_MATCHER(clang::CXXRecordDecl, hasNonTrivialDestructor) {
+    // TODO: If the dtor is there but empty we don't want to warn either.
+    return Node.hasDefinition() && Node.hasNonTrivialDestructor();
+}
+} // namespace
+
+namespace bitcoin {
+
+void NonTrivialThreadLocal::registerMatchers(clang::ast_matchers::MatchFinder* finder)
+{
+    using namespace clang::ast_matchers;
+
+    /*
+      thread_local std::string foo;
+    */
+
+    finder->addMatcher(
+        varDecl(
+            hasThreadStorageDuration(),
+            hasType(hasCanonicalType(recordType(hasDeclaration(cxxRecordDecl(hasNonTrivialDestructor())))))
+        ).bind("nontrivial_threadlocal"),
+        this);
+}
+
+void NonTrivialThreadLocal::check(const clang::ast_matchers::MatchFinder::MatchResult& Result)
+{
+    if (const clang::VarDecl* var = Result.Nodes.getNodeAs<clang::VarDecl>("nontrivial_threadlocal")) {
+        const auto user_diag = diag(var->getBeginLoc(), "Variable with non-trivial destructor cannot be thread_local.");
+    }
+}
+
+} // namespace bitcoin

contrib/devtools/bitcoin-tidy/nontrivial-threadlocal.h

@@ -0,0 +1,29 @@
+// Copyright (c) 2023 Bitcoin Developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef NONTRIVIAL_THREADLOCAL_CHECK_H
+#define NONTRIVIAL_THREADLOCAL_CHECK_H
+
+#include <clang-tidy/ClangTidyCheck.h>
+
+namespace bitcoin {
+
+// Warn about any thread_local variable with a non-trivial destructor.
+class NonTrivialThreadLocal final : public clang::tidy::ClangTidyCheck
+{
+public:
+    NonTrivialThreadLocal(clang::StringRef Name, clang::tidy::ClangTidyContext* Context)
+        : clang::tidy::ClangTidyCheck(Name, Context) {}
+
+    bool isLanguageVersionSupported(const clang::LangOptions& LangOpts) const override
+    {
+        return LangOpts.CPlusPlus;
+    }
+    void registerMatchers(clang::ast_matchers::MatchFinder* Finder) override;
+    void check(const clang::ast_matchers::MatchFinder::MatchResult& Result) override;
+};
+
+} // namespace bitcoin
+
+#endif // NONTRIVIAL_THREADLOCAL_CHECK_H

contrib/devtools/check-deps.sh

@@ -0,0 +1,203 @@
+#!/usr/bin/env bash
+
+export LC_ALL=C
+set -Eeuo pipefail
+
+# Declare paths to libraries
+declare -A LIBS
+LIBS[cli]="libbitcoin_cli.a"
+LIBS[common]="libbitcoin_common.a"
+LIBS[consensus]="libbitcoin_consensus.a"
+LIBS[crypto]="crypto/.libs/libbitcoin_crypto_base.a crypto/.libs/libbitcoin_crypto_x86_shani.a crypto/.libs/libbitcoin_crypto_sse41.a crypto/.libs/libbitcoin_crypto_avx2.a"
+LIBS[node]="libbitcoin_node.a"
+LIBS[util]="libbitcoin_util.a"
+LIBS[wallet]="libbitcoin_wallet.a"
+LIBS[wallet_tool]="libbitcoin_wallet_tool.a"
+
+# Declare allowed dependencies "X Y" where X is allowed to depend on Y. This
+# list is taken from doc/design/libraries.md.
+ALLOWED_DEPENDENCIES=(
+    "cli common"
+    "cli util"
+    "common consensus"
+    "common crypto"
+    "common util"
+    "consensus crypto"
+    "node common"
+    "node consensus"
+    "node crypto"
+    "node kernel"
+    "node util"
+    "util crypto"
+    "wallet common"
+    "wallet crypto"
+    "wallet util"
+    "wallet_tool util"
+    "wallet_tool wallet"
+)
+
+# Add minor dependencies omitted from doc/design/libraries.md to keep the
+# dependency diagram simple.
+ALLOWED_DEPENDENCIES+=(
+    "wallet consensus"
+    "wallet_tool common"
+    "wallet_tool crypto"
+)
+
+# Declare list of known errors that should be suppressed.
+declare -A SUPPRESS
+# init.cpp file currently calls Berkeley DB sanity check function on startup, so
+# there is an undocumented dependency of the node library on the wallet library.
+SUPPRESS["libbitcoin_node_a-init.o libbitcoin_wallet_a-bdb.o _ZN6wallet27BerkeleyDatabaseSanityCheckEv"]=1
+# init/common.cpp file calls InitError and InitWarning from interface_ui which
+# is currently part of the node library. interface_ui should just be part of the
+# common library instead, and is moved in
+# https://github.com/bitcoin/bitcoin/issues/10102
+SUPPRESS["libbitcoin_common_a-common.o libbitcoin_node_a-interface_ui.o _Z11InitWarningRK13bilingual_str"]=1
+SUPPRESS["libbitcoin_common_a-common.o libbitcoin_node_a-interface_ui.o _Z9InitErrorRK13bilingual_str"]=1
+# rpc/external_signer.cpp adds defines node RPC methods but is built as part of the
+# common library. It should be moved to the node library instead.
+SUPPRESS["libbitcoin_common_a-external_signer.o libbitcoin_node_a-server.o _ZN9CRPCTable13appendCommandERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPK11CRPCCommand"]=1
+
+usage() {
+   echo "Usage: $(basename "${BASH_SOURCE[0]}") [BUILD_DIR]"
+}
+
+# Output makefile targets, converting library .a paths to libtool .la targets
+lib_targets() {
+  for lib in "${!LIBS[@]}"; do
+      for lib_path in ${LIBS[$lib]}; do
+          # shellcheck disable=SC2001
+          sed 's:/.libs/\(.*\)\.a$:/\1.la:g' <<<"$lib_path"
+      done
+  done
+}
+
+# Extract symbol names and object names and write to text files
+extract_symbols() {
+    local temp_dir="$1"
+    for lib in "${!LIBS[@]}"; do
+        for lib_path in ${LIBS[$lib]}; do
+            nm -o "$lib_path" | grep ' T ' | awk '{print $3, $1}' >> "${temp_dir}/${lib}_exports.txt"
+            nm -o "$lib_path" | grep ' U ' | awk '{print $3, $1}' >> "${temp_dir}/${lib}_imports.txt"
+            awk '{print $1}' "${temp_dir}/${lib}_exports.txt" | sort -u > "${temp_dir}/${lib}_exported_symbols.txt"
+            awk '{print $1}' "${temp_dir}/${lib}_imports.txt" | sort -u > "${temp_dir}/${lib}_imported_symbols.txt"
+        done
+    done
+}
+
+# Lookup object name(s) corresponding to symbol name in text file
+obj_names() {
+    local symbol="$1"
+    local txt_file="$2"
+    sed -n "s/^$symbol [^:]\\+:\\([^:]\\+\\):[^:]*\$/\\1/p" "$txt_file" | sort -u
+}
+
+# Iterate through libraries and find disallowed dependencies
+check_libraries() {
+    local temp_dir="$1"
+    local result=0
+    for src in "${!LIBS[@]}"; do
+        for dst in "${!LIBS[@]}"; do
+            if [ "$src" != "$dst" ] && ! is_allowed "$src" "$dst"; then
+                if ! check_disallowed "$src" "$dst" "$temp_dir"; then
+                    result=1
+                fi
+            fi
+        done
+    done
+    check_not_suppressed
+    return $result
+}
+
+# Return whether src library is allowed to depend on dst.
+is_allowed() {
+    local src="$1"
+    local dst="$2"
+    for allowed in "${ALLOWED_DEPENDENCIES[@]}"; do
+        if [ "$src $dst" = "$allowed" ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+# Return whether src library imports any symbols from dst, assuming src is not
+# allowed to depend on dst.
+check_disallowed() {
+    local src="$1"
+    local dst="$2"
+    local temp_dir="$3"
+    local result=0
+
+    # Loop over symbol names exported by dst and imported by src
+    while read symbol; do
+        local dst_obj
+        dst_obj=$(obj_names "$symbol" "${temp_dir}/${dst}_exports.txt")
+        while read src_obj; do
+            if ! check_suppress "$src_obj" "$dst_obj" "$symbol"; then
+                echo "Error: $src_obj depends on $dst_obj symbol '$(c++filt "$symbol")', can suppess with:"
+                echo "    SUPPRESS[\"$src_obj $dst_obj $symbol\"]=1"
+                result=1
+            fi
+        done < <(obj_names "$symbol" "${temp_dir}/${src}_imports.txt")
+    done < <(comm -12 "${temp_dir}/${dst}_exported_symbols.txt" "${temp_dir}/${src}_imported_symbols.txt")
+    return $result
+}
+
+# Declare array to track errors which were suppressed.
+declare -A SUPPRESSED
+
+# Return whether error should be suppressed and record suppresssion in
+# SUPPRESSED array.
+check_suppress() {
+    local src_obj="$1"
+    local dst_obj="$2"
+    local symbol="$3"
+    for suppress in "${!SUPPRESS[@]}"; do
+        read suppress_src suppress_dst suppress_pattern <<<"$suppress"
+        if [[ "$src_obj" == "$suppress_src" && "$dst_obj" == "$suppress_dst" && "$symbol" =~ $suppress_pattern ]]; then
+            SUPPRESSED["$suppress"]=1
+            return 0
+        fi
+    done
+    return 1
+}
+
+# Warn about error which were supposed to be suppress, but were not encountered.
+check_not_suppressed() {
+    for suppress in "${!SUPPRESS[@]}"; do
+        if [[ ! -v SUPPRESSED[$suppress] ]]; then
+            echo >&2 "Warning: suppression '$suppress' was ignored, consider deleting."
+        fi
+    done
+}
+
+# Check arguments.
+if [ "$#" = 0 ]; then
+    BUILD_DIR="$(dirname "${BASH_SOURCE[0]}")/../../src"
+elif [ "$#" = 1 ]; then
+    BUILD_DIR="$1"
+else
+    echo >&2 "Error: wrong number of arguments."
+    usage >&2
+    exit 1
+fi
+if [ ! -f "$BUILD_DIR/Makefile" ]; then
+    echo >&2 "Error: directory '$BUILD_DIR' does not contain a makefile, please specify path to build directory for library targets."
+    usage >&2
+    exit 1
+fi
+
+# Build libraries and run checks.
+cd "$BUILD_DIR"
+# shellcheck disable=SC2046
+make -j"$(nproc)" $(lib_targets)
+TEMP_DIR="$(mktemp -d)"
+extract_symbols "$TEMP_DIR"
+if check_libraries "$TEMP_DIR"; then
+    echo "Success! No unexpected dependencies were detected."
+else
+    echo >&2 "Error: Unexpected dependencies were detected. Check previous output."
+fi
+rm -r "$TEMP_DIR"

contrib/devtools/circular-dependencies.py

@@ -5,7 +5,6 @@
 
 import sys
 import re
-from typing import Dict, List, Set
 
 MAPPING = {
     'core_read.cpp': 'core_io.cpp',
@@ -33,7 +32,7 @@ def module_name(path):
     return None
 
 files = dict()
-deps: Dict[str, Set[str]] = dict()
+deps: dict[str, set[str]] = dict()
 
 RE = re.compile("^#include <(.*)>")
 
@@ -65,7 +64,7 @@ while True:
     shortest_cycle = None
     for module in sorted(deps.keys()):
         # Build the transitive closure of dependencies of module
-        closure: Dict[str, List[str]] = dict()
+        closure: dict[str, list[str]] = dict()
         for dep in deps[module]:
             closure[dep] = []
         while True:

contrib/devtools/clang-format-diff.py

@@ -1,166 +1,190 @@
 #!/usr/bin/env python3
 #
-#===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===#
+# ===- clang-format-diff.py - ClangFormat Diff Reformatter ----*- python -*--===#
 #
-#                     The LLVM Compiler Infrastructure
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
-# This file is distributed under the University of Illinois Open Source
-# License.
-#
-#           ============================================================
-#
-# University of Illinois/NCSA
-# Open Source License
-#
-# Copyright (c) 2007-2015 University of Illinois at Urbana-Champaign.
-# All rights reserved.
-#
-# Developed by:
-#
-#     LLVM Team
-#
-#     University of Illinois at Urbana-Champaign
-#
-#     http://llvm.org
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy of
-# this software and associated documentation files (the "Software"), to deal with
-# the Software without restriction, including without limitation the rights to
-# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-# of the Software, and to permit persons to whom the Software is furnished to do
-# so, subject to the following conditions:
-#
-#     * Redistributions of source code must retain the above copyright notice,
-#       this list of conditions and the following disclaimers.
-#
-#     * Redistributions in binary form must reproduce the above copyright notice,
-#       this list of conditions and the following disclaimers in the
-#       documentation and/or other materials provided with the distribution.
-#
-#     * Neither the names of the LLVM Team, University of Illinois at
-#       Urbana-Champaign, nor the names of its contributors may be used to
-#       endorse or promote products derived from this Software without specific
-#       prior written permission.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-# CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
-# SOFTWARE.
-#
-#           ============================================================
-#
-#===------------------------------------------------------------------------===#
-
-r"""
-ClangFormat Diff Reformatter
-============================
+# ===------------------------------------------------------------------------===#
 
+"""
 This script reads input from a unified diff and reformats all the changed
 lines. This is useful to reformat all the lines touched by a specific patch.
 Example usage for git/svn users:
 
-  git diff -U0 HEAD^ | clang-format-diff.py -p1 -i
-  svn diff --diff-cmd=diff -x-U0 | clang-format-diff.py -i
+  git diff -U0 --no-color --relative HEAD^ | {clang_format_diff} -p1 -i
+  svn diff --diff-cmd=diff -x-U0 | {clang_format_diff} -i
 
+It should be noted that the filename contained in the diff is used unmodified
+to determine the source file to update. Users calling this script directly
+should be careful to ensure that the path in the diff is correct relative to the
+current working directory.
 """
+from __future__ import absolute_import, division, print_function
 
 import argparse
 import difflib
-import io
 import re
 import subprocess
 import sys
 
-
-# Change this to the full path if clang-format is not on the path.
-binary = 'clang-format'
+from io import StringIO
 
 
 def main():
-  parser = argparse.ArgumentParser(description=
-                                   'Reformat changed lines in diff. Without -i '
-                                   'option just output the diff that would be '
-                                   'introduced.')
-  parser.add_argument('-i', action='store_true', default=False,
-                      help='apply edits to files instead of displaying a diff')
-  parser.add_argument('-p', metavar='NUM', default=0,
-                      help='strip the smallest prefix containing P slashes')
-  parser.add_argument('-regex', metavar='PATTERN', default=None,
-                      help='custom pattern selecting file paths to reformat '
-                      '(case sensitive, overrides -iregex)')
-  parser.add_argument('-iregex', metavar='PATTERN', default=
-                      r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc|js|ts|proto'
-                      r'|protodevel|java)',
-                      help='custom pattern selecting file paths to reformat '
-                      '(case insensitive, overridden by -regex)')
-  parser.add_argument('-sort-includes', action='store_true', default=False,
-                      help='let clang-format sort include blocks')
-  parser.add_argument('-v', '--verbose', action='store_true',
-                      help='be more verbose, ineffective without -i')
-  args = parser.parse_args()
+    parser = argparse.ArgumentParser(
+        description=__doc__.format(clang_format_diff="%(prog)s"),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "-i",
+        action="store_true",
+        default=False,
+        help="apply edits to files instead of displaying a diff",
+    )
+    parser.add_argument(
+        "-p",
+        metavar="NUM",
+        default=0,
+        help="strip the smallest prefix containing P slashes",
+    )
+    parser.add_argument(
+        "-regex",
+        metavar="PATTERN",
+        default=None,
+        help="custom pattern selecting file paths to reformat "
+        "(case sensitive, overrides -iregex)",
+    )
+    parser.add_argument(
+        "-iregex",
+        metavar="PATTERN",
+        default=r".*\.(?:cpp|cc|c\+\+|cxx|cppm|ccm|cxxm|c\+\+m|c|cl|h|hh|hpp"
+        r"|hxx|m|mm|inc|js|ts|proto|protodevel|java|cs|json|s?vh?)",
+        help="custom pattern selecting file paths to reformat "
+        "(case insensitive, overridden by -regex)",
+    )
+    parser.add_argument(
+        "-sort-includes",
+        action="store_true",
+        default=False,
+        help="let clang-format sort include blocks",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="be more verbose, ineffective without -i",
+    )
+    parser.add_argument(
+        "-style",
+        help="formatting style to apply (LLVM, GNU, Google, Chromium, "
+        "Microsoft, Mozilla, WebKit)",
+    )
+    parser.add_argument(
+        "-fallback-style",
+        help="The name of the predefined style used as a"
+        "fallback in case clang-format is invoked with"
+        "-style=file, but can not find the .clang-format"
+        "file to use.",
+    )
+    parser.add_argument(
+        "-binary",
+        default="clang-format",
+        help="location of binary to use for clang-format",
+    )
+    args = parser.parse_args()
 
-  # Extract changed lines for each file.
-  filename = None
-  lines_by_file = {}
-  for line in sys.stdin:
-    match = re.search(r'^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line)
-    if match:
-      filename = match.group(2)
-    if filename is None:
-      continue
+    # Extract changed lines for each file.
+    filename = None
+    lines_by_file = {}
+    for line in sys.stdin:
+        match = re.search(r"^\+\+\+\ (.*?/){%s}(\S*)" % args.p, line)
+        if match:
+            filename = match.group(2)
+        if filename is None:
+            continue
 
-    if args.regex is not None:
-      if not re.match('^%s$' % args.regex, filename):
-        continue
-    else:
-      if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
-        continue
+        if args.regex is not None:
+            if not re.match("^%s$" % args.regex, filename):
+                continue
+        else:
+            if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE):
+                continue
 
-    match = re.search(r'^@@.*\+(\d+)(,(\d+))?', line)
-    if match:
-      start_line = int(match.group(1))
-      line_count = 1
-      if match.group(3):
-        line_count = int(match.group(3))
-      if line_count == 0:
-        continue
-      end_line = start_line + line_count - 1
-      lines_by_file.setdefault(filename, []).extend(
-          ['-lines', str(start_line) + ':' + str(end_line)])
+        match = re.search(r"^@@.*\+(\d+)(?:,(\d+))?", line)
+        if match:
+            start_line = int(match.group(1))
+            line_count = 1
+            if match.group(2):
+                line_count = int(match.group(2))
+                # The input is something like
+                #
+                # @@ -1, +0,0 @@
+                #
+                # which means no lines were added.
+                if line_count == 0:
+                    continue
+            # Also format lines range if line_count is 0 in case of deleting
+            # surrounding statements.
+            end_line = start_line
+            if line_count != 0:
+                end_line += line_count - 1
+            lines_by_file.setdefault(filename, []).extend(
+                ["-lines", str(start_line) + ":" + str(end_line)]
+            )
 
-  # Reformat files containing changes in place.
-  for filename, lines in lines_by_file.items():
-    if args.i and args.verbose:
-      print('Formatting {}'.format(filename))
-    command = [binary, filename]
-    if args.i:
-      command.append('-i')
-    if args.sort_includes:
-      command.append('-sort-includes')
-    command.extend(lines)
-    command.extend(['-style=file', '-fallback-style=none'])
-    p = subprocess.Popen(command,
-                         stdout=subprocess.PIPE,
-                         stderr=None,
-                         stdin=subprocess.PIPE,
-                         text=True)
-    stdout, stderr = p.communicate()
-    if p.returncode != 0:
-      sys.exit(p.returncode)
+    # Reformat files containing changes in place.
+    for filename, lines in lines_by_file.items():
+        if args.i and args.verbose:
+            print("Formatting {}".format(filename))
+        command = [args.binary, filename]
+        if args.i:
+            command.append("-i")
+        if args.sort_includes:
+            command.append("-sort-includes")
+        command.extend(lines)
+        if args.style:
+            command.extend(["-style", args.style])
+        if args.fallback_style:
+            command.extend(["-fallback-style", args.fallback_style])
 
-    if not args.i:
-      with open(filename, encoding="utf8") as f:
-        code = f.readlines()
-      formatted_code = io.StringIO(stdout).readlines()
-      diff = difflib.unified_diff(code, formatted_code,
-                                  filename, filename,
-                                  '(before formatting)', '(after formatting)')
-      diff_string = ''.join(diff)
-      if len(diff_string) > 0:
-        sys.stdout.write(diff_string)
+        try:
+            p = subprocess.Popen(
+                command,
+                stdout=subprocess.PIPE,
+                stderr=None,
+                stdin=subprocess.PIPE,
+                universal_newlines=True,
+            )
+        except OSError as e:
+            # Give the user more context when clang-format isn't
+            # found/isn't executable, etc.
+            raise RuntimeError(
+                'Failed to run "%s" - %s"' % (" ".join(command), e.strerror)
+            )
 
-if __name__ == '__main__':
-  main()
+        stdout, stderr = p.communicate()
+        if p.returncode != 0:
+            sys.exit(p.returncode)
+
+        if not args.i:
+            with open(filename, encoding="utf8") as f:
+                code = f.readlines()
+            formatted_code = StringIO(stdout).readlines()
+            diff = difflib.unified_diff(
+                code,
+                formatted_code,
+                filename,
+                filename,
+                "(before formatting)",
+                "(after formatting)",
+            )
+            diff_string = "".join(diff)
+            if len(diff_string) > 0:
+                sys.stdout.write(diff_string)
+                sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()