Merge bitcoin/bitcoin#27660: [24.1] Final Changes

89a5a416de doc: update release notes for 24.1 (fanquake)
b36dff9fa8 doc: update manual pages for 24.1 (fanquake)
8af2d2f762 build: bump version to 24.1 final (fanquake)

Pull request description:

  Final changes for `v24.1`.
  PR for bitcoincore.org is here: https://github.com/bitcoin-core/bitcoincore.org/pull/968.

ACKs for top commit:
  josibake:
    ACK 89a5a416de
  hebasto:
    ACK 89a5a416de, I have reviewed the code and it looks OK.
  stickies-v:
    ACK 89a5a416de

Tree-SHA512: 6eb7d30790b50cfc7232cf7c69b5c8b1fbb943cc715d275ff4d8c2ade93d82bf2bc659b82a06b8453e2e0382e9d375aee2c71d0fee44285f9eb8a029980a29d3

.cirrus.yml

@@ -2,6 +2,7 @@ env:  # Global defaults
   PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y"
   MAKEJOBS: "-j10"
   TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
+  CI_FAILFAST_TEST_LEAVE_DANGLING: "1"  # Cirrus CI does not care about dangling process and setting this variable avoids killing the CI script itself on error
   CCACHE_SIZE: "200M"
   CCACHE_DIR: "/tmp/ccache_dir"
   CCACHE_NOHASHDIR: "1"  # Debug info might contain a stale path if the build dir changes, but this is fine
@@ -23,33 +24,38 @@ filter_template: &FILTER_TEMPLATE
 base_template: &BASE_TEMPLATE
   << : *FILTER_TEMPLATE
   merge_base_script:
-    # Unconditionally install git (used in fingerprint_script) and set the
-    # default git author name (used in verify-commits.py)
+    # Unconditionally install git (used in fingerprint_script).
     - bash -c "$PACKAGE_MANAGER_INSTALL git"
-    - git config --global user.email "ci@ci.ci"
-    - git config --global user.name "ci"
     - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
-    - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH
-    - git merge FETCH_HEAD  # Merge base to detect silent merge conflicts
+    - git fetch $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge"
+    - git checkout FETCH_HEAD  # Use merged changes to detect silent merge conflicts
 
 main_template: &MAIN_TEMPLATE
   timeout_in: 120m  # https://cirrus-ci.org/faq/#instance-timed-out
+  ccache_cache:
+    folder: "/tmp/ccache_dir"
+  ci_script:
+    - ./ci/test_run_all.sh
+
+global_task_template: &GLOBAL_TASK_TEMPLATE
+  << : *BASE_TEMPLATE
   container:
     # https://cirrus-ci.org/faq/#are-there-any-limits
     # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel
     cpu: 2
     greedy: true
     memory: 8G  # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers
-  ccache_cache:
-    folder: "/tmp/ccache_dir"
   depends_built_cache:
     folder: "depends/built"
     fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-list -1 HEAD ./depends)
-  ci_script:
-    - ./ci/test_run_all.sh
+  << : *MAIN_TEMPLATE
 
-global_task_template: &GLOBAL_TASK_TEMPLATE
+macos_native_task_template: &MACOS_NATIVE_TASK_TEMPLATE
   << : *BASE_TEMPLATE
+  check_clang_script:
+    - clang --version
+  brew_install_script:
+    - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt
   << : *MAIN_TEMPLATE
 
 compute_credits_template: &CREDITS_TEMPLATE
@@ -72,39 +78,48 @@ task:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
 
 task:
-  name: "Win64 native [msvc]"
+  name: 'tidy [jammy]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:jammy
+    cpu: 2
+    memory: 5G
+  # For faster CI feedback, immediately schedule the linters
+  << : *CREDITS_TEMPLATE
+  env:
+    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+    FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh"
+
+task:
+  name: "Win64 native [vs2022]"
   << : *FILTER_TEMPLATE
   windows_container:
-    cpu: 4
-    memory: 8G
-    image: cirrusci/windowsservercore:visualstudio2019
+    cpu: 6
+    memory: 12G
+    image: cirrusci/windowsservercore:visualstudio2022
   timeout_in: 120m
   env:
-    PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%'
+    PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Current\Bin;%PATH%'
     PYTHONUTF8: 1
-    CI_VCPKG_TAG: '2021.05.12'
+    CI_VCPKG_TAG: '2022.06.16.1'
     VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads'
     VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives'
-    QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.3/single/qt-everywhere-opensource-src-5.15.3.zip'
-    QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.3.zip'
-    QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.3'
+    CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache'
+    WRAPPED_CL: 'C:\Users\ContainerAdministrator\AppData\Local\Temp\cirrus-ci-build\ci\test\wrapped-cl.bat'
+    QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip'
+    QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.5.zip'
+    QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.5'
     QTBASEDIR: 'C:\Qt_static'
-    x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"'
+    x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"'
+    QT_CONFIGURE_COMMAND: '..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml'
     IgnoreWarnIntDirInTempDetected: 'true'
   merge_script:
-    - git config --global user.email "ci@ci.ci"
-    - git config --global user.name "ci"
-    # Windows filesystem loses the executable bit, and all of the executable
-    # files are considered "modified" now. It will break the following `git merge`
-    # command. The next two commands make git ignore this issue.
-    - git config core.filemode false
-    - git reset --hard
-    - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL $env:CIRRUS_BASE_BRANCH; git merge FETCH_HEAD; }
+    - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL pull/$env:CIRRUS_PR/merge; git checkout FETCH_HEAD; }
   msvc_qt_built_cache:
     folder: "%QTBASEDIR%"
     reupload_on_changes: false
     fingerprint_script:
-      - echo %QT_DOWNLOAD_URL%
+      - echo %QT_DOWNLOAD_URL% %QT_CONFIGURE_COMMAND%
       - msbuild -version
     populate_script:
       - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip
@@ -116,7 +131,7 @@ task:
       - cd %QT_SOURCE_DIR%
       - mkdir build
       - cd build
-      - ..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml -prefix %QTBASEDIR%
+      - '%QT_CONFIGURE_COMMAND% -prefix %QTBASEDIR%'
       - jom
       - jom install
   vcpkg_tools_cache:
@@ -130,12 +145,17 @@ task:
     reupload_on_changes: true
     fingerprint_script:
       - echo %CI_VCPKG_TAG%
+      - type build_msvc\vcpkg.json
       - msbuild -version
     populate_script:
       - mkdir %VCPKG_DEFAULT_BINARY_CACHE%
-  install_python_script:
+  ccache_cache:
+    folder: '%CCACHE_DIR%'
+  install_tools_script:
+    - choco install --yes --no-progress ccache --version=4.6.1
     - choco install --yes --no-progress python3 --version=3.9.6
     - pip install zmq
+    - ccache --version
     - python -VV
   install_vcpkg_script:
     - cd ..
@@ -147,9 +167,12 @@ task:
     - .\vcpkg integrate install
     - .\vcpkg version
   build_script:
+    - '%x64_NATIVE_TOOLS%'
     - cd %CIRRUS_WORKING_DIR%
+    - ccache --zero-stats --max-size=%CCACHE_SIZE%
     - python build_msvc\msvc-autogen.py
-    - msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo
+    - msbuild build_msvc\bitcoin.sln -property:CLToolExe=%WRAPPED_CL% -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo
+    - ccache --show-stats
   unit_tests_script:
     - src\test_bitcoin.exe -l test_suite
     - src\bench_bitcoin.exe > NUL
@@ -195,7 +218,7 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
 
 task:
-  name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [bionic]'
+  name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [buster]'
   previous_releases_cache:
     folder: "releases"
   << : *GLOBAL_TASK_TEMPLATE
@@ -226,12 +249,20 @@ task:
     MAKEJOBS: "-j4"  # Avoid excessive memory use due to MSan
 
 task:
-  name: '[ASan + LSan + UBSan + integer, no depends] [jammy]'
+  name: '[ASan + LSan + UBSan + integer, no depends, USDT] [jammy]'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:jammy
+  # We can't use a 'container' for the USDT interface tests as the CirrusCI
+  # containers don't have privileges to hook into bitcoind. CirrusCI uses
+  # Google Compute Engine instances: https://cirrus-ci.org/guide/custom-vms/
+  # Images can be found here: https://cloud.google.com/compute/docs/images/os-details
+  compute_engine_instance:
+    image_project: ubuntu-os-cloud
+    image: family/ubuntu-2204-lts # when upgrading, check if we can drop "ADD_UNTRUSTED_BPFCC_PPA"
+    cpu: 4
+    memory: 12G
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+    HOME: /root/  # Only needed for compute_engine_instance
     FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
     MAKEJOBS: "-j4"  # Avoid excessive memory use
 
@@ -258,13 +289,13 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh"
 
 task:
-  name: '[no wallet] [bionic]'
+  name: '[no wallet, libbitcoinkernel] [bionic]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
     image: ubuntu:bionic
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh"
+    FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh"
 
 task:
   name: 'macOS 10.15 [gui, no tests] [focal]'
@@ -281,25 +312,23 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_mac.sh"
 
 task:
-  name: 'macOS 12 native [gui, system sqlite only] [no depends]'
-  brew_install_script:
-    - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt
-  << : *GLOBAL_TASK_TEMPLATE
+  name: 'macOS 13 native arm64 [gui, sqlite only] [no depends]'
   macos_instance:
     # Use latest image, but hardcode version to avoid silent upgrades (and breaks)
-    image: monterey-xcode-13.2  # https://cirrus-ci.org/guide/macOS
+    image: ghcr.io/cirruslabs/macos-ventura-xcode:14.1  # https://cirrus-ci.org/guide/macOS
+  << : *MACOS_NATIVE_TASK_TEMPLATE
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     CI_USE_APT_INSTALL: "no"
     PACKAGE_MANAGER_INSTALL: "echo"  # Nothing to do
-    FILE_ENV: "./ci/test/00_setup_env_mac_host.sh"
+    FILE_ENV: "./ci/test/00_setup_env_mac_native_arm64.sh"
 
 task:
   name: 'ARM64 Android APK [focal]'
   << : *BASE_TEMPLATE
   android_sdk_cache:
     folder: "depends/SDKs/android"
-    fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.1.7779620"
+    fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.2.8568313"
   depends_sources_cache:
     folder: "depends/sources"
     fingerprint_script: git rev-list -1 HEAD ./depends

.gitignore

@@ -9,6 +9,7 @@ src/bitcoin-gui
 src/bitcoin-node
 src/bitcoin-tx
 src/bitcoin-util
+src/bitcoin-chainstate
 src/bitcoin-wallet
 src/test/fuzz/fuzz
 src/test/test_bitcoin
@@ -43,8 +44,6 @@ src/obj
 share/setup.nsi
 share/qt/Info.plist
 
-src/univalue/gen
-
 src/qt/*.moc
 src/qt/moc_*.cpp
 src/qt/forms/ui_*.h
@@ -76,7 +75,6 @@ src/qt/bitcoin-qt.includes
 *.log
 *.trs
 *.dmg
-*.iso
 
 *.json.h
 *.raw.h
@@ -150,3 +148,5 @@ osx_volname
 dist/
 
 /guix-build-*
+
+/ci/scratch/

.tx/config

@@ -1,7 +1,7 @@
 [main]
 host = https://www.transifex.com
 
-[o:bitcoin:p:bitcoin:r:qt-translation-023x]
+[o:bitcoin:p:bitcoin:r:qt-translation-024x]
 file_filter = src/qt/locale/bitcoin_<lang>.xlf
 source_file = src/qt/locale/bitcoin_en.xlf
 source_lang = en

CONTRIBUTING.md

@@ -10,10 +10,9 @@ First, in terms of structure, there is no particular concept of "Bitcoin Core
 developers" in the sense of privileged people. Open source often naturally
 revolves around a meritocracy where contributors earn trust from the developer
 community over time. Nevertheless, some hierarchy is necessary for practical
-purposes. As such, there are repository "maintainers" who are responsible for
-merging pull requests, as well as a "lead maintainer" who is responsible for the
-[release cycle](/doc/release-process.md) as well as overall merging, moderation
-and appointment of maintainers.
+purposes. As such, there are repository maintainers who are responsible for
+merging pull requests, the [release cycle](/doc/release-process.md), and
+moderation.
 
 Getting Started
 ---------------
@@ -153,7 +152,8 @@ the pull request affects. Valid areas as:
   - `test`, `qa` or `ci` for changes to the unit tests, QA tests or CI code
   - `util` or `lib` for changes to the utils or libraries
   - `wallet` for changes to the wallet code
-  - `build` for changes to the GNU Autotools or reproducible builds
+  - `build` for changes to the GNU Autotools or MSVC builds
+  - `guix` for changes to the GUIX reproducible builds
 
 Examples:
 
@@ -216,9 +216,9 @@ Please update the resulting commit message, if needed. It should read as a
 coherent message. In most cases, this means not just listing the interim
 commits.
 
-If you have problems with squashing or other git workflows, you can enable
-"Allow edits from maintainers" in the right-hand sidebar of the GitHub web
-interface and ask for help in the pull request.
+If your change contains a merge commit, the above workflow may not work and you
+will need to remove the merge commit first. See the next section for details on
+how to rebase.
 
 Please refrain from creating several pull requests for the same change.
 Use the pull request that is already open (or was created earlier) to amend
@@ -231,7 +231,9 @@ pull request to pull request.
 ### Rebasing Changes
 
 When a pull request conflicts with the target branch, you may be asked to rebase it on top of the current target branch.
-The `git rebase` command will take care of rebuilding your commits on top of the new base.
+
+    git fetch https://github.com/bitcoin/bitcoin  # Fetch the latest upstream commit
+    git rebase FETCH_HEAD  # Rebuild commits on top of the new base
 
 This project aims to have a clean git history, where code changes are only made in non-merge commits. This simplifies
 auditability because merge commits can be assumed to not contain arbitrary code changes. Merge commits should be signed,
@@ -292,7 +294,7 @@ projects such as libsecp256k1), and is not to be confused with overall Bitcoin
 Network Protocol consensus changes.
 
 Whether a pull request is merged into Bitcoin Core rests with the project merge
-maintainers and ultimately the project lead.
+maintainers.
 
 Maintainers will take into consideration if a patch is in line with the general
 principles of the project; meets the minimum standards for inclusion; and will

Makefile.am

@@ -12,7 +12,7 @@ if ENABLE_MAN
 SUBDIRS += doc/man
 endif
 .PHONY: deploy FORCE
-.INTERMEDIATE: $(OSX_TEMP_ISO) $(COVERAGE_INFO)
+.INTERMEDIATE: $(COVERAGE_INFO)
 
 export PYTHONPATH
 
@@ -23,6 +23,7 @@ endif
 
 BITCOIND_BIN=$(top_builddir)/src/$(BITCOIN_DAEMON_NAME)$(EXEEXT)
 BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT)
+BITCOIN_TEST_BIN=$(top_builddir)/src/test/$(BITCOIN_TEST_NAME)$(EXEEXT)
 BITCOIN_CLI_BIN=$(top_builddir)/src/$(BITCOIN_CLI_NAME)$(EXEEXT)
 BITCOIN_TX_BIN=$(top_builddir)/src/$(BITCOIN_TX_NAME)$(EXEEXT)
 BITCOIN_UTIL_BIN=$(top_builddir)/src/$(BITCOIN_UTIL_NAME)$(EXEEXT)
@@ -37,8 +38,6 @@ space := $(empty) $(empty)
 OSX_APP=Bitcoin-Qt.app
 OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME))
 OSX_DMG = $(OSX_VOLNAME).dmg
-OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso
-OSX_BACKGROUND_IMAGE=$(top_srcdir)/contrib/macdeploy/background.tiff
 OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus
 OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns
 OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed
@@ -48,7 +47,8 @@ DIST_CONTRIB = \
 	       $(top_srcdir)/test/sanitizer_suppressions/tsan \
 	       $(top_srcdir)/test/sanitizer_suppressions/ubsan \
 	       $(top_srcdir)/contrib/linearize/linearize-data.py \
-	       $(top_srcdir)/contrib/linearize/linearize-hashes.py
+	       $(top_srcdir)/contrib/linearize/linearize-hashes.py \
+	       $(top_srcdir)/contrib/signet/miner
 
 DIST_SHARE = \
   $(top_srcdir)/share/genbuild.sh \
@@ -64,7 +64,6 @@ WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \
   $(top_srcdir)/doc/README_windows.txt
 
 OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_INSTALLER_ICONS) \
-  $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \
   $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh
 
 COVERAGE_INFO = $(COV_TOOL_WRAPPER) baseline.info \
@@ -80,6 +79,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive
 	$(MKDIR_P) $(top_builddir)/release
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIND_BIN) $(top_builddir)/release
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_QT_BIN) $(top_builddir)/release
+	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TEST_BIN) $(top_builddir)/release
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TX_BIN) $(top_builddir)/release
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_WALLET_BIN) $(top_builddir)/release
@@ -130,31 +130,16 @@ $(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING)
 deploydir: $(OSX_DMG)
 else !BUILD_DARWIN
 APP_DIST_DIR=$(top_builddir)/dist
-APP_DIST_EXTRAS=$(APP_DIST_DIR)/.background/background.tiff $(APP_DIST_DIR)/.DS_Store $(APP_DIST_DIR)/Applications
 
-$(APP_DIST_DIR)/Applications:
-	@rm -f $@
-	@cd $(@D); $(LN_S) /Applications $(@F)
-
-$(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt
-
-$(OSX_TEMP_ISO): $(APP_DIST_EXTRAS)
+$(OSX_DMG): deploydir
 	$(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ $(APP_DIST_DIR) -- $(if $(SOURCE_DATE_EPOCH),-volume_date all_file_dates =$(SOURCE_DATE_EPOCH))
 
-$(OSX_DMG): $(OSX_TEMP_ISO)
-	$(DMG) dmg "$<" "$@"
-
-$(APP_DIST_DIR)/.background/background.tiff:
-	$(MKDIR_P) $(@D)
-	cp $(OSX_BACKGROUND_IMAGE) $@
-
 $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING)
-	INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR)
+	INSTALL_NAME_TOOL=$(INSTALL_NAME_TOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR)
 
-deploydir: $(APP_DIST_EXTRAS)
+deploydir: $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt
 endif !BUILD_DARWIN
 
-appbundle: $(OSX_APP_BUILT)
 deploy: $(OSX_DMG)
 endif
 
@@ -191,7 +176,6 @@ LCOV_FILTER_PATTERN = \
 	-p "src/leveldb/" \
 	-p "src/crc32c/" \
 	-p "src/bench/" \
-	-p "src/univalue" \
 	-p "src/crypto/ctaes" \
 	-p "src/minisketch" \
 	-p "src/secp256k1" \

README.md

@@ -6,19 +6,15 @@ https://bitcoincore.org
 For an immediately usable, binary version of the Bitcoin Core software, see
 https://bitcoincore.org/en/download/.
 
+What is Bitcoin Core?
+---------------------
+
+Bitcoin Core connects to the Bitcoin peer-to-peer network to download and fully
+validate blocks and transactions. It also includes a wallet and graphical user
+interface, which can be optionally built.
+
 Further information about Bitcoin Core is available in the [doc folder](/doc).
 
-What is Bitcoin?
-----------------
-
-Bitcoin is an experimental digital currency that enables instant payments to
-anyone, anywhere in the world. Bitcoin uses peer-to-peer technology to operate
-with no central authority: managing transactions and issuing money are carried
-out collectively by the network. Bitcoin Core is the name of open source
-software which enables the use of this currency.
-
-For more information read the original Bitcoin whitepaper.
-
 License
 -------
 

REVIEWERS

@@ -15,126 +15,3 @@
 # review a pull request. Peer review is always welcome and is a critical
 # component of the progress of the codebase. Information on peer review
 # guidelines can be found in the CONTRIBUTING.md doc.
-
-
-# Maintainers
-#   @achow101
-#   @fanquake
-#   @hebasto
-#   @laanwj
-#   @marcofalke
-#   @sipa
-
-# Docs
-/doc/*[a-zA-Z-].md                          @harding
-/doc/Doxyfile.in                            @fanquake
-/doc/REST-interface.md                      @jonasschnelli
-/doc/benchmarking.md                        @ariard
-/doc/bitcoin-conf.md                        @hebasto
-/doc/build-freebsd.md                       @fanquake
-/doc/build-netbsd.md                        @fanquake
-/doc/build-openbsd.md                       @laanwj
-/doc/build-osx.md                           @fanquake
-/doc/build-unix.md                          @laanwj
-/doc/build-windows.md                       @sipsorcery
-/doc/dependencies.md                        @fanquake
-/doc/developer-notes.md                     @laanwj
-/doc/files.md                               @hebasto
-/doc/reduce-memory.md                       @fanquake
-/doc/reduce-traffic.md                      @jonasschnelli
-/doc/release-process.md                     @laanwj
-/doc/translation_strings_policy.md          @laanwj
-
-# Build aux
-/build-aux/m4/bitcoin_qt.m4                 @hebasto
-
-# MSVC build system
-/build_msvc/                                @sipsorcery
-
-# Settings
-/src/util/settings.*                        @ryanofsky
-
-# Fuzzing
-
-# Tests
-/src/test/net_peer_eviction_tests.cpp       @jonatack
-/test/functional/mempool_updatefromblock.py @hebasto
-/test/functional/feature_asmap.py           @jonatack
-/test/functional/interface_bitcoin_cli.py   @jonatack
-
-# Backwards compatibility tests
-*_compatibility.py                          @sjors
-/test/functional/wallet_upgradewallet.py    @sjors @achow101
-/test/get_previous_releases.py              @sjors
-
-# Translations
-/src/util/translation.h                     @hebasto
-
-# Dev Tools
-/contrib/devtools/security-check.py         @fanquake
-/contrib/devtools/test-security-check.py    @fanquake
-/contrib/devtools/symbol-check.py           @fanquake
-
-# Guix
-/contrib/guix/                              @dongcarl
-
-# Compatibility
-/src/compat/glibc_*                         @fanquake
-
-# GUI
-/src/qt/forms/                              @hebasto
-
-# Wallet
-/src/wallet/                                @achow101
-
-# CLI
-/src/bitcoin-cli.cpp                        @jonatack
-
-# Coinstats
-/src/node/coinstats.*                       @fjahr
-
-# Index
-/src/index/                                 @fjahr
-
-# Descriptors
-*descriptor*                                @achow101 @sipa
-
-# External signer
-*external_signer*                           @sjors
-/doc/external-signer.md                     @sjors
-*signer.py                                  @sjors
-
-# Interfaces
-/src/interfaces/                            @ryanofsky
-
-# DB
-/src/txdb.*                                 @jamesob
-/src/dbwrapper.*                            @jamesob
-
-# Linter
-/test/lint/lint-shell.sh                    @hebasto
-
-# Bech32
-/src/bech32.*                               @sipa
-/src/bench/bech32.*                         @sipa
-
-# PSBT
-/src/psbt*                                  @achow101
-/src/node/psbt*                             @achow101
-/doc/psbt.md                                @achow101
-
-# P2P
-/src/net_processing.*                       @sipa
-/src/protocol.*                             @sipa
-
-# Consensus
-/src/coins.*                                @sipa @jamesob
-/src/script/script.*                        @sipa
-/src/script/interpreter.*                   @sipa
-/src/validation.*                           @sipa
-/src/consensus/                             @sipa
-
-# Tracing
-/doc/tracing.md                             @jb55 @0xB10C
-/src/util/trace.h                           @jb55 @0xB10C
-/contrib/tracing/                           @jb55 @0xB10C

SECURITY.md

@@ -13,8 +13,8 @@ The following keys may be used to communicate sensitive information to developer
 
 | Name | Fingerprint |
 |------|-------------|
-| Wladimir van der Laan | 71A3 B167 3540 5025 D447  E8F2 7481 0B01 2346 C9A6 |
 | Pieter Wuille | 133E AC17 9436 F14A 5CF1  B794 860F EB80 4E66 9320 |
 | Michael Ford | E777 299F C265 DD04 7930  70EB 944D 35F9 AC3D B76A |
+| Andrew Chow | 1528 1230 0785 C964 44D3  334D 1756 5732 E08E 5E41 |
 
 You can import a key by running the following command with that individual’s fingerprint: `gpg --keyserver hkps://keys.openpgp.org --recv-keys "<fingerprint>"` Ensure that you put quotes around fingerprints containing spaces.

build-aux/m4/ax_boost_base.m4

@@ -11,9 +11,9 @@
 #   Test for the Boost C++ libraries of a particular version (or newer)
 #
 #   If no path to the installed boost library is given the macro searchs
-#   under /usr, /usr/local, /opt, /opt/local and /opt/homebrew and evaluates the
-#   $BOOST_ROOT environment variable. Further documentation is available at
-#   <http://randspringer.de/boost/index.html>.
+#   under /usr, /usr/local, /opt, /opt/local and /opt/homebrew and evaluates
+#   the $BOOST_ROOT environment variable. Further documentation is available
+#   at <http://randspringer.de/boost/index.html>.
 #
 #   This macro calls:
 #
@@ -33,7 +33,7 @@
 #   and this notice are preserved. This file is offered as-is, without any
 #   warranty.
 
-#serial 48
+#serial 51
 
 # example boost program (need to pass version)
 m4_define([_AX_BOOST_BASE_PROGRAM],
@@ -114,7 +114,7 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[
     AS_CASE([${host_cpu}],
       [x86_64],[libsubdirs="lib64 libx32 lib lib64"],
       [mips*64*],[libsubdirs="lib64 lib32 lib lib64"],
-      [ppc64|powerpc64|s390x|sparc64|aarch64|ppc64le|powerpc64le|riscv64],[libsubdirs="lib64 lib lib64"],
+      [ppc64|powerpc64|s390x|sparc64|aarch64|ppc64le|powerpc64le|riscv64|e2k],[libsubdirs="lib64 lib lib64"],
       [libsubdirs="lib"]
     )
 
@@ -128,7 +128,7 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[
     )
 
     dnl first we check the system location for boost libraries
-    dnl this location ist chosen if boost libraries are installed with the --layout=system option
+    dnl this location is chosen if boost libraries are installed with the --layout=system option
     dnl or if you install boost with RPM
     AS_IF([test "x$_AX_BOOST_BASE_boost_path" != "x"],[
         AC_MSG_CHECKING([for boostlib >= $1 ($WANT_BOOST_VERSION) includes in "$_AX_BOOST_BASE_boost_path/include"])
@@ -151,7 +151,7 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[
         else
             search_libsubdirs="$multiarch_libsubdir $libsubdirs"
         fi
-        for _AX_BOOST_BASE_boost_path_tmp in /usr /usr/local /opt /opt/local /opt/homebrew/; do
+        for _AX_BOOST_BASE_boost_path_tmp in /usr /usr/local /opt /opt/local /opt/homebrew ; do
             if test -d "$_AX_BOOST_BASE_boost_path_tmp/include/boost" && test -r "$_AX_BOOST_BASE_boost_path_tmp/include/boost" ; then
                 for libsubdir in $search_libsubdirs ; do
                     if ls "$_AX_BOOST_BASE_boost_path_tmp/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi

build-aux/m4/ax_cxx_compile_stdcxx.m4

@@ -10,13 +10,13 @@
 #
 #   Check for baseline language coverage in the compiler for the specified
 #   version of the C++ standard.  If necessary, add switches to CXX and
-#   CXXCPP to enable support.  VERSION may be '11' (for the C++11 standard)
-#   or '14' (for the C++14 standard).
+#   CXXCPP to enable support.  VERSION may be '11', '14', '17', or '20' for
+#   the respective C++ standard version.
 #
 #   The second argument, if specified, indicates whether you insist on an
 #   extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g.
 #   -std=c++11).  If neither is specified, you get whatever works, with
-#   preference for an extended mode.
+#   preference for no added switch, and then for an extended mode.
 #
 #   The third argument, if specified 'mandatory' or if left unspecified,
 #   indicates that baseline support for the specified C++ standard is
@@ -35,13 +35,15 @@
 #   Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu>
 #   Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com>
 #   Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com>
+#   Copyright (c) 2020 Jason Merrill <jason@redhat.com>
+#   Copyright (c) 2021 Jörn Heusipp <osmanx@problemloesungsmaschine.de>
 #
 #   Copying and distribution of this file, with or without modification, are
 #   permitted in any medium without royalty provided the copyright notice
 #   and this notice are preserved.  This file is offered as-is, without any
 #   warranty.
 
-#serial 11
+#serial 14
 
 dnl  This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro
 dnl  (serial version number 13).
@@ -50,6 +52,7 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl
   m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"],
         [$1], [14], [ax_cxx_compile_alternatives="14 1y"],
         [$1], [17], [ax_cxx_compile_alternatives="17 1z"],
+        [$1], [20], [ax_cxx_compile_alternatives="20"],
         [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl
   m4_if([$2], [], [],
         [$2], [ext], [],
@@ -62,6 +65,16 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl
   AC_LANG_PUSH([C++])dnl
   ac_success=no
 
+  m4_if([$2], [], [dnl
+    AC_CACHE_CHECK(whether $CXX supports C++$1 features by default,
+		   ax_cv_cxx_compile_cxx$1,
+      [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
+        [ax_cv_cxx_compile_cxx$1=yes],
+        [ax_cv_cxx_compile_cxx$1=no])])
+    if test x$ax_cv_cxx_compile_cxx$1 = xyes; then
+      ac_success=yes
+    fi])
+
   m4_if([$2], [noext], [], [dnl
   if test x$ac_success = xno; then
     for alternative in ${ax_cxx_compile_alternatives}; do
@@ -140,7 +153,6 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11],
   _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
 )
 
-
 dnl  Test body for checking C++14 support
 
 m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14],
@@ -148,12 +160,24 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14],
   _AX_CXX_COMPILE_STDCXX_testbody_new_in_14
 )
 
+dnl  Test body for checking C++17 support
+
 m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17],
   _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
   _AX_CXX_COMPILE_STDCXX_testbody_new_in_14
   _AX_CXX_COMPILE_STDCXX_testbody_new_in_17
 )
 
+dnl  Test body for checking C++20 support
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_20],
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_14
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_17
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_20
+)
+
+
 dnl  Tests for new features in C++11
 
 m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[
@@ -949,3 +973,33 @@ namespace cxx17
 #endif  // __cplusplus < 201703L
 
 ]])
+
+
+dnl  Tests for new features in C++20
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_20], [[
+
+#ifndef __cplusplus
+
+#error "This is not a C++ compiler"
+
+#elif __cplusplus < 202002L
+
+#error "This is not a C++20 compiler"
+
+#else
+
+#include <version>
+
+namespace cxx20
+{
+
+// As C++20 supports feature test macros in the standard, there is no
+// immediate need to actually test for feature availability on the
+// Autoconf side.
+
+}  // namespace cxx20
+
+#endif  // __cplusplus < 202002L
+
+]])

build-aux/m4/bitcoin_qt.m4

@@ -116,8 +116,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[
   BITCOIN_QT_CHECK([
   TEMP_CPPFLAGS=$CPPFLAGS
   TEMP_CXXFLAGS=$CXXFLAGS
-  CPPFLAGS="$QT_INCLUDES $CPPFLAGS"
-  CXXFLAGS="$PIC_FLAGS $CXXFLAGS"
+  CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS"
+  CXXFLAGS="$PIC_FLAGS $CORE_CXXFLAGS $CXXFLAGS"
   _BITCOIN_QT_IS_STATIC
   if test "$bitcoin_cv_static_qt" = "yes"; then
     _BITCOIN_QT_CHECK_STATIC_LIBS
@@ -178,8 +178,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[
     AC_MSG_CHECKING([whether -fPIE can be used with this Qt config])
     TEMP_CPPFLAGS=$CPPFLAGS
     TEMP_CXXFLAGS=$CXXFLAGS
-    CPPFLAGS="$QT_INCLUDES $CPPFLAGS"
-    CXXFLAGS="$PIE_FLAGS $CXXFLAGS"
+    CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS"
+    CXXFLAGS="$PIE_FLAGS $CORE_CXXFLAGS $CXXFLAGS"
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
         #include <QtCore/qconfig.h>
         #ifndef QT_VERSION
@@ -201,7 +201,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[
     BITCOIN_QT_CHECK([
     AC_MSG_CHECKING([whether -fPIC is needed with this Qt config])
     TEMP_CPPFLAGS=$CPPFLAGS
-    CPPFLAGS="$QT_INCLUDES $CPPFLAGS"
+    CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS"
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
         #include <QtCore/qconfig.h>
         #ifndef QT_VERSION

build-aux/m4/l_atomic.m4

@@ -18,7 +18,7 @@ m4_define([_CHECK_ATOMIC_testbody], [[
 
   int main() {
     std::atomic<bool> lock{true};
-    std::atomic_exchange(&lock, false);
+    lock.exchange(false);
 
     std::atomic<std::chrono::seconds> t{0s};
     t.store(2s);
@@ -34,6 +34,8 @@ m4_define([_CHECK_ATOMIC_testbody], [[
 AC_DEFUN([CHECK_ATOMIC], [
 
   AC_LANG_PUSH(C++)
+  TEMP_CXXFLAGS="$CXXFLAGS"
+  CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS"
 
   AC_MSG_CHECKING([whether std::atomic can be used without link library])
 
@@ -51,5 +53,6 @@ AC_DEFUN([CHECK_ATOMIC], [
         ])
     ])
 
+  CXXFLAGS="$TEMP_CXXFLAGS"
   AC_LANG_POP
 ])

build_msvc/.gitignore

@@ -22,6 +22,7 @@ bench_bitcoin/bench_bitcoin.vcxproj
 libtest_util/libtest_util.vcxproj
 
 /bitcoin_config.h
+/common.init.vcxproj
 
 */Win32
 libbitcoin_qt/QtGeneratedFiles/*

build_msvc/README.md

@@ -3,7 +3,9 @@ Building Bitcoin Core with Visual Studio
 
 Introduction
 ---------------------
-Solution and project files to build Bitcoin Core with `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2019 (building with earlier versions of Visual Studio should not be expected to work).
+Visual Studio 2022 is minimum required to build Bitcoin Core.
+
+Solution and project files to build with `msbuild` or Visual Studio can be found in the `build_msvc` directory.
 
 To build Bitcoin Core from the command-line, it is sufficient to only install the Visual Studio Build Tools component.
 
@@ -28,9 +30,9 @@ Qt
 ---------------------
 To build Bitcoin Core with the GUI, a static build of Qt is required.
 
-1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.3.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.3/single/qt-everywhere-opensource-src-5.15.3.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`.
+1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.5.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`.
 
-2. Open "x64 Native Tools Command Prompt for VS 2019", and input the following commands:
+2. Open "x64 Native Tools Command Prompt for VS 2022", and input the following commands:
 ```cmd
 cd C:\dev\qt-source
 mkdir build
@@ -47,21 +49,21 @@ To build Bitcoin Core without Qt, unload or disable the `bitcoin-qt`, `libbitcoi
 
 Building
 ---------------------
-1. Use Python to generate `*.vcxproj` from Makefile:
+1. Use Python to generate `*.vcxproj` for the Visual Studio 2022 toolchain from Makefile:
 
-```
-PS >py -3 msvc-autogen.py
+```cmd
+python build_msvc\msvc-autogen.py
 ```
 
 2. An optional step is to adjust the settings in the `build_msvc` directory and the `common.init.vcxproj` file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. To specify a non-default path to a static Qt package directory, use the `QTBASEDIR` environment variable.
 
-3. To build from the command-line with the Visual Studio 2019 toolchain use:
+3. To build from the command-line with the Visual Studio toolchain use:
 
 ```cmd
-msbuild -property:Configuration=Release -maxCpuCount -verbosity:minimal bitcoin.sln
+msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal
 ```
 
-Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio 2019.
+Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio.
 
 Security
 ---------------------

build_msvc/bitcoin_config.h.in

@@ -50,8 +50,8 @@
 /* Define this symbol if the consensus lib has been built */
 #define HAVE_CONSENSUS_LIB 1
 
-/* define if the compiler supports basic C++17 syntax */
-#define HAVE_CXX17 1
+/* define if the compiler supports basic C++20 syntax */
+#define HAVE_CXX20 1
 
 /* Define to 1 if you have the declaration of `be16toh', and to 0 if you
    don't. */
@@ -125,10 +125,6 @@
    don't. */
 #define HAVE_DECL_STRERROR_R 0
 
-/* Define to 1 if you have the declaration of `strnlen', and to 0 if you
-   don't. */
-#define HAVE_DECL_STRNLEN 1
-
 /* Define if the dllexport attribute is supported. */
 #define HAVE_DLLEXPORT_ATTRIBUTE 1
 

build_msvc/common.init.vcxproj.in

@@ -39,7 +39,7 @@
   <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
     <LinkIncremental>false</LinkIncremental>
     <UseDebugLibraries>false</UseDebugLibraries>
-    <PlatformToolset>v142</PlatformToolset>
+    <PlatformToolset>@TOOLSET@</PlatformToolset>
     <CharacterSet>Unicode</CharacterSet>
     <GenerateManifest>No</GenerateManifest>
     <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir>
@@ -49,7 +49,7 @@
   <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
     <LinkIncremental>true</LinkIncremental>
     <UseDebugLibraries>true</UseDebugLibraries>
-    <PlatformToolset>v142</PlatformToolset>
+    <PlatformToolset>@TOOLSET@</PlatformToolset>
     <CharacterSet>Unicode</CharacterSet>
     <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir>
     <IntDir>$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
@@ -87,10 +87,10 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <PrecompiledHeader>NotUsing</PrecompiledHeader>
-      <AdditionalOptions>/utf-8 /Zc:__cplusplus /std:c++17 %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalOptions>/utf-8 /Zc:__cplusplus /std:c++20 %(AdditionalOptions)</AdditionalOptions>
       <DisableSpecificWarnings>4018;4244;4267;4334;4715;4805;4834</DisableSpecificWarnings>
       <TreatWarningAsError>true</TreatWarningAsError>
-      <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <AdditionalIncludeDirectories>..\..\src;..\..\src\minisketch\include;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>

build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj

@@ -53,6 +53,7 @@
     <ClCompile Include="..\..\src\qt\transactiondesc.cpp" />
     <ClCompile Include="..\..\src\qt\transactiondescdialog.cpp" />
     <ClCompile Include="..\..\src\qt\transactionfilterproxy.cpp" />
+    <ClCompile Include="..\..\src\qt\transactionoverviewwidget.cpp" />
     <ClCompile Include="..\..\src\qt\transactionrecord.cpp" />
     <ClCompile Include="..\..\src\qt\transactiontablemodel.cpp" />
     <ClCompile Include="..\..\src\qt\transactionview.cpp" />

build_msvc/libsecp256k1/libsecp256k1.vcxproj

@@ -8,6 +8,8 @@
     <ConfigurationType>StaticLibrary</ConfigurationType>
   </PropertyGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\src\secp256k1\src\precomputed_ecmult.c" />
+    <ClCompile Include="..\..\src\secp256k1\src\precomputed_ecmult_gen.c" />
     <ClCompile Include="..\..\src\secp256k1\src\secp256k1.c" />
   </ItemGroup>
   <ItemDefinitionGroup>

build_msvc/libsecp256k1_config.h

@@ -8,23 +8,6 @@
 #define BITCOIN_LIBSECP256K1_CONFIG_H
 
 #undef USE_ASM_X86_64
-#undef USE_ENDOMORPHISM
-#undef USE_FIELD_10X26
-#undef USE_FIELD_5X52
-#undef USE_FIELD_INV_BUILTIN
-#undef USE_FIELD_INV_NUM
-#undef USE_NUM_GMP
-#undef USE_NUM_NONE
-#undef USE_SCALAR_4X64
-#undef USE_SCALAR_8X32
-#undef USE_SCALAR_INV_BUILTIN
-#undef USE_SCALAR_INV_NUM
-
-#define USE_NUM_NONE 1
-#define USE_FIELD_INV_BUILTIN 1
-#define USE_SCALAR_INV_BUILTIN 1
-#define USE_FIELD_10X26 1
-#define USE_SCALAR_8X32 1
 
 #define ECMULT_GEN_PREC_BITS 4
 #define ECMULT_WINDOW_SIZE 15

build_msvc/msvc-autogen.py

@@ -9,7 +9,7 @@ import argparse
 from shutil import copyfile
 
 SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'src'))
-DEFAULT_PLATFORM_TOOLSET = R'v142'
+DEFAULT_PLATFORM_TOOLSET = R'v143'
 
 libs = [
     'libbitcoin_cli',
@@ -50,13 +50,6 @@ def parse_makefile(makefile):
                     lib_sources[current_lib] = []
                     break
 
-def set_common_properties(toolset):
-    with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'r', encoding='utf-8') as rfile:
-        s = rfile.read()
-        s = re.sub('<PlatformToolset>.*?</PlatformToolset>', '<PlatformToolset>'+toolset+'</PlatformToolset>', s)
-    with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'w', encoding='utf-8',newline='\n') as wfile:
-        wfile.write(s)
-
 def parse_config_into_btc_config():
     def find_between( s, first, last ):
         try:
@@ -92,13 +85,18 @@ def parse_config_into_btc_config():
     with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), "w", encoding="utf8") as btc_config:
         btc_config.writelines(template)
 
+def set_properties(vcxproj_filename, placeholder, content):
+    with open(vcxproj_filename + '.in', 'r', encoding='utf-8') as vcxproj_in_file:
+        with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file:
+            vcxproj_file.write(vcxproj_in_file.read().replace(placeholder, content))
+
 def main():
     parser = argparse.ArgumentParser(description='Bitcoin-core msbuild configuration initialiser.')
-    parser.add_argument('-toolset', nargs='?',help='Optionally sets the msbuild platform toolset, e.g. v142 for Visual Studio 2019.'
+    parser.add_argument('-toolset', nargs='?', default=DEFAULT_PLATFORM_TOOLSET,
+        help='Optionally sets the msbuild platform toolset, e.g. v143 for Visual Studio 2022.'
          ' default is %s.'%DEFAULT_PLATFORM_TOOLSET)
     args = parser.parse_args()
-    if args.toolset:
-        set_common_properties(args.toolset)
+    set_properties(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), '@TOOLSET@', args.toolset)
 
     for makefile_name in os.listdir(SOURCE_DIR):
         if 'Makefile' in makefile_name:
@@ -110,10 +108,7 @@ def main():
             content += '    <ClCompile Include="..\\..\\src\\' + source_filename + '">\n'
             content += '      <ObjectFileName>$(IntDir)' + object_filename + '</ObjectFileName>\n'
             content += '    </ClCompile>\n'
-        with open(vcxproj_filename + '.in', 'r', encoding='utf-8') as vcxproj_in_file:
-            with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file:
-                vcxproj_file.write(vcxproj_in_file.read().replace(
-                    '@SOURCE_FILES@\n', content))
+        set_properties(vcxproj_filename, '@SOURCE_FILES@\n', content)
     parse_config_into_btc_config()
     copyfile(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), os.path.join(SOURCE_DIR, 'config/bitcoin-config.h'))
     copyfile(os.path.join(SOURCE_DIR,'../build_msvc/libsecp256k1_config.h'), os.path.join(SOURCE_DIR, 'secp256k1/src/libsecp256k1-config.h'))

build_msvc/vcpkg.json

@@ -8,7 +8,6 @@
     "boost-signals2",
     "boost-test",
     "sqlite3",
-    "double-conversion",
     {
       "name": "libevent",
       "features": ["thread"]

ci/lint/04_install.sh

@@ -11,9 +11,9 @@ ${CI_RETRY_EXE} apt-get install -y clang-format-9 python3-pip curl git gawk jq
 update-alternatives --install /usr/bin/clang-format      clang-format      "$(which clang-format-9     )" 100
 update-alternatives --install /usr/bin/clang-format-diff clang-format-diff "$(which clang-format-diff-9)" 100
 
-${CI_RETRY_EXE} pip3 install codespell==2.0.0
-${CI_RETRY_EXE} pip3 install flake8==3.8.3
-${CI_RETRY_EXE} pip3 install mypy==0.910
+${CI_RETRY_EXE} pip3 install codespell==2.1.0
+${CI_RETRY_EXE} pip3 install flake8==4.0.1
+${CI_RETRY_EXE} pip3 install mypy==0.942
 ${CI_RETRY_EXE} pip3 install pyzmq==22.3.0
 ${CI_RETRY_EXE} pip3 install vulture==2.3
 

ci/lint/06_script.sh

@@ -9,7 +9,12 @@ export LC_ALL=C
 GIT_HEAD=$(git rev-parse HEAD)
 if [ -n "$CIRRUS_PR" ]; then
   COMMIT_RANGE="${CIRRUS_BASE_SHA}..$GIT_HEAD"
+  echo
+  git log --no-merges --oneline "$COMMIT_RANGE"
+  echo
   test/lint/commit-script-check.sh "$COMMIT_RANGE"
+else
+  COMMIT_RANGE="SKIP_EMPTY_NOT_A_PR"
 fi
 export COMMIT_RANGE
 
@@ -18,11 +23,10 @@ export COMMIT_RANGE
 test/lint/git-subtree-check.sh src/crypto/ctaes
 test/lint/git-subtree-check.sh src/secp256k1
 test/lint/git-subtree-check.sh src/minisketch
-test/lint/git-subtree-check.sh src/univalue
 test/lint/git-subtree-check.sh src/leveldb
 test/lint/git-subtree-check.sh src/crc32c
 test/lint/check-doc.py
-test/lint/lint-all.sh
+test/lint/all-lint.py
 
 if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; then
     # Sanity check only the last few commits to get notified of missing sigs,
@@ -32,11 +36,8 @@ if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; t
     git log HEAD~10 -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit
     git log HEAD~10 -1 --format='%H' > ./contrib/verify-commits/trusted-git-root
     mapfile -t KEYS < contrib/verify-commits/trusted-keys
+    git config user.email "ci@ci.ci"
+    git config user.name "ci"
     ${CI_RETRY_EXE} gpg --keyserver hkps://keys.openpgp.org --recv-keys "${KEYS[@]}" &&
     ./contrib/verify-commits/verify-commits.py;
 fi
-
-if [ -n "$COMMIT_RANGE" ]; then
-  echo
-  git log --no-merges --oneline "$COMMIT_RANGE"
-fi

ci/test/00_setup_env.sh

@@ -37,6 +37,7 @@ export USE_BUSY_BOX=${USE_BUSY_BOX:-false}
 
 export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
 export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true}
+export RUN_TIDY=${RUN_TIDY:-false}
 export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false}
 # By how much to scale the test_runner timeouts (option --timeout-factor).
 # This is needed because some ci machines have slow CPU or disk, so sanitizers

ci/test/00_setup_env_android.sh

@@ -7,7 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export HOST=aarch64-linux-android
-export PACKAGES="clang llvm unzip openjdk-8-jdk gradle"
+export PACKAGES="unzip openjdk-8-jdk gradle"
 export CONTAINER_NAME=ci_android
 export DOCKER_NAME_TAG="ubuntu:focal"
 
@@ -16,8 +16,8 @@ export RUN_FUNCTIONAL_TESTS=false
 
 export ANDROID_API_LEVEL=28
 export ANDROID_BUILD_TOOLS_VERSION=28.0.3
-export ANDROID_NDK_VERSION=23.1.7779620
-export ANDROID_TOOLS_URL=https://dl.google.com/android/repository/commandlinetools-linux-6609375_latest.zip
+export ANDROID_NDK_VERSION=23.2.8568313
+export ANDROID_TOOLS_URL=https://dl.google.com/android/repository/commandlinetools-linux-8512546_latest.zip
 export ANDROID_HOME="${DEPENDS_DIR}/SDKs/android"
 export ANDROID_NDK_HOME="${ANDROID_HOME}/ndk/${ANDROID_NDK_VERSION}"
 export DEP_OPTS="ANDROID_SDK=${ANDROID_HOME} ANDROID_NDK=${ANDROID_NDK_HOME} ANDROID_API_LEVEL=${ANDROID_API_LEVEL} ANDROID_TOOLCHAIN_BIN=${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/"

ci/test/00_setup_env_i686_centos.sh

@@ -9,7 +9,8 @@ export LC_ALL=C.UTF-8
 export HOST=i686-pc-linux-gnu
 export CONTAINER_NAME=ci_i686_centos
 export DOCKER_NAME_TAG=quay.io/centos/centos:stream8
-export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-zmq which patch lbzip2 xz procps-ng dash rsync coreutils bison"
+export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison"
+export PIP_PACKAGES="pyzmq"
 export GOAL="install"
 export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports"
 export CONFIG_SHELL="/bin/dash"

ci/test/00_setup_env_i686_multiprocess.sh

@@ -9,10 +9,9 @@ export LC_ALL=C.UTF-8
 export HOST=i686-pc-linux-gnu
 export CONTAINER_NAME=ci_i686_multiprocess
 export DOCKER_NAME_TAG=ubuntu:20.04
-export PACKAGES="cmake python3 python3-pip llvm clang g++-multilib"
+export PACKAGES="cmake python3 llvm clang g++-multilib"
 export DEP_OPTS="DEBUG=1 MULTIPROCESS=1"
 export GOAL="install"
 export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' LDFLAGS='--rtlib=compiler-rt -lgcc_s'"
 export TEST_RUNNER_ENV="BITCOIND=bitcoin-node"
 export TEST_RUNNER_EXTRA="--nosandbox"
-export PIP_PACKAGES="lief"

ci/test/00_setup_env_mac_native_arm64.sh

@@ -6,12 +6,11 @@
 
 export LC_ALL=C.UTF-8
 
-export HOST=x86_64-apple-darwin
-export PIP_PACKAGES="zmq lief"
+export HOST=arm64-apple-darwin
+export PIP_PACKAGES="zmq"
 export GOAL="install"
-export BITCOIN_CONFIG="--with-gui --enable-reduce-exports"
+export BITCOIN_CONFIG="--with-gui --with-miniupnpc --with-natpmp --enable-reduce-exports"
 export CI_OS_NAME="macos"
 export NO_DEPENDS=1
 export OSX_SDK=""
 export CCACHE_SIZE=300M
-export RUN_SECURITY_TESTS="true"

ci/test/00_setup_env_native_asan.sh

@@ -6,9 +6,14 @@
 
 export LC_ALL=C.UTF-8
 
+# We install an up-to-date 'bpfcc-tools' package from an untrusted PPA.
+# This can be dropped with the next Ubuntu or Debian release that includes up-to-date packages.
+# See the if-then in ci/test/04_install.sh too.
+export ADD_UNTRUSTED_BPFCC_PPA=true
+
 export CONTAINER_NAME=ci_native_asan
-export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev"
-export DOCKER_NAME_TAG=ubuntu:22.04
+export PACKAGES="systemtap-sdt-dev bpfcc-tools clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev"
+export DOCKER_NAME_TAG=ubuntu:22.04  # May not run in docker unless --enable-usdt is dropped
 export NO_DEPENDS=1
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"
+export BITCOIN_CONFIG="--enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"

ci/test/00_setup_env_native_fuzz_with_msan.sh

@@ -13,10 +13,11 @@ LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_
 export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
 
 export CONTAINER_NAME="ci_native_msan"
-export PACKAGES="clang-9 llvm-9 cmake"
+export PACKAGES="clang-12 llvm-12 cmake"
+# BDB generates false-positives and will be removed in future
 export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'"
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export USE_MEMORY_SANITIZER="true"
 export RUN_UNIT_TESTS="false"
 export RUN_FUNCTIONAL_TESTS="false"

ci/test/00_setup_env_native_fuzz_with_valgrind.sh

@@ -15,5 +15,6 @@ export RUN_FUNCTIONAL_TESTS=false
 export RUN_FUZZ_TESTS=true
 export FUZZ_TESTS_CONFIG="--valgrind"
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++"
+# Temporarily pin dwarf 4, until valgrind can understand clang's dwarf 5
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'"
 export CCACHE_SIZE=200M

ci/test/00_setup_env_native_msan.sh

@@ -11,13 +11,13 @@ LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
 LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument"
 export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
-export BDB_PREFIX="${BASE_ROOT_DIR}/db4"
 
 export CONTAINER_NAME="ci_native_msan"
-export PACKAGES="clang-9 llvm-9 cmake"
+export PACKAGES="clang-12 llvm-12 cmake"
+# BDB generates false-positives and will be removed in future
 export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'"
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'"
+export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export USE_MEMORY_SANITIZER="true"
 export RUN_FUNCTIONAL_TESTS="false"
 export CCACHE_SIZE=250M

ci/test/00_setup_env_native_nowallet.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019-2021 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-export CONTAINER_NAME=ci_native_nowallet
-export DOCKER_NAME_TAG=ubuntu:18.04  # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md
-export PACKAGES="python3-zmq clang-7 llvm-7 libc++abi-7-dev libc++-7-dev"  # Use clang-7 to test C++17 compatibility, see doc/dependencies.md
-export DEP_OPTS="NO_WALLET=1 CC=clang-7 CXX='clang++-7 -stdlib=libc++'"
-export GOAL="install"
-export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-7 CXX='clang++-7 -stdlib=libc++'"

ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2021 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel
+export DOCKER_NAME_TAG=ubuntu:18.04  # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md
+export PACKAGES="python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev"  # Use clang-8 to test C++17 compatibility, see doc/dependencies.md
+export DEP_OPTS="NO_WALLET=1 CC=clang-8 CXX='clang++-8 -stdlib=libc++'"
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared"

ci/test/00_setup_env_native_qt5.sh

@@ -7,13 +7,13 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_qt5
-export DOCKER_NAME_TAG=ubuntu:18.04  # Check that bionic gcc-8 can compile our C++17 and run our functional tests in python3, see doc/dependencies.md
+export DOCKER_NAME_TAG=debian:buster  # Check that buster gcc-8 can compile our C++17 and run our functional tests in python3, see doc/dependencies.md
 export PACKAGES="gcc-8 g++-8 python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev"
 export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-8 CXX=g++-8"
 export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash"  # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash
 export RUN_UNIT_TESTS_SEQUENTIAL="true"
 export RUN_UNIT_TESTS="false"
 export GOAL="install"
-export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1 v0.20.1 v0.21.0 v22.0"
+export DOWNLOAD_PREVIOUS_RELEASES="true"
 export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports \
---enable-debug --disable-fuzz-binary  CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8"
+--enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8"

ci/test/00_setup_env_native_tidy.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export DOCKER_NAME_TAG="ubuntu:22.04"
+export CONTAINER_NAME=ci_native_tidy
+export PACKAGES="clang libclang-dev llvm-dev clang-tidy bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev"
+export NO_DEPENDS=1
+export RUN_UNIT_TESTS=false
+export RUN_FUNCTIONAL_TESTS=false
+export RUN_FUZZ_TESTS=false
+export RUN_TIDY=true
+export GOAL="install"
+export BITCOIN_CONFIG="CC=clang CXX=clang++ --enable-c++20 --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0'"
+export CCACHE_SIZE=200M

ci/test/00_setup_env_native_tsan.sh

@@ -11,4 +11,4 @@ export DOCKER_NAME_TAG=ubuntu:22.04
 export PACKAGES="clang-13 llvm-13 libc++abi-13-dev libc++-13-dev python3-zmq"
 export DEP_OPTS="CC=clang-13 CXX='clang++-13 -stdlib=libc++'"
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang-13 CXX='clang++-13 -stdlib=libc++'"
+export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' CXXFLAGS='-g' --with-sanitizers=thread CC=clang-13 CXX='clang++-13 -stdlib=libc++'"

ci/test/00_setup_env_native_valgrind.sh

@@ -13,4 +13,5 @@ export USE_VALGRIND=1
 export NO_DEPENDS=1
 export TEST_RUNNER_EXTRA="--nosandbox --exclude feature_init,rpc_bind,feature_bind_extra"  # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++"  # TODO enable GUI
+# Temporarily pin dwarf 4, until valgrind can understand clang's dwarf 5
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'"  # TODO enable GUI

ci/test/00_setup_env_win64.sh

@@ -10,7 +10,7 @@ export CONTAINER_NAME=ci_win64
 export DOCKER_NAME_TAG=ubuntu:22.04  # Check that Jammy can cross-compile to win64
 export HOST=x86_64-w64-mingw32
 export DPKG_ADD_ARCH="i386"
-export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 wine32 file"
+export PACKAGES="python3 nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file"
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="deploy"
 export BITCOIN_CONFIG="--enable-reduce-exports --disable-external-signer --disable-gui-tests"

ci/test/04_install.sh

@@ -10,12 +10,6 @@ if [[ $QEMU_USER_CMD == qemu-s390* ]]; then
   export LC_ALL=C
 fi
 
-if [ "$CI_OS_NAME" == "macos" ]; then
-  sudo -H pip3 install --upgrade pip
-  # shellcheck disable=SC2086
-  IN_GETOPT_BIN="/usr/local/opt/gnu-getopt/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
-fi
-
 # Create folders that are mounted into the docker
 mkdir -p "${CCACHE_DIR}"
 mkdir -p "${PREVIOUS_RELEASES_DIR}"
@@ -68,11 +62,26 @@ if [[ $DOCKER_NAME_TAG == *centos* ]]; then
   ${CI_RETRY_EXE} CI_EXEC dnf -y install epel-release
   ${CI_RETRY_EXE} CI_EXEC dnf -y --allowerasing install "$DOCKER_PACKAGES" "$PACKAGES"
 elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
+  if [[ "${ADD_UNTRUSTED_BPFCC_PPA}" == "true" ]]; then
+    # Ubuntu 22.04 LTS and Debian 11 both have an outdated bpfcc-tools packages.
+    # The iovisor PPA is outdated as well. The next Ubuntu and Debian releases will contain updated
+    # packages. Meanwhile, use an untrusted PPA to install an up-to-date version of the bpfcc-tools
+    # package.
+    # TODO: drop this once we can use newer images in GCE
+    CI_EXEC add-apt-repository ppa:hadret/bpfcc
+  fi
   ${CI_RETRY_EXE} CI_EXEC apt-get update
   ${CI_RETRY_EXE} CI_EXEC apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$DOCKER_PACKAGES"
-  if [ -n "$PIP_PACKAGES" ]; then
+fi
+
+if [ -n "$PIP_PACKAGES" ]; then
+  if [ "$CI_OS_NAME" == "macos" ]; then
+    sudo -H pip3 install --upgrade pip
     # shellcheck disable=SC2086
-    ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
+    IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
+  else
+    # shellcheck disable=SC2086
+    ${CI_RETRY_EXE} CI_EXEC pip3 install --user $PIP_PACKAGES
   fi
 fi
 
@@ -103,14 +112,24 @@ fi
 CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/"
 
 if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
-  CI_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-9) 100"
-  CI_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-9) 100"
+  CI_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100"
+  CI_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100"
   CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/"
   CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project"
-  CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/"
+  CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/"
   CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx"
 fi
 
+if [[ "${RUN_TIDY}" == "true" ]]; then
+  export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu"
+  if [ ! -d "${DIR_IWYU}" ]; then
+    CI_EXEC "mkdir -p ${DIR_IWYU}/build/"
+    CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_14 ${DIR_IWYU}/include-what-you-use"
+    CI_EXEC "cd ${DIR_IWYU}/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-14 ../include-what-you-use"
+    CI_EXEC "cd ${DIR_IWYU}/build && make install $MAKEJOBS"
+  fi
+fi
+
 if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
   echo "Create $BASE_ROOT_DIR"
   CI_EXEC rsync -a /ro_base/ "$BASE_ROOT_DIR"

ci/test/05_before_script.sh

@@ -31,16 +31,9 @@ if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then
   if [ ! -f "$ANDROID_TOOLS_PATH" ]; then
     CI_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH"
   fi
-  CI_EXEC mkdir -p "${ANDROID_HOME}/cmdline-tools"
-  CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "${ANDROID_HOME}/cmdline-tools"
-  CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/tools/bin/sdkmanager --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\""
-fi
-
-if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
-  # Use BDB compiled using install_db4.sh script to work around linking issue when using BDB
-  # from depends. See https://github.com/bitcoin/bitcoin/pull/18288#discussion_r433189350 for
-  # details.
-  CI_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+  CI_EXEC mkdir -p "$ANDROID_HOME"
+  CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME"
+  CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/bin/sdkmanager --sdk_root=\"${ANDROID_HOME}\" --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\""
 fi
 
 if [ -z "$NO_DEPENDS" ]; then
@@ -52,8 +45,8 @@ if [ -z "$NO_DEPENDS" ]; then
   else
     SHELL_OPTS="CONFIG_SHELL="
   fi
-  CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS"
+  CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" LOG=1
 fi
-if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then
-  CI_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}"
+if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then
+  CI_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR"
 fi

ci/test/06_script_a.sh

@@ -6,19 +6,24 @@
 
 export LC_ALL=C.UTF-8
 
-if [ -n "$ANDROID_TOOLS_URL" ]; then
-  CI_EXEC make distclean || true
-  CI_EXEC ./autogen.sh
-  CI_EXEC ./configure "$BITCOIN_CONFIG" --prefix="${DEPENDS_DIR}/aarch64-linux-android" || ( (CI_EXEC cat config.log) && false)
-  CI_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk"
-  exit 0
-fi
-
-BITCOIN_CONFIG_ALL="--enable-external-signer --enable-suppress-external-warnings --disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib"
+BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST"
 if [ -z "$NO_WERROR" ]; then
   BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror"
 fi
+
 CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE"
+PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats"
+
+if [ -n "$ANDROID_TOOLS_URL" ]; then
+  CI_EXEC make distclean || true
+  CI_EXEC ./autogen.sh
+  CI_EXEC ./configure "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false)
+  CI_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk"
+  CI_EXEC "${PRINT_CCACHE_STATISTICS}"
+  exit 0
+fi
+
+BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib"
 
 if [ -n "$CONFIG_SHELL" ]; then
   CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh"
@@ -48,8 +53,13 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
   CI_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h'
 fi
 
-CI_EXEC make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false )
+if [[ "${RUN_TIDY}" == "true" ]]; then
+  MAYBE_BEAR="bear --config src/.bear-tidy-config"
+  MAYBE_TOKEN="--"
+fi
 
-CI_EXEC "ccache --version | head -n 1 && ccache --show-stats"
+CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false )
+
+CI_EXEC "${PRINT_CCACHE_STATISTICS}"
 CI_EXEC du -sh "${DEPENDS_DIR}"/*/
 CI_EXEC du -sh "${PREVIOUS_RELEASES_DIR}"

ci/test/06_script_b.sh

@@ -27,13 +27,50 @@ if [ "$RUN_UNIT_TESTS" = "true" ]; then
 fi
 
 if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then
-  CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_BUILD_DIR}/bitcoin-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite
+  CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_OUTDIR}/bin/test_bitcoin" --catch_system_errors=no -l test_suite
 fi
 
 if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
   CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast
 fi
 
+if [ "${RUN_TIDY}" = "true" ]; then
+  set -eo pipefail
+  export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/src/"
+  ( CI_EXEC run-clang-tidy -quiet "${MAKEJOBS}" ) | grep -C5 "error"
+  export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/"
+  CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/iwyu_tool.py"\
+          " src/compat"\
+          " src/dbwrapper.cpp"\
+          " src/init"\
+          " src/kernel"\
+          " src/node/chainstate.cpp"\
+          " src/node/mempool_args.cpp"\
+          " src/node/validation_cache_args.cpp"\
+          " src/policy/feerate.cpp"\
+          " src/policy/packages.cpp"\
+          " src/policy/settings.cpp"\
+          " src/primitives/transaction.cpp"\
+          " src/rpc/fees.cpp"\
+          " src/rpc/signmessage.cpp"\
+          " src/test/fuzz/txorphan.cpp"\
+          " src/threadinterrupt.cpp"\
+          " src/util/bip32.cpp"\
+          " src/util/bytevectorhash.cpp"\
+          " src/util/error.cpp"\
+          " src/util/getuniquepath.cpp"\
+          " src/util/hasher.cpp"\
+          " src/util/message.cpp"\
+          " src/util/moneystr.cpp"\
+          " src/util/serfloat.cpp"\
+          " src/util/spanparsing.cpp"\
+          " src/util/strencodings.cpp"\
+          " src/util/string.cpp"\
+          " src/util/syserror.cpp"\
+          " src/util/url.cpp"\
+          " -p . ${MAKEJOBS} -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp"
+fi
+
 if [ "$RUN_SECURITY_TESTS" = "true" ]; then
   CI_EXEC make test-security-check
 fi

ci/test/wrap-qemu.sh

@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}; do
+for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{test_json,unitester,object}}; do
     # shellcheck disable=SC2044
     for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do
       echo "Wrap $b ..."

ci/test/wrap-wine.sh

@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}.exe; do
+for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{test_json,unitester,object}}.exe; do
     # shellcheck disable=SC2044
     for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do
       if (file "$b" | grep "Windows"); then

ci/test/wrapped-cl.bat

@@ -0,0 +1 @@
+ccache cl %*

configure.ac

@@ -1,5 +1,5 @@
 AC_PREREQ([2.69])
-define(_CLIENT_VERSION_MAJOR, 23)
+define(_CLIENT_VERSION_MAJOR, 24)
 define(_CLIENT_VERSION_MINOR, 1)
 define(_CLIENT_VERSION_BUILD, 0)
 define(_CLIENT_VERSION_RC, 0)
@@ -19,17 +19,30 @@ if test "$PKG_CONFIG" = ""; then
   AC_MSG_ERROR([pkg-config not found])
 fi
 
+# When compiling with depends, the `PKG_CONFIG_PATH` and `PKG_CONFIG_LIBDIR` variables,
+# being set in a `config.site` file, are not exported to let the `--config-cache` option
+# work properly.
+if test -n "$PKG_CONFIG_PATH"; then
+  PKG_CONFIG="env PKG_CONFIG_PATH=$PKG_CONFIG_PATH $PKG_CONFIG"
+fi
+if test -n "$PKG_CONFIG_LIBDIR"; then
+  PKG_CONFIG="env PKG_CONFIG_LIBDIR=$PKG_CONFIG_LIBDIR $PKG_CONFIG"
+fi
+
 BITCOIN_DAEMON_NAME=bitcoind
 BITCOIN_GUI_NAME=bitcoin-qt
+BITCOIN_TEST_NAME=test_bitcoin
 BITCOIN_CLI_NAME=bitcoin-cli
 BITCOIN_TX_NAME=bitcoin-tx
 BITCOIN_UTIL_NAME=bitcoin-util
+BITCOIN_CHAINSTATE_NAME=bitcoin-chainstate
 BITCOIN_WALLET_TOOL_NAME=bitcoin-wallet
 dnl Multi Process
 BITCOIN_MP_NODE_NAME=bitcoin-node
 BITCOIN_MP_GUI_NAME=bitcoin-gui
 
 dnl Unless the user specified ARFLAGS, force it to be cr
+dnl This is also the default as-of libtool 2.4.7
 AC_ARG_VAR([ARFLAGS], [Flags for the archiver, defaults to <cr> if not set])
 if test "${ARFLAGS+set}" != "set"; then
   ARFLAGS="cr"
@@ -41,14 +54,9 @@ AH_TOP([#ifndef BITCOIN_CONFIG_H])
 AH_TOP([#define BITCOIN_CONFIG_H])
 AH_BOTTOM([#endif //BITCOIN_CONFIG_H])
 
-dnl faketime breaks configure and is only needed for make. Disable it here.
-unset FAKETIME
-
 dnl Automake init set-up and checks
 AM_INIT_AUTOMAKE([1.13 no-define subdir-objects foreign])
 
-dnl faketime messes with timestamps and causes configure to be re-run.
-dnl --disable-maintainer-mode can be used to bypass this.
 AM_MAINTAINER_MODE([enable])
 
 dnl make the compilation flags quiet unless V=1 is used
@@ -77,11 +85,18 @@ AC_ARG_WITH([seccomp],
   [seccomp_found=$withval],
   [seccomp_found=auto])
 
-dnl Require C++17 compiler (no GNU extensions)
-AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory])
+AC_ARG_ENABLE([c++20],
+  [AS_HELP_STRING([--enable-c++20],
+  [enable compilation in c++20 mode (disabled by default)])],
+  [use_cxx20=$enableval],
+  [use_cxx20=no])
 
-dnl Check if -latomic is required for <std::atomic>
-CHECK_ATOMIC
+dnl Require C++17 compiler (no GNU extensions)
+if test "$use_cxx20" = "no"; then
+AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory])
+else
+AX_CXX_COMPILE_STDCXX([20], [noext], [mandatory])
+fi
 
 dnl check if additional link flags are required for std::filesystem
 CHECK_FILESYSTEM
@@ -95,17 +110,13 @@ fi
 AC_PROG_OBJCXX
 ])
 
-dnl Since libtool 1.5.2 (released 2004-01-25), on Linux libtool no longer
-dnl sets RPATH for any directories in the dynamic linker search path.
-dnl See more: https://wiki.debian.org/RpathIssue
-LT_PREREQ([1.5.2])
+dnl OpenBSD ships with 2.4.2
+LT_PREREQ([2.4.2])
 dnl Libtool init checks.
 LT_INIT([pic-only win32-dll])
 
 dnl Check/return PATH for base programs.
 AC_PATH_TOOL([AR], [ar])
-AC_PATH_TOOL([RANLIB], [ranlib])
-AC_PATH_TOOL([STRIP], [strip])
 AC_PATH_TOOL([GCOV], [gcov])
 AC_PATH_TOOL([LLVM_COV], [llvm-cov])
 AC_PATH_PROG([LCOV], [lcov])
@@ -322,7 +333,7 @@ AC_ARG_ENABLE([werror],
     [enable_werror=no])
 
 AC_ARG_ENABLE([external-signer],
-    [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is yes, requires Boost::Process)])],
+    [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is auto, requires Boost::Process)])],
     [use_external_signer=$enableval],
     [use_external_signer=auto])
 
@@ -357,7 +368,9 @@ case $host in
 esac
 
 if test "$enable_debug" = "yes"; then
-  dnl Clear default -g -O2 flags
+  dnl If debugging is enabled, and the user hasn't overridden CXXFLAGS, clear
+  dnl them, to prevent autoconfs "-g -O2" being added. Otherwise we'd end up
+  dnl with "-O0 -g3 -g -O2".
   if test "$CXXFLAGS_overridden" = "no"; then
   CXXFLAGS=""
   fi
@@ -374,6 +387,8 @@ if test "$enable_debug" = "yes"; then
 
   AX_CHECK_PREPROC_FLAG([-DDEBUG], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG"], [], [$CXXFLAG_WERROR])
   AX_CHECK_PREPROC_FLAG([-DDEBUG_LOCKORDER], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG_LOCKORDER"], [], [$CXXFLAG_WERROR])
+  AX_CHECK_PREPROC_FLAG([-DDEBUG_LOCKCONTENTION], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG_LOCKCONTENTION"], [], [$CXXFLAG_WERROR])
+  AX_CHECK_PREPROC_FLAG([-DRPC_DOC_CHECK], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DRPC_DOC_CHECK"], [], [$CXXFLAG_WERROR])
   AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"], [], [$CXXFLAG_WERROR])
   AX_CHECK_COMPILE_FLAG([-ftrapv], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -ftrapv"], [], [$CXXFLAG_WERROR])
 fi
@@ -464,8 +479,10 @@ if test "$CXXFLAGS_overridden" = "no"; then
 fi
 
 dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review.
-AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CXXFLAGS="$CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR])
+AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR])
 
+enable_arm_crc=no
+enable_arm_shani=no
 enable_sse42=no
 enable_sse41=no
 enable_avx2=no
@@ -502,7 +519,7 @@ if test "$enable_clmul" = "yes"; then
 fi
 
 TEMP_CXXFLAGS="$CXXFLAGS"
-CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS"
+CXXFLAGS="$SSE42_CXXFLAGS $CXXFLAGS"
 AC_MSG_CHECKING([for SSE4.2 intrinsics])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
     #include <stdint.h>
@@ -524,7 +541,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 CXXFLAGS="$TEMP_CXXFLAGS"
 
 TEMP_CXXFLAGS="$CXXFLAGS"
-CXXFLAGS="$CXXFLAGS $SSE41_CXXFLAGS"
+CXXFLAGS="$SSE41_CXXFLAGS $CXXFLAGS"
 AC_MSG_CHECKING([for SSE4.1 intrinsics])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
     #include <stdint.h>
@@ -539,7 +556,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 CXXFLAGS="$TEMP_CXXFLAGS"
 
 TEMP_CXXFLAGS="$CXXFLAGS"
-CXXFLAGS="$CXXFLAGS $AVX2_CXXFLAGS"
+CXXFLAGS="$AVX2_CXXFLAGS $CXXFLAGS"
 AC_MSG_CHECKING([for AVX2 intrinsics])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
     #include <stdint.h>
@@ -554,7 +571,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 CXXFLAGS="$TEMP_CXXFLAGS"
 
 TEMP_CXXFLAGS="$CXXFLAGS"
-CXXFLAGS="$CXXFLAGS $X86_SHANI_CXXFLAGS"
+CXXFLAGS="$X86_SHANI_CXXFLAGS $CXXFLAGS"
 AC_MSG_CHECKING([for x86 SHA-NI intrinsics])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
     #include <stdint.h>
@@ -575,7 +592,7 @@ AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_CRC_CXXFLAGS="-march=arm
 AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_SHANI_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR])
 
 TEMP_CXXFLAGS="$CXXFLAGS"
-CXXFLAGS="$CXXFLAGS $ARM_CRC_CXXFLAGS"
+CXXFLAGS="$ARM_CRC_CXXFLAGS $CXXFLAGS"
 AC_MSG_CHECKING([for ARMv8 CRC32 intrinsics])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
     #include <arm_acle.h>
@@ -594,7 +611,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 CXXFLAGS="$TEMP_CXXFLAGS"
 
 TEMP_CXXFLAGS="$CXXFLAGS"
-CXXFLAGS="$CXXFLAGS $ARM_SHANI_CXXFLAGS"
+CXXFLAGS="$ARM_SHANI_CXXFLAGS $CXXFLAGS"
 AC_MSG_CHECKING([for ARMv8 SHA-NI intrinsics])
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
     #include <arm_acle.h>
@@ -613,7 +630,7 @@ CXXFLAGS="$TEMP_CXXFLAGS"
 
 fi
 
-CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO"
+CORE_CPPFLAGS="$CORE_CPPFLAGS -DHAVE_BUILD_INFO"
 
 AC_ARG_WITH([utils],
   [AS_HELP_STRING([--with-utils],
@@ -645,12 +662,24 @@ AC_ARG_ENABLE([util-util],
   [build_bitcoin_util=$enableval],
   [build_bitcoin_util=$build_bitcoin_utils])
 
+AC_ARG_ENABLE([experimental-util-chainstate],
+  [AS_HELP_STRING([--enable-experimental-util-chainstate],
+  [build experimental bitcoin-chainstate executable (default=no)])],
+  [build_bitcoin_chainstate=$enableval],
+  [build_bitcoin_chainstate=no])
+
 AC_ARG_WITH([libs],
   [AS_HELP_STRING([--with-libs],
   [build libraries (default=yes)])],
   [build_bitcoin_libs=$withval],
   [build_bitcoin_libs=yes])
 
+AC_ARG_WITH([experimental-kernel-lib],
+  [AS_HELP_STRING([--with-experimental-kernel-lib],
+  [build experimental bitcoinkernel library (default is to build if we're building libraries and the experimental build-chainstate executable)])],
+  [build_experimental_kernel_lib=$withval],
+  [build_experimental_kernel_lib=auto])
+
 AC_ARG_WITH([daemon],
   [AS_HELP_STRING([--with-daemon],
   [build bitcoind daemon (default=yes)])],
@@ -689,7 +718,10 @@ case $host in
        AC_MSG_ERROR([windres not found])
      fi
 
-     CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN"
+     CORE_CPPFLAGS="$CORE_CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN"
+     dnl Prevent the definition of min/max macros.
+     dnl We always want to use the standard library.
+     CORE_CPPFLAGS="$CORE_CPPFLAGS -DNOMINMAX"
 
      dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against.
      dnl That breaks our ability to build dll's with static libgcc/libstdc++/libssp. Override
@@ -700,7 +732,7 @@ case $host in
      postdeps_CXX=
 
      dnl We require Windows 7 (NT 6.1) or later
-     AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [LDFLAGS="$LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR])
+     AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR])
      ;;
   *darwin*)
      TARGET_OS=darwin
@@ -714,6 +746,16 @@ case $host in
          dnl It's safe to add these paths even if the functionality is disabled by
          dnl the user (--without-wallet or --without-gui for example).
 
+         dnl Homebrew may create symlinks in /usr/local/include for some packages.
+         dnl Because MacOS's clang internally adds "-I /usr/local/include" to its search
+         dnl paths, this will negate efforts to use -isystem for those packages, as they
+         dnl will be found first in /usr/local. Use the internal "-internal-isystem"
+         dnl option to system-ify all /usr/local/include paths without adding it to the list
+         dnl of search paths in case it's not already there.
+         if test "$suppress_external_warnings" != "no"; then
+           AX_CHECK_PREPROC_FLAG([-Xclang -internal-isystem/usr/local/include], [CORE_CPPFLAGS="$CORE_CPPFLAGS -Xclang -internal-isystem/usr/local/include"], [], [$CXXFLAG_WERROR])
+         fi
+
          if test "$use_bdb" != "no" && $BREW list --versions berkeley-db@4 >/dev/null && test "$BDB_CFLAGS" = "" && test "$BDB_LIBS" = ""; then
            bdb_prefix=$($BREW --prefix berkeley-db@4 2>/dev/null)
            dnl This must precede the call to BITCOIN_FIND_BDB48 below.
@@ -734,20 +776,20 @@ case $host in
              if test "$use_upnp" != "no" && $BREW list --versions miniupnpc >/dev/null; then
                miniupnpc_prefix=$($BREW --prefix miniupnpc 2>/dev/null)
                if test "$suppress_external_warnings" != "no"; then
-                 CPPFLAGS="$CPPFLAGS -isystem $miniupnpc_prefix/include"
+                 MINIUPNPC_CPPFLAGS="-isystem $miniupnpc_prefix/include"
                else
-                 CPPFLAGS="$CPPFLAGS -I$miniupnpc_prefix/include"
+                 MINIUPNPC_CPPFLAGS="-I$miniupnpc_prefix/include"
                fi
-               LDFLAGS="$LDFLAGS -L$miniupnpc_prefix/lib"
+               MINIUPNPC_LIBS="-L$miniupnpc_prefix/lib"
              fi
              if test "$use_natpmp" != "no" && $BREW list --versions libnatpmp >/dev/null; then
                libnatpmp_prefix=$($BREW --prefix libnatpmp 2>/dev/null)
                if test "$suppress_external_warnings" != "no"; then
-                 CPPFLAGS="$CPPFLAGS -isystem $libnatpmp_prefix/include"
+                 NATPMP_CPPFLAGS="-isystem $libnatpmp_prefix/include"
                else
-                 CPPFLAGS="$CPPFLAGS -I$libnatpmp_prefix/include"
+                 NATPMP_CPPFLAGS="-I$libnatpmp_prefix/include"
                fi
-               LDFLAGS="$LDFLAGS -L$libnatpmp_prefix/lib"
+               NATPMP_LIBS="-L$libnatpmp_prefix/lib"
              fi
              ;;
          esac
@@ -759,10 +801,9 @@ case $host in
            ;;
          *)
            AC_PATH_TOOL([DSYMUTIL], [dsymutil], [dsymutil])
-           AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], [install_name_tool])
+           AC_PATH_TOOL([INSTALL_NAME_TOOL], [install_name_tool], [install_name_tool])
            AC_PATH_TOOL([OTOOL], [otool], [otool])
            AC_PATH_PROGS([XORRISOFS], [xorrisofs], [xorrisofs])
-           AC_PATH_PROGS([DMG], [dmg], [dmg])
 
            dnl libtool will try to strip the static lib, which is a problem for
            dnl cross-builds because strip attempts to call a hard-coded ld,
@@ -773,8 +814,8 @@ case $host in
        esac
      fi
 
-     AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR])
-     CPPFLAGS="$CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0"
+     AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR])
+     CORE_CPPFLAGS="$CORE_CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0"
      OBJCXXFLAGS="$CXXFLAGS"
      ;;
    *android*)
@@ -837,11 +878,17 @@ if test "$use_lcov" = "yes"; then
   AC_SUBST(COV_TOOL_WRAPPER, "cov_tool_wrapper.sh")
   LCOV="$LCOV --gcov-tool $(pwd)/$COV_TOOL_WRAPPER"
 
-  AX_CHECK_LINK_FLAG([--coverage], [LDFLAGS="$LDFLAGS --coverage"],
+  AX_CHECK_LINK_FLAG([--coverage], [CORE_LDFLAGS="$CORE_LDFLAGS --coverage"],
     [AC_MSG_ERROR([lcov testing requested but --coverage linker flag does not work])])
-  AX_CHECK_COMPILE_FLAG([--coverage],[CXXFLAGS="$CXXFLAGS --coverage"],
+  AX_CHECK_COMPILE_FLAG([--coverage],[CORE_CXXFLAGS="$CORE_CXXFLAGS --coverage"],
     [AC_MSG_ERROR([lcov testing requested but --coverage flag does not work])])
-  CXXFLAGS="$CXXFLAGS -Og"
+  dnl If coverage is enabled, and the user hasn't overridden CXXFLAGS, clear
+  dnl them, to prevent autoconfs "-g -O2" being added. Otherwise we'd end up
+  dnl with "--coverage -Og -O0 -g -O2".
+  if test "$CXXFLAGS_overridden" = "no"; then
+  CXXFLAGS=""
+  fi
+  CORE_CXXFLAGS="$CORE_CXXFLAGS -Og -O0"
 fi
 
 if test "$use_lcov_branch" != "no"; then
@@ -854,6 +901,9 @@ AC_C_BIGENDIAN
 dnl Check for pthread compile/link requirements
 AX_PTHREAD
 
+dnl Check if -latomic is required for <std::atomic>
+CHECK_ATOMIC
+
 dnl The following macro will add the necessary defines to bitcoin-config.h, but
 dnl they also need to be passed down to any subprojects. Pull the results out of
 dnl the cache and add them to CPPFLAGS.
@@ -864,13 +914,13 @@ AC_FUNC_STRERROR_R
 if test "$ac_cv_sys_file_offset_bits" != "" &&
    test "$ac_cv_sys_file_offset_bits" != "no" &&
    test "$ac_cv_sys_file_offset_bits" != "unknown"; then
-  CPPFLAGS="$CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits"
+  CORE_CPPFLAGS="$CORE_CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits"
 fi
 
 if test "$ac_cv_sys_large_files" != "" &&
    test "$ac_cv_sys_large_files" != "no" &&
    test "$ac_cv_sys_large_files" != "unknown"; then
-  CPPFLAGS="$CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files"
+  CORE_CPPFLAGS="$CORE_CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files"
 fi
 
 AC_SEARCH_LIBS([clock_gettime],[rt])
@@ -906,10 +956,7 @@ if test "$use_hardening" != "no"; then
   AX_CHECK_COMPILE_FLAG([-Wstack-protector], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"])
   AX_CHECK_COMPILE_FLAG([-fstack-protector-all], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"])
 
-  dnl -fcf-protection used with Clang 7 causes ld to emit warnings:
-  dnl ld: error: ... <corrupt x86 feature size: 0x8>
-  dnl Use CHECK_LINK_FLAG & --fatal-warnings to ensure we won't use the flag in this case.
-  AX_CHECK_LINK_FLAG([-fcf-protection=full], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"], [], [$LDFLAG_WERROR])
+  AX_CHECK_COMPILE_FLAG([-fcf-protection=full], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"])
 
   case $host in
     *mingw*)
@@ -954,8 +1001,8 @@ dnl These flags are specific to ld64, and may cause issues with other linkers.
 dnl For example: GNU ld will interpret -dead_strip as -de and then try and use
 dnl "ad_strip" as the symbol for the entry point.
 if test "$TARGET_OS" = "darwin"; then
-  AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR])
-  AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR])
+  AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR])
+  AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR])
   AX_CHECK_LINK_FLAG([-Wl,-bind_at_load], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"], [], [$LDFLAG_WERROR])
 fi
 
@@ -965,7 +1012,6 @@ AC_CHECK_DECLS([getifaddrs, freeifaddrs],[CHECK_SOCKET],,
     [#include <sys/types.h>
     #include <ifaddrs.h>]
 )
-AC_CHECK_DECLS([strnlen])
 
 dnl These are used for daemonization in bitcoind
 AC_CHECK_DECLS([fork])
@@ -973,6 +1019,8 @@ AC_CHECK_DECLS([setsid])
 
 AC_CHECK_DECLS([pipe2])
 
+AC_CHECK_FUNCS([timingsafe_bcmp])
+
 AC_CHECK_DECLS([le16toh, le32toh, le64toh, htole16, htole32, htole64, be16toh, be32toh, be64toh, htobe16, htobe32, htobe64],,,
     [#if HAVE_ENDIAN_H
                  #include <endian.h>
@@ -1234,8 +1282,8 @@ AC_LINK_IFELSE(
 AC_MSG_CHECKING([for ::_wsystem])
 AC_LINK_IFELSE(
     [ AC_LANG_PROGRAM(
-        [[ ]],
-        [[ int nErr = ::_wsystem(""); ]]
+        [[ #include <stdlib.h> ]],
+        [[ int nErr = ::_wsystem(NULL); ]]
     )],
     [ AC_MSG_RESULT([yes]); have_any_system=yes],
     [ AC_MSG_RESULT([no]) ]
@@ -1255,7 +1303,7 @@ dnl Do not change "-I/usr/include" to "-isystem /usr/include" because that
 dnl is not necessary (/usr/include is already a system directory) and because
 dnl it would break GCC's #include_next.
 AC_DEFUN([SUPPRESS_WARNINGS],
-         [$(echo $1 |${SED} -E -e 's/(^| )-I/\1-isystem /g' -e 's;-isystem /usr/include([/ ]|$);-I/usr/include\1;g')])
+         [[$(echo $1 |${SED} -E -e 's/(^| )-I/\1-isystem /g' -e 's;-isystem /usr/include/*( |$);-I/usr/include\1;g')]])
 
 dnl enable-fuzz should disable all other targets
 if test "$enable_fuzz" = "yes"; then
@@ -1264,6 +1312,7 @@ if test "$enable_fuzz" = "yes"; then
   build_bitcoin_cli=no
   build_bitcoin_tx=no
   build_bitcoin_util=no
+  build_bitcoin_chainstate=no
   build_bitcoin_wallet=no
   build_bitcoind=no
   build_bitcoin_libs=no
@@ -1271,6 +1320,7 @@ if test "$enable_fuzz" = "yes"; then
   bitcoin_enable_qt_test=no
   bitcoin_enable_qt_dbus=no
   use_bench=no
+  use_tests=no
   use_external_signer=no
   use_upnp=no
   use_natpmp=no
@@ -1278,26 +1328,11 @@ if test "$enable_fuzz" = "yes"; then
   enable_fuzz_binary=yes
 
   AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"], [], [$CXXFLAG_WERROR])
-
-  AC_MSG_CHECKING([whether main function is needed for fuzz binary])
-  AX_CHECK_LINK_FLAG(
-    [-fsanitize=$use_sanitizers],
-    [AC_MSG_RESULT([no])],
-    [AC_MSG_RESULT([yes]); CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"],
-    [],
-    [AC_LANG_PROGRAM([[
-      #include <cstdint>
-      #include <cstddef>
-      extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { return 0; }
-      /* comment to remove the main function ...
-     ]],[[
-      */ int not_main() {
-     ]])])
 else
   BITCOIN_QT_INIT
 
   dnl sets $bitcoin_enable_qt, $bitcoin_enable_qt_test, $bitcoin_enable_qt_dbus
-  BITCOIN_QT_CONFIGURE([5.9.5])
+  BITCOIN_QT_CONFIGURE([5.11.3])
 
   dnl Keep a copy of the original $QT_INCLUDES and use it when invoking qt's moc
   QT_INCLUDES_UNSUPPRESSED=$QT_INCLUDES
@@ -1306,8 +1341,25 @@ else
     QT_DBUS_INCLUDES=SUPPRESS_WARNINGS($QT_DBUS_INCLUDES)
     QT_TEST_INCLUDES=SUPPRESS_WARNINGS($QT_TEST_INCLUDES)
   fi
+fi
 
-  CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"
+if test "$enable_fuzz_binary" = "yes"; then
+  AC_MSG_CHECKING([whether main function is needed for fuzz binary])
+  AX_CHECK_LINK_FLAG(
+    [],
+    [AC_MSG_RESULT([no])],
+    [AC_MSG_RESULT([yes]); CORE_CPPFLAGS="$CORE_CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"],
+    [$SANITIZER_LDFLAGS],
+    [AC_LANG_PROGRAM([[
+      #include <cstdint>
+      #include <cstddef>
+      extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { return 0; }
+      /* comment to remove the main function ...
+     ]],[[
+      */ int not_main() {
+     ]])])
+
+  CHECK_RUNTIME_LIB
 fi
 
 if test "$enable_wallet" != "no"; then
@@ -1359,6 +1411,7 @@ if test "$use_usdt" != "no"; then
     [AC_MSG_RESULT([no]); use_usdt=no;]
   )
 fi
+AM_CONDITIONAL([ENABLE_USDT_TRACEPOINTS], [test "$use_usdt" = "yes"])
 
 if test "$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nonononononono"; then
   use_upnp=no
@@ -1368,38 +1421,44 @@ fi
 
 dnl Check for libminiupnpc (optional)
 if test "$use_upnp" != "no"; then
+  TEMP_CPPFLAGS="$CPPFLAGS"
+  CPPFLAGS="$CPPFLAGS $MINIUPNPC_CPPFLAGS"
   AC_CHECK_HEADERS(
     [miniupnpc/miniupnpc.h miniupnpc/upnpcommands.h miniupnpc/upnperrors.h],
-    [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS=-lminiupnpc], [have_miniupnpc=no])],
+    [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS="$MINIUPNPC_LIBS -lminiupnpc"], [have_miniupnpc=no], [$MINIUPNPC_LIBS])],
     [have_miniupnpc=no]
   )
-dnl The minimum supported miniUPnPc API version is set to 10. This keeps compatibility
-dnl with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages.
-if test "$have_miniupnpc" != "no"; then
-  AC_MSG_CHECKING([whether miniUPnPc API version is supported])
-  AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[
-      @%:@include <miniupnpc/miniupnpc.h>
-    ]], [[
-      #if MINIUPNPC_API_VERSION >= 10
-      // Everything is okay
-      #else
-      #  error miniUPnPc API version is too old
-      #endif
-    ]])],[
-      AC_MSG_RESULT([yes])
-    ],[
-    AC_MSG_RESULT([no])
-    AC_MSG_WARN([miniUPnPc API version < 10 is unsupported, disabling UPnP support.])
-    have_miniupnpc=no
-  ])
-fi
+  dnl The minimum supported miniUPnPc API version is set to 10. This keeps compatibility
+  dnl with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages.
+  if test "$have_miniupnpc" != "no"; then
+    AC_MSG_CHECKING([whether miniUPnPc API version is supported])
+    AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[
+        @%:@include <miniupnpc/miniupnpc.h>
+      ]], [[
+        #if MINIUPNPC_API_VERSION >= 10
+        // Everything is okay
+        #else
+        #  error miniUPnPc API version is too old
+        #endif
+      ]])],[
+        AC_MSG_RESULT([yes])
+      ],[
+      AC_MSG_RESULT([no])
+      AC_MSG_WARN([miniUPnPc API version < 10 is unsupported, disabling UPnP support.])
+      have_miniupnpc=no
+    ])
+  fi
+  CPPFLAGS="$TEMP_CPPFLAGS"
 fi
 
 dnl Check for libnatpmp (optional).
 if test "$use_natpmp" != "no"; then
+  TEMP_CPPFLAGS="$CPPFLAGS"
+  CPPFLAGS="$CPPFLAGS $NATPMP_CPPFLAGS"
   AC_CHECK_HEADERS([natpmp.h],
-                   [AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS=-lnatpmp], [have_natpmp=no])],
+                   [AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS="$NATPMP_LIBS -lnatpmp"], [have_natpmp=no], [$NATPMP_LIBS])],
                    [have_natpmp=no])
+  CPPFLAGS="$TEMP_CPPFLAGS"
 fi
 
 if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" = "nonononononono"; then
@@ -1416,6 +1475,18 @@ if test "$use_boost" = "yes"; then
     AC_MSG_ERROR([only libbitcoinconsensus can be built without Boost])
   fi
 
+  dnl we don't use multi_index serialization
+  BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_DISABLE_SERIALIZATION"
+
+  dnl Prevent use of std::unary_function, which was removed in C++17,
+  dnl and will generate warnings with newer compilers.
+  dnl See: https://github.com/boostorg/container_hash/issues/22.
+  BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_NO_CXX98_FUNCTION_BASE"
+
+  if test "$enable_debug" = "yes" || test "$enable_fuzz" = "yes"; then
+    BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE"
+  fi
+
   if test "$suppress_external_warnings" != "no"; then
     BOOST_CPPFLAGS=SUPPRESS_WARNINGS($BOOST_CPPFLAGS)
   fi
@@ -1496,7 +1567,7 @@ AM_CONDITIONAL([ENABLE_SYSCALL_SANDBOX], [test "$use_syscall_sandbox" != "no"])
 
 dnl Check for reduced exports
 if test "$use_reduce_exports" = "yes"; then
-  AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CXXFLAGS="$CXXFLAGS -fvisibility=hidden"],
+  AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fvisibility=hidden"],
   [AC_MSG_ERROR([Cannot set hidden symbol visibility. Use --disable-reduce-exports.])], [$CXXFLAG_WERROR])
   AX_CHECK_LINK_FLAG([-Wl,--exclude-libs,ALL], [RELDFLAGS="-Wl,--exclude-libs,ALL"], [], [$LDFLAG_WERROR])
 fi
@@ -1510,10 +1581,11 @@ fi
 
 dnl libevent check
 
-if test "$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" != "nonononono"; then
-  PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.0.21 or greater not found.])])
+use_libevent=no
+if test "$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_tests$use_bench" != "nononononono"; then
+  PKG_CHECK_MODULES([EVENT], [libevent >= 2.1.8], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.1.8 or greater not found.])])
   if test "$TARGET_OS" != "windows"; then
-    PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR([libevent_pthreads version 2.0.21 or greater not found.])])
+    PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.1.8], [], [AC_MSG_ERROR([libevent_pthreads version 2.1.8 or greater not found.])])
   fi
 
   if test "$suppress_external_warnings" != "no"; then
@@ -1628,12 +1700,25 @@ AC_MSG_CHECKING([whether to build bitcoin-util])
 AM_CONDITIONAL([BUILD_BITCOIN_UTIL], [test $build_bitcoin_util = "yes"])
 AC_MSG_RESULT($build_bitcoin_util)
 
+AC_MSG_CHECKING([whether to build experimental bitcoin-chainstate])
+if test "$build_bitcoin_chainstate" = "yes"; then
+  if test "$build_experimental_kernel_lib" = "no"; then
+    AC_MSG_ERROR([experimental bitcoin-chainstate cannot be built without the experimental bitcoinkernel library. Use --with-experimental-kernel-lib]);
+  fi
+fi
+AM_CONDITIONAL([BUILD_BITCOIN_CHAINSTATE], [test $build_bitcoin_chainstate = "yes"])
+AC_MSG_RESULT($build_bitcoin_chainstate)
+
 AC_MSG_CHECKING([whether to build libraries])
 AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test $build_bitcoin_libs = "yes"])
+
 if test "$build_bitcoin_libs" = "yes"; then
   AC_DEFINE([HAVE_CONSENSUS_LIB], [1], [Define this symbol if the consensus lib has been built])
   AC_CONFIG_FILES([libbitcoinconsensus.pc:libbitcoinconsensus.pc.in])
 fi
+
+AM_CONDITIONAL([BUILD_BITCOIN_KERNEL_LIB], [test "$build_experimental_kernel_lib" != "no" && ( test "$build_experimental_kernel_lib" = "yes" || test "$build_bitcoin_chainstate" = "yes" )])
+
 AC_MSG_RESULT($build_bitcoin_libs)
 
 AC_LANG_POP
@@ -1690,7 +1775,7 @@ else
     AC_MSG_RESULT([$use_upnp_default])
     AC_DEFINE_UNQUOTED([USE_UPNP],[$upnp_setting],[UPnP support not compiled if undefined, otherwise value (0 or 1) determines default state])
     if test "$TARGET_OS" = "windows"; then
-      MINIUPNPC_CPPFLAGS="-DSTATICLIB -DMINIUPNP_STATICLIB"
+      MINIUPNPC_CPPFLAGS="$MINIUPNPC_CPPFLAGS -DSTATICLIB -DMINIUPNP_STATICLIB"
     fi
   else
     AC_MSG_RESULT([no])
@@ -1718,7 +1803,7 @@ else
     AC_MSG_RESULT($use_natpmp_default)
     AC_DEFINE_UNQUOTED([USE_NATPMP], [$natpmp_setting], [NAT-PMP support not compiled if undefined, otherwise value (0 or 1) determines default state])
     if test "$TARGET_OS" = "windows"; then
-      NATPMP_CPPFLAGS="-DSTATICLIB -DNATPMP_STATICLIB"
+      NATPMP_CPPFLAGS="$NATPMP_CPPFLAGS -DSTATICLIB -DNATPMP_STATICLIB"
     fi
   else
     AC_MSG_RESULT([no])
@@ -1785,12 +1870,8 @@ else
   AC_MSG_RESULT([no])
 fi
 
-if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nononononononono"; then
-  AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-bench or --enable-tests])
-fi
-
-if test "$enable_fuzz_binary" = "yes"; then
-  CHECK_RUNTIME_LIB
+if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_bench$use_tests" = "nonononononononono"; then
+  AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-fuzz(-binary) --enable-bench or --enable-tests])
 fi
 
 AM_CONDITIONAL([TARGET_DARWIN], [test "$TARGET_OS" = "darwin"])
@@ -1844,14 +1925,19 @@ AC_SUBST(COPYRIGHT_HOLDERS_SUBSTITUTION, "_COPYRIGHT_HOLDERS_SUBSTITUTION")
 AC_SUBST(COPYRIGHT_HOLDERS_FINAL, "_COPYRIGHT_HOLDERS_FINAL")
 AC_SUBST(BITCOIN_DAEMON_NAME)
 AC_SUBST(BITCOIN_GUI_NAME)
+AC_SUBST(BITCOIN_TEST_NAME)
 AC_SUBST(BITCOIN_CLI_NAME)
 AC_SUBST(BITCOIN_TX_NAME)
 AC_SUBST(BITCOIN_UTIL_NAME)
+AC_SUBST(BITCOIN_CHAINSTATE_NAME)
 AC_SUBST(BITCOIN_WALLET_TOOL_NAME)
 AC_SUBST(BITCOIN_MP_NODE_NAME)
 AC_SUBST(BITCOIN_MP_GUI_NAME)
 
 AC_SUBST(RELDFLAGS)
+AC_SUBST(CORE_LDFLAGS)
+AC_SUBST(CORE_CPPFLAGS)
+AC_SUBST(CORE_CXXFLAGS)
 AC_SUBST(DEBUG_CPPFLAGS)
 AC_SUBST(WARN_CXXFLAGS)
 AC_SUBST(NOWARN_CXXFLAGS)
@@ -1902,12 +1988,18 @@ AC_CONFIG_LINKS([contrib/devtools/security-check.py:contrib/devtools/security-ch
 AC_CONFIG_LINKS([contrib/devtools/symbol-check.py:contrib/devtools/symbol-check.py])
 AC_CONFIG_LINKS([contrib/devtools/test-security-check.py:contrib/devtools/test-security-check.py])
 AC_CONFIG_LINKS([contrib/devtools/test-symbol-check.py:contrib/devtools/test-symbol-check.py])
+AC_CONFIG_LINKS([contrib/devtools/iwyu/bitcoin.core.imp:contrib/devtools/iwyu/bitcoin.core.imp])
 AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py])
 AC_CONFIG_LINKS([contrib/macdeploy/background.tiff:contrib/macdeploy/background.tiff])
+AC_CONFIG_LINKS([src/.bear-tidy-config:src/.bear-tidy-config])
+AC_CONFIG_LINKS([src/.clang-tidy:src/.clang-tidy])
 AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py])
 AC_CONFIG_LINKS([test/fuzz/test_runner.py:test/fuzz/test_runner.py])
 AC_CONFIG_LINKS([test/util/test_runner.py:test/util/test_runner.py])
 AC_CONFIG_LINKS([test/util/rpcauth-test.py:test/util/rpcauth-test.py])
+AC_CONFIG_LINKS([src/qt/Makefile:src/qt/Makefile])
+AC_CONFIG_LINKS([src/qt/test/Makefile:src/qt/test/Makefile])
+AC_CONFIG_LINKS([src/test/Makefile:src/test/Makefile])
 
 dnl boost's m4 checks do something really nasty: they export these vars. As a
 dnl result, they leak into secp256k1's configure and crazy things happen.
@@ -1924,15 +2016,7 @@ LIBS_TEMP="$LIBS"
 unset LIBS
 LIBS="$LIBS_TEMP"
 
-PKGCONFIG_PATH_TEMP="$PKG_CONFIG_PATH"
-unset PKG_CONFIG_PATH
-PKG_CONFIG_PATH="$PKGCONFIG_PATH_TEMP"
-
-PKGCONFIG_LIBDIR_TEMP="$PKG_CONFIG_LIBDIR"
-unset PKG_CONFIG_LIBDIR
-PKG_CONFIG_LIBDIR="$PKGCONFIG_LIBDIR_TEMP"
-
-ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --enable-module-schnorrsig --enable-experimental"
+ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --enable-module-schnorrsig"
 AC_CONFIG_SUBDIRS([src/secp256k1])
 
 AC_OUTPUT
@@ -1983,9 +2067,10 @@ echo "  build os        = $build_os"
 echo
 echo "  CC              = $CC"
 echo "  CFLAGS          = $PTHREAD_CFLAGS $CFLAGS"
-echo "  CPPFLAGS        = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS"
+echo "  CPPFLAGS        = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CORE_CPPFLAGS $CPPFLAGS"
 echo "  CXX             = $CXX"
-echo "  CXXFLAGS        = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS"
-echo "  LDFLAGS         = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS"
+echo "  CXXFLAGS        = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CORE_CXXFLAGS $CXXFLAGS"
+echo "  LDFLAGS         = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $CORE_LDFLAGS $LDFLAGS"
+echo "  AR              = $AR"
 echo "  ARFLAGS         = $ARFLAGS"
 echo

contrib/builder-keys/README.md

@@ -19,9 +19,15 @@ gpg --refresh-keys
 To fetch keys of builders and active developers, feed the list of fingerprints
 of the primary keys into gpg:
 
+On \*NIX:
 ```sh
 while read fingerprint keyholder_name; do gpg --keyserver hkps://keys.openpgp.org --recv-keys ${fingerprint}; done < ./keys.txt
 ```
 
+On Windows (requires Gpg4win >= 4.0.0):
+```
+FOR /F "tokens=1" %i IN (keys.txt) DO gpg --keyserver hkps://keys.openpgp.org --recv-keys %i
+```
+
 Add your key to the list if you provided Guix attestations for two major or
 minor releases of Bitcoin Core.

contrib/builder-keys/keys.txt

@@ -13,11 +13,13 @@ F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur (centaur)
 C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields (cfields)
 BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random (devrandom)
 6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark (droark)
+948444FCE03B05BA5AB0591EC37B1C1D44C786EE Duncan Dean (dunxen)
 1C6621605EC50319C463D56C7F81D87985D61612 Emanuele Cisbani (cisba)
 9A1689B60D1B3CCE9262307A2F40A9BF167FBA47 Erik Mossberg (erkmos)
 D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece)
 6F993B250557E7B016ADE5713BDCDA2D87A881D9 Fuzzbawls (Fuzzbawls)
 01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen (gavinandresen)
+6B002C6EA3F91B1B0DF0C9BC8F617F1200A6D25C Gloria Zhao (glozow)
 D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto)
 A2FD494D0021AA9B4FA58F759102B7AE654A4A5A Ilyas Ridhuan (IlyasRidhuan)
 2688F5A9A4BE0F295E921E8A25F27A38A47AD566 James O'Beirne (jamesob)
@@ -50,5 +52,6 @@ ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost (sjors)
 9EDAFF80E080659604F4A76B2EBB056FD847F8A7 Stephan Oeste (Emzy)
 6DEEF79B050C4072509B743F8C275BC595448867 Tomas Kanocz (KanoczTomas)
 AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami (wtogami)
+74E2DEF5D77260B98BC19438099BAD163C70FBFA Will Clark (will8clark)
 79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko (willyko)
 71A3B16735405025D447E8F274810B012346C9A6 Wladimir J. van der Laan (laanwj)

contrib/devtools/README.md

@@ -90,6 +90,21 @@ example:
 BUILDDIR=$PWD/build contrib/devtools/gen-manpages.py
 ```
 
+gen-bitcoin-conf.sh
+===================
+
+Generates a bitcoin.conf file in `share/examples/` by parsing the output from `bitcoind --help`. This script is run during the
+release process to include a bitcoin.conf with the release binaries and can also be run by users to generate a file locally.
+When generating a file as part of the release process, make sure to commit the changes after running the script.
+
+With in-tree builds this tool can be run from any directory within the
+repository. To use this tool with out-of-tree builds set `BUILDDIR`. For
+example:
+
+```bash
+BUILDDIR=$PWD/build contrib/devtools/gen-bitcoin-conf.sh
+```
+
 security-check.py and test-security-check.py
 ============================================
 

contrib/devtools/copyright_header.py

@@ -35,7 +35,6 @@ EXCLUDE_DIRS = [
     "src/leveldb/",
     "src/minisketch",
     "src/secp256k1/",
-    "src/univalue/",
     "src/crc32c/",
 ]
 
@@ -320,15 +319,13 @@ def get_most_recent_git_change_year(filename):
 ################################################################################
 
 def read_file_lines(filename):
-    f = open(filename, 'r', encoding="utf8")
-    file_lines = f.readlines()
-    f.close()
+    with open(filename, 'r', encoding="utf8") as f:
+        file_lines = f.readlines()
     return file_lines
 
 def write_file_lines(filename, file_lines):
-    f = open(filename, 'w', encoding="utf8")
-    f.write(''.join(file_lines))
-    f.close()
+    with open(filename, 'w', encoding="utf8") as f:
+        f.write(''.join(file_lines))
 
 ################################################################################
 # update header years execution

contrib/devtools/gen-bitcoin-conf.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Copyright (c) 2021 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C
+TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)}
+BUILDDIR=${BUILDDIR:-$TOPDIR}
+BINDIR=${BINDIR:-$BUILDDIR/src}
+BITCOIND=${BITCOIND:-$BINDIR/bitcoind}
+SHARE_EXAMPLES_DIR=${SHARE_EXAMPLES_DIR:-$TOPDIR/share/examples}
+EXAMPLE_CONF_FILE=${EXAMPLE_CONF_FILE:-$SHARE_EXAMPLES_DIR/bitcoin.conf}
+
+[ ! -x "$BITCOIND" ] && echo "$BITCOIND not found or not executable." && exit 1
+
+DIRTY=""
+VERSION_OUTPUT=$($BITCOIND --version)
+if [[ $VERSION_OUTPUT == *"dirty"* ]]; then
+  DIRTY="${DIRTY}${BITCOIND}\n"
+fi
+
+if [ -n "$DIRTY" ]
+then
+  echo -e "WARNING: $BITCOIND was built from a dirty tree.\n"
+  echo -e "To safely generate a bitcoin.conf file, please commit your changes to $BITCOIND, rebuild, then run this script again.\n"
+fi
+
+echo 'Generating example bitcoin.conf file in share/examples/'
+
+# create the directory, if it doesn't exist
+mkdir -p "${SHARE_EXAMPLES_DIR}"
+
+# create the header text
+cat > "${EXAMPLE_CONF_FILE}" << 'EOF'
+##
+## bitcoin.conf configuration file.
+## Generated by contrib/devtools/gen-bitcoin-conf.sh.
+##
+## Lines beginning with # are comments.
+## All possible configuration options are provided. To use, copy this file
+## to your data directory (default or specified by -datadir), uncomment
+## options you would like to change, and save the file.
+##
+
+
+### Options
+EOF
+
+# parse the output from bitcoind --help
+# adding newlines is a bit funky to ensure portability for BSD
+# see here for more details: https://stackoverflow.com/a/24575385
+${BITCOIND} --help \
+    | sed '1,/Print this help message and exit/d' \
+    | sed -E 's/^[[:space:]]{2}\-/#/' \
+    | sed -E 's/^[[:space:]]{7}/# /' \
+    | sed -E '/[=[:space:]]/!s/#.*$/&=1/' \
+    | awk '/^#[a-z]/{x=$0;next}{if (NF==0) print x"\n",x="";else print}' \
+    | sed 's,\(^[[:upper:]].*\)\:$,\
+### \1,' \
+    | sed 's/[[:space:]]*$//' >> "${EXAMPLE_CONF_FILE}"
+
+# create the footer text
+cat >> "${EXAMPLE_CONF_FILE}" << 'EOF'
+
+# [Sections]
+# Most options will apply to all networks. To confine an option to a specific
+# network, add it under the relevant section below.
+#
+# Note: If not specified under a network section, the options addnode, connect,
+# port, bind, rpcport, rpcbind, and wallet will only apply to mainnet.
+
+# Options for mainnet
+[main]
+
+# Options for testnet
+[test]
+
+# Options for signet
+[signet]
+
+# Options for regtest
+[regtest]
+EOF

contrib/devtools/iwyu/bitcoin.core.imp

@@ -0,0 +1,7 @@
+# Fixups / upstreamed changes
+[
+  { include: [ "<bits/termios-c_lflag.h>", private, "<termios.h>", public ] },
+  { include: [ "<bits/termios-struct.h>", private, "<termios.h>", public ] },
+  { include: [ "<bits/termios-tcflow.h>", private, "<termios.h>", public ] },
+  { include: [ "<bits/chrono.h>", private, "<chrono>", public ] },
+]

contrib/devtools/security-check.py

@@ -12,10 +12,6 @@ from typing import List
 
 import lief #type:ignore
 
-# temporary constant, to be replaced with lief.ELF.ARCH.RISCV
-# https://github.com/lief-project/LIEF/pull/562
-LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243)
-
 def check_ELF_RELRO(binary) -> bool:
     '''
     Check for read-only relocations.
@@ -101,7 +97,6 @@ def check_ELF_separate_code(binary):
     for segment in binary.segments:
         if segment.type ==  lief.ELF.SEGMENT_TYPES.LOAD:
             for section in segment.sections:
-                assert(section.name not in flags_per_section)
                 flags_per_section[section.name] = segment.flags
     # Spot-check ELF LOAD program header flags per section
     # If these sections exist, check them against the expected R/W/E flags
@@ -222,7 +217,7 @@ CHECKS = {
         lief.ARCHITECTURES.ARM: BASE_ELF,
         lief.ARCHITECTURES.ARM64: BASE_ELF,
         lief.ARCHITECTURES.PPC: BASE_ELF,
-        LIEF_ELF_ARCH_RISCV: BASE_ELF,
+        lief.ARCHITECTURES.RISCV: BASE_ELF,
     },
     lief.EXE_FORMATS.PE: {
         lief.ARCHITECTURES.X86: BASE_PE,
@@ -250,12 +245,9 @@ if __name__ == '__main__':
                 continue
 
             if arch == lief.ARCHITECTURES.NONE:
-                if binary.header.machine_type == LIEF_ELF_ARCH_RISCV:
-                    arch = LIEF_ELF_ARCH_RISCV
-                else:
-                    print(f'{filename}: unknown architecture')
-                    retval = 1
-                    continue
+                print(f'{filename}: unknown architecture')
+                retval = 1
+                continue
 
             failed: List[str] = []
             for (name, func) in CHECKS[etype][arch]:

contrib/devtools/symbol-check.py

@@ -15,10 +15,6 @@ from typing import List, Dict
 
 import lief #type:ignore
 
-# temporary constant, to be replaced with lief.ELF.ARCH.RISCV
-# https://github.com/lief-project/LIEF/pull/562
-LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243)
-
 # Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases
 #
 # - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B)
@@ -39,32 +35,25 @@ LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243)
 MAX_VERSIONS = {
 'GCC':       (4,8,0),
 'GLIBC': {
-    lief.ELF.ARCH.i386:   (2,18),
     lief.ELF.ARCH.x86_64: (2,18),
     lief.ELF.ARCH.ARM:    (2,18),
     lief.ELF.ARCH.AARCH64:(2,18),
     lief.ELF.ARCH.PPC64:  (2,18),
-    LIEF_ELF_ARCH_RISCV:  (2,27),
+    lief.ELF.ARCH.RISCV:  (2,27),
 },
 'LIBATOMIC': (1,0),
 'V':         (0,5,0),  # xkb (bitcoin-qt only)
 }
-# See here for a description of _IO_stdin_used:
-# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109
 
 # Ignore symbols that are exported as part of every executable
 IGNORE_EXPORTS = {
-'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__',
-'__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr',
-'environ', '_environ', '__environ',
+'environ', '_environ', '__environ', '_fini', '_init', 'stdin',
+'stdout', 'stderr',
 }
 
 # Expected linker-loader names can be found here:
 # https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16
 ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = {
-    lief.ELF.ARCH.i386:    {
-        lief.ENDIANNESS.LITTLE: "/lib/ld-linux.so.2",
-    },
     lief.ELF.ARCH.x86_64:  {
         lief.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2",
     },
@@ -78,7 +67,7 @@ ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = {
         lief.ENDIANNESS.BIG: "/lib64/ld64.so.1",
         lief.ENDIANNESS.LITTLE: "/lib64/ld64.so.2",
     },
-    LIEF_ELF_ARCH_RISCV:    {
+    lief.ELF.ARCH.RISCV:    {
         lief.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1",
     },
 }
@@ -200,7 +189,7 @@ def check_exported_symbols(binary) -> bool:
         if not symbol.exported:
             continue
         name = symbol.name
-        if binary.header.machine_type == LIEF_ELF_ARCH_RISCV or name in IGNORE_EXPORTS:
+        if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS:
             continue
         print(f'{binary.name}: export of symbol {name} not allowed!')
         ok = False

contrib/devtools/test-symbol-check.py

@@ -39,7 +39,7 @@ class TestSymbolChecks(unittest.TestCase):
         cc = determine_wellknown_cmd('CC', 'gcc')
 
         # there's no way to do this test for RISC-V at the moment; we build for
-        # RISC-V in a glibc 2.27 envinonment and we allow all symbols from 2.27.
+        # RISC-V in a glibc 2.27 environment and we allow all symbols from 2.27.
         if 'riscv' in get_machine(cc):
             self.skipTest("test not available for RISC-V")
 
@@ -187,7 +187,7 @@ class TestSymbolChecks(unittest.TestCase):
         executable = 'test3.exe'
         with open(source, 'w', encoding="utf8") as f:
             f.write('''
-                #include <windows.h>
+                #include <combaseapi.h>
 
                 int main()
                 {

contrib/guix/INSTALL.md

@@ -72,11 +72,11 @@ writing (July 2021). Guix is expected to be more widely packaged over time. For
 an up-to-date view on Guix's package status/version across distros, please see:
 https://repology.org/project/guix/versions
 
-### Debian 11 (Bullseye)/Ubuntu 21.04 (Hirsute Hippo)
+### Debian / Ubuntu
 
 Guix v1.2.0 is available as a distribution package starting in [Debian
 11](https://packages.debian.org/bullseye/guix) and [Ubuntu
-21.04](https://packages.ubuntu.com/hirsute/guix).
+21.04](https://packages.ubuntu.com/search?keywords=guix).
 
 Note that if you intend on using Guix without using any substitutes (more
 details [here][security-model]), v1.2.0 has a known problem when building GnuTLS

contrib/guix/README.md

@@ -382,7 +382,7 @@ https://ci.guix.gnu.org is automatically used unless the `--no-substitutes` flag
 is supplied. This default list of substitute servers is overridable both on a
 `guix-daemon` level and when you invoke `guix` commands. See examples below for
 the various ways of adding dongcarl's substitute server after having [authorized
-his signing key](#authorize-the-signing-keys).
+his signing key](#step-1-authorize-the-signing-keys).
 
 Change the **default list** of substitute servers by starting `guix-daemon` with
 the `--substitute-urls` option (you will likely need to edit your init script):

contrib/guix/guix-attest

@@ -19,8 +19,16 @@ source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash"
 ################
 
 check_tools cat env basename mkdir diff sort
+
 if [ -z "$NO_SIGN" ]; then
-    check_tools gpg
+    # make it possible to override the gpg binary
+    GPG=${GPG:-gpg}
+
+    # $GPG can contain extra arguments passed to the binary
+    # so let's check only the existence of arg[0]
+    # shellcheck disable=SC2206
+    GPG_ARRAY=($GPG)
+    check_tools "${GPG_ARRAY[0]}"
 fi
 
 ################
@@ -90,7 +98,7 @@ if [ -z "${signer_name}" ]; then
     signer_name="$gpg_key_name"
 fi
 
-if [ -z "$NO_SIGN" ] && ! gpg --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then
+if [ -z "$NO_SIGN" ] && ! ${GPG} --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then
     echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'"
     exit 1
 fi
@@ -239,11 +247,11 @@ mkdir -p "$outsigdir"
         echo "Signing SHA256SUMS to produce SHA256SUMS.asc"
         for i in *.SHA256SUMS; do
             if [ ! -e "$i".asc ]; then
-                gpg --detach-sign \
-                    --digest-algo sha256 \
-                    --local-user "$gpg_key_name" \
-                    --armor \
-                    --output "$i".asc "$i"
+                ${GPG} --detach-sign \
+                       --digest-algo sha256 \
+                       --local-user "$gpg_key_name" \
+                       --armor \
+                       --output "$i".asc "$i"
             else
                 echo "Signature already there"
             fi

contrib/guix/guix-build

@@ -121,7 +121,7 @@ else
 fi
 
 ################
-# When building for darwin, the macOS SDK should exists
+# When building for darwin, the macOS SDK should exist
 ################
 
 for host in $HOSTS; do
@@ -130,8 +130,9 @@ for host in $HOSTS; do
             OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')"
             if [ -e "$OSX_SDK" ]; then
                 echo "Found macOS SDK at '${OSX_SDK}', using..."
+                break
             else
-                echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..."
+                echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..."
                 exit 1
             fi
             ;;
@@ -234,21 +235,6 @@ host_to_commonname() {
 # Determine the reference time used for determinism (overridable by environment)
 SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}"
 
-# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility
-# across time.
-time-machine() {
-    # shellcheck disable=SC2086
-    guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \
-                      --commit=ae03f401381e956c4c41b4cf495cbde964fa43d0 \
-                      --cores="$JOBS" \
-                      --keep-failed \
-                      --fallback \
-                      ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \
-                      ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \
-                      -- "$@"
-}
-
-
 # Precious directories are those which should not be cleaned between successive
 # guix builds
 depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH'

contrib/guix/guix-codesign

@@ -222,20 +222,6 @@ JOBS="${JOBS:-$(nproc)}"
 # Determine the reference time used for determinism (overridable by environment)
 SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}"
 
-# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility
-# across time.
-time-machine() {
-    # shellcheck disable=SC2086
-    guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \
-                      --commit=ae03f401381e956c4c41b4cf495cbde964fa43d0 \
-                      --cores="$JOBS" \
-                      --keep-failed \
-                      --fallback \
-                      ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \
-                      ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \
-                      -- "$@"
-}
-
 # Make sure an output directory exists for our builds
 OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}"
 mkdir -p "$OUTDIR_BASE"

contrib/guix/libexec/build.sh

@@ -79,19 +79,6 @@ prepend_to_search_env_var() {
     export "${1}=${2}${!1:+:}${!1}"
 }
 
-case "$HOST" in
-    *darwin*)
-        # When targeting darwin, zlib is required by native_libdmg-hfsplus.
-        zlib_store_path=$(store_path "zlib")
-        zlib_static_store_path=$(store_path "zlib" static)
-
-        prepend_to_search_env_var LIBRARY_PATH "${zlib_static_store_path}/lib:${zlib_store_path}/lib"
-        prepend_to_search_env_var C_INCLUDE_PATH "${zlib_store_path}/include"
-        prepend_to_search_env_var CPLUS_INCLUDE_PATH "${zlib_store_path}/include"
-        prepend_to_search_env_var OBJC_INCLUDE_PATH "${zlib_store_path}/include"
-        prepend_to_search_env_var OBJCPLUS_INCLUDE_PATH "${zlib_store_path}/include"
-esac
-
 # Set environment variables to point the CROSS toolchain to the right
 # includes/libs for $HOST
 case "$HOST" in
@@ -167,7 +154,6 @@ case "$HOST" in
     *linux*)
         glibc_dynamic_linker=$(
             case "$HOST" in
-                i686-linux-gnu)        echo /lib/ld-linux.so.2 ;;
                 x86_64-linux-gnu)      echo /lib64/ld-linux-x86-64.so.2 ;;
                 arm-linux-gnueabihf)   echo /lib/ld-linux-armhf.so.3 ;;
                 aarch64-linux-gnu)     echo /lib/ld-linux-aarch64.so.1 ;;
@@ -204,20 +190,12 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \
                                    ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \
                                    ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \
                                    ${SDK_PATH+SDK_PATH="$SDK_PATH"} \
-                                   i686_linux_CC=i686-linux-gnu-gcc \
-                                   i686_linux_CXX=i686-linux-gnu-g++ \
-                                   i686_linux_AR=i686-linux-gnu-ar \
-                                   i686_linux_RANLIB=i686-linux-gnu-ranlib \
-                                   i686_linux_NM=i686-linux-gnu-nm \
-                                   i686_linux_STRIP=i686-linux-gnu-strip \
                                    x86_64_linux_CC=x86_64-linux-gnu-gcc \
                                    x86_64_linux_CXX=x86_64-linux-gnu-g++ \
                                    x86_64_linux_AR=x86_64-linux-gnu-ar \
                                    x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \
                                    x86_64_linux_NM=x86_64-linux-gnu-nm \
                                    x86_64_linux_STRIP=x86_64-linux-gnu-strip \
-                                   qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \
-                                   qt_config_opts_x86_64_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \
                                    FORCE_USE_SYSTEM_CLANG=1
 
 
@@ -244,6 +222,7 @@ CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disab
 
 # CFLAGS
 HOST_CFLAGS="-O2 -g"
+HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;)
 case "$HOST" in
     *linux*)  HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;;
     *mingw*)  HOST_CFLAGS+=" -fno-ident" ;;
@@ -270,10 +249,6 @@ case "$HOST" in
     *powerpc64*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,--no-tls-get-addr-optimize" ;;
 esac
 
-case "$HOST" in
-    powerpc64-linux-*|riscv64-linux-*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,-z,noexecstack" ;;
-esac
-
 # Make $HOST-specific native binaries from depends available in $PATH
 export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}"
 mkdir -p "$DISTSRC"
@@ -340,8 +315,7 @@ mkdir -p "$DISTSRC"
             mkdir -p "unsigned-app-${HOST}"
             cp  --target-directory="unsigned-app-${HOST}" \
                 osx_volname \
-                contrib/macdeploy/detached-sig-{apply,create}.sh \
-                "${BASEPREFIX}/${HOST}"/native/bin/dmg
+                contrib/macdeploy/detached-sig-create.sh
             mv --target-directory="unsigned-app-${HOST}" dist
             (
                 cd "unsigned-app-${HOST}"
@@ -390,6 +364,12 @@ mkdir -p "$DISTSRC"
                 ;;
         esac
 
+        # copy over the example bitcoin.conf file. if contrib/devtools/gen-bitcoin-conf.sh
+        # has not been run before buildling, this file will be a stub
+        cp "${DISTSRC}/share/examples/bitcoin.conf" "${DISTNAME}/"
+
+        cp -r "${DISTSRC}/share/rpcauth" "${DISTNAME}/share/"
+
         # Finally, deterministically produce {non-,}debug binary tarballs ready
         # for release
         case "$HOST" in

contrib/guix/libexec/codesign.sh

@@ -84,14 +84,11 @@ mkdir -p "$DISTSRC"
             # Apply detached codesignatures to dist/ (in-place)
             signapple apply dist/Bitcoin-Qt.app codesignatures/osx/dist
 
-            # Make an uncompressed DMG from dist/
+            # Make a DMG from dist/
             xorrisofs -D -l -V "$(< osx_volname)" -no-pad -r -dir-mode 0755 \
-                      -o uncompressed.dmg \
+                      -o "${OUTDIR}/${DISTNAME}-${HOST}.dmg" \
                       dist \
                       -- -volume_date all_file_dates ="$SOURCE_DATE_EPOCH"
-
-            # Compress uncompressed.dmg and output to OUTDIR
-            ./dmg dmg uncompressed.dmg "${OUTDIR}/${DISTNAME}-${HOST}.dmg"
             ;;
         *)
             exit 1

contrib/guix/libexec/prelude.bash

@@ -45,6 +45,22 @@ EOF
 exit 1
 fi
 
+################
+# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility
+# across time.
+time-machine() {
+    # shellcheck disable=SC2086
+    guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \
+                      --commit=998eda3067c7d21e0d9bb3310d2f5a14b8f1c681 \
+                      --cores="$JOBS" \
+                      --keep-failed \
+                      --fallback \
+                      ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \
+                      ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \
+                      -- "$@"
+}
+
+
 ################
 # Set common variables
 ################

contrib/guix/manifest.scm

@@ -21,7 +21,6 @@
              (gnu packages llvm)
              (gnu packages mingw)
              (gnu packages moreutils)
-             (gnu packages perl)
              (gnu packages pkg-config)
              (gnu packages python)
              (gnu packages python-crypto)
@@ -135,27 +134,21 @@ chain for " target " development."))
       (license (package-license xgcc)))))
 
 (define base-gcc gcc-10)
+(define base-linux-kernel-headers linux-libre-headers-5.15)
 
-;; Building glibc with stack smashing protector first landed in glibc 2.25, use
-;; this function to disable for older glibcs
-;;
-;; From glibc 2.25 changelog:
-;;
-;;   * Most of glibc can now be built with the stack smashing protector enabled.
-;;     It is recommended to build glibc with --enable-stack-protector=strong.
-;;     Implemented by Nick Alcock (Oracle).
-(define (make-glibc-without-ssp xglibc)
-  (package-with-extra-configure-variable
-   (package-with-extra-configure-variable
-    xglibc "libc_cv_ssp" "no")
-   "libc_cv_ssp_strong" "no"))
+;; https://gcc.gnu.org/install/configure.html
+(define (hardened-gcc gcc)
+  (package-with-extra-configure-variable (
+    package-with-extra-configure-variable gcc
+    "--enable-default-ssp" "yes")
+    "--enable-default-pie" "yes"))
 
 (define* (make-bitcoin-cross-toolchain target
                                        #:key
-                                       (base-gcc-for-libc gcc-7)
-                                       (base-kernel-headers linux-libre-headers-4.9)
-                                       (base-libc (make-glibc-without-ssp glibc-2.24))
-                                       (base-gcc (make-gcc-rpath-link base-gcc)))
+                                       (base-gcc-for-libc base-gcc)
+                                       (base-kernel-headers base-linux-kernel-headers)
+                                       (base-libc (make-glibc-with-bind-now (make-glibc-without-werror glibc-2.24)))
+                                       (base-gcc (make-gcc-rpath-link (hardened-gcc base-gcc))))
   "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values
 desirable for building Bitcoin Core release binaries."
   (make-cross-toolchain target
@@ -165,11 +158,10 @@ desirable for building Bitcoin Core release binaries."
                         base-gcc))
 
 (define (make-gcc-with-pthreads gcc)
-  (package-with-extra-configure-variable gcc "--enable-threads" "posix"))
-
-;; Required to support std::filesystem for mingw-w64 target.
-(define (make-gcc-without-newlib gcc)
-  (package-with-extra-configure-variable gcc "--with-newlib" "no"))
+  (package-with-extra-configure-variable
+    (package-with-extra-patches gcc
+      (search-our-patches "gcc-10-remap-guix-store.patch"))
+    "--enable-threads" "posix"))
 
 (define (make-mingw-w64-cross-gcc cross-gcc)
   (package-with-extra-patches cross-gcc
@@ -182,7 +174,7 @@ desirable for building Bitcoin Core release binaries."
          (pthreads-xlibc mingw-w64-x86_64-winpthreads)
          (pthreads-xgcc (make-gcc-with-pthreads
                          (cross-gcc target
-                                    #:xgcc (make-gcc-without-newlib (make-ssp-fixed-gcc (make-mingw-w64-cross-gcc base-gcc)))
+                                    #:xgcc (make-ssp-fixed-gcc (make-mingw-w64-cross-gcc base-gcc))
                                     #:xbinutils xbinutils
                                     #:libc pthreads-xlibc))))
     ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and
@@ -209,10 +201,14 @@ chain for " target " development."))
     (search-our-patches "nsis-gcc-10-memmove.patch"
                         "nsis-disable-installer-reloc.patch")))
 
+(define (fix-ppc64-nx-default lief)
+  (package-with-extra-patches lief
+    (search-our-patches "lief-fix-ppc64-nx-default.patch")))
+
 (define-public lief
   (package
    (name "python-lief")
-   (version "0.11.5")
+   (version "0.12.1")
    (source
     (origin
      (method git-fetch)
@@ -222,8 +218,15 @@ chain for " target " development."))
      (file-name (git-file-name name version))
      (sha256
       (base32
-       "0qahjfg1n0x76ps2mbyljvws1l3qhkqvmxqbahps4qgywl2hbdkj"))))
+       "1xzbh3bxy4rw1yamnx68da1v5s56ay4g081cyamv67256g0qy2i1"))))
    (build-system python-build-system)
+   (arguments
+    `(#:phases
+      (modify-phases %standard-phases
+        (add-after 'unpack 'parallel-jobs
+          ;; build with multiple cores
+          (lambda _
+            (substitute* "setup.py" (("self.parallel if self.parallel else 1") (number->string (parallel-job-count)))))))))
    (native-inputs
     `(("cmake" ,cmake)))
    (home-page "https://github.com/lief-project/LIEF")
@@ -265,7 +268,7 @@ thus should be able to compile on most platforms where these exist.")
     (license license:gpl3+))) ; license is with openssl exception
 
 (define-public python-elfesteem
-  (let ((commit "87bbd79ab7e361004c98cc8601d4e5f029fd8bd5"))
+  (let ((commit "2eb1e5384ff7a220fd1afacd4a0170acff54fe56"))
     (package
       (name "python-elfesteem")
       (version (git-version "0.1" "1" commit))
@@ -278,8 +281,7 @@ thus should be able to compile on most platforms where these exist.")
          (file-name (git-file-name name commit))
          (sha256
           (base32
-           "1nyvjisvyxyxnd0023xjf5846xd03lwawp5pfzr8vrky7wwm5maz"))
-      (patches (search-our-patches "elfsteem-value-error-python-39.patch"))))
+           "07x6p8clh11z8s1n2kdxrqwqm2almgc5qpkcr9ckb6y5ivjdr5r6"))))
       (build-system python-build-system)
       ;; There are no tests, but attempting to run python setup.py test leads to
       ;; PYTHONPATH problems, just disable the test
@@ -363,7 +365,7 @@ thus should be able to compile on most platforms where these exist.")
              #t)))))))
 
 (define-public python-certvalidator
-  (let ((commit "e5bdb4bfcaa09fa0af355eb8867d00dfeecba08c"))
+  (let ((commit "a145bf25eb75a9f014b3e7678826132efbba6213"))
     (package
       (name "python-certvalidator")
       (version (git-version "0.1" "1" commit))
@@ -376,7 +378,7 @@ thus should be able to compile on most platforms where these exist.")
          (file-name (git-file-name name commit))
          (sha256
           (base32
-           "18pvxkvpkfkzgvfylv0kx65pmxfcv1hpsg03cip93krfvrrl4c75"))))
+           "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k"))))
       (build-system python-build-system)
       (propagated-inputs
        `(("python-asn1crypto" ,python-asn1crypto)
@@ -522,8 +524,7 @@ and endian independent.")
          ("python-certvalidator" ,python-certvalidator)
          ("python-elfesteem" ,python-elfesteem)
          ("python-requests" ,python-requests)
-         ("python-macholib" ,python-macholib)
-         ("libcrypto" ,openssl)))
+         ("python-macholib" ,python-macholib)))
       ;; There are no tests, but attempting to run python setup.py test leads to
       ;; problems, just disable the test
       (arguments '(#:tests? #f))
@@ -533,6 +534,15 @@ and endian independent.")
 inspecting signatures in Mach-O binaries.")
       (license license:expat))))
 
+(define (make-glibc-without-werror glibc)
+  (package-with-extra-configure-variable glibc "enable_werror" "no"))
+
+(define (make-glibc-with-stack-protector glibc)
+  (package-with-extra-configure-variable glibc "--enable-stack-protector" "all"))
+
+(define (make-glibc-with-bind-now glibc)
+  (package-with-extra-configure-variable glibc "--enable-bind-now" "yes"))
+
 (define-public glibc-2.24
   (package
     (inherit glibc-2.31)
@@ -549,7 +559,9 @@ inspecting signatures in Mach-O binaries.")
               (patches (search-our-patches "glibc-ldd-x86_64.patch"
                                            "glibc-versioned-locpath.patch"
                                            "glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch"
-                                           "glibc-2.24-no-build-time-cxx-header-run.patch"))))))
+                                           "glibc-2.24-no-build-time-cxx-header-run.patch"
+                                           "glibc-2.24-fcommon.patch"
+                                           "glibc-2.24-guix-prefix.patch"))))))
 
 (define-public glibc-2.27/bitcoin-patched
   (package
@@ -565,7 +577,9 @@ inspecting signatures in Mach-O binaries.")
                (base32
                 "1b2n1gxv9f4fd5yy68qjbnarhf8mf4vmlxk10i3328c1w5pmp0ca"))
               (patches (search-our-patches "glibc-ldd-x86_64.patch"
-                                           "glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch"))))))
+                                           "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch"
+                                           "glibc-2.27-dont-redefine-nss-database.patch"
+                                           "glibc-2.27-guix-prefix.patch"))))))
 
 (packages->manifest
  (append
@@ -589,25 +603,22 @@ inspecting signatures in Mach-O binaries.")
         bzip2
         gzip
         xz
-        zlib
-        (list zlib "static")
         ;; Build tools
         gnu-make
-        libtool
+        libtool-2.4.7
         autoconf-2.71
         automake
         pkg-config
         bison
+        ;; Native GCC 10 toolchain
+        gcc-toolchain-10
+        (list gcc-toolchain-10 "static")
         ;; Scripting
-        perl
         python-3
         ;; Git
-        git
+        git-minimal
         ;; Tests
-        lief
-        ;; Native gcc 7 toolchain
-        gcc-toolchain-7
-        (list gcc-toolchain-7 "static"))
+        (fix-ppc64-nx-default lief))
   (let ((target (getenv "HOST")))
     (cond ((string-suffix? "-mingw32" target)
            ;; Windows
@@ -618,8 +629,8 @@ inspecting signatures in Mach-O binaries.")
           ((string-contains target "-linux-")
            (list (cond ((string-contains target "riscv64-")
                         (make-bitcoin-cross-toolchain target
-                                                      #:base-libc glibc-2.27/bitcoin-patched
-                                                      #:base-kernel-headers linux-libre-headers-4.19))
+                                                      #:base-libc (make-glibc-with-stack-protector
+                                                        (make-glibc-with-bind-now (make-glibc-without-werror glibc-2.27/bitcoin-patched)))))
                        (else
                         (make-bitcoin-cross-toolchain target)))))
           ((string-contains target "darwin")

contrib/guix/patches/elfsteem-value-error-python-39.patch

@@ -1,13 +0,0 @@
-diff --git a/examples/otool.py b/examples/otool.py
-index 2b8efc0..d797b2e 100755
---- a/examples/otool.py
-+++ b/examples/otool.py
-@@ -342,7 +342,7 @@ if __name__ == '__main__':
-         try:
-             e = macho_init.MACHO(raw,
-                 parseSymbols = False)
--        except ValueError, err:
-+        except ValueError as err:
-             print("%s:" %file)
-             print("    %s" % err)
-             continue

contrib/guix/patches/gcc-10-remap-guix-store.patch

@@ -0,0 +1,25 @@
+From aad25427e74f387412e8bc9a9d7bbc6c496c792f Mon Sep 17 00:00:00 2001
+From: Andrew Chow <achow101-github@achow101.com>
+Date: Wed, 6 Jul 2022 16:49:41 -0400
+Subject: [PATCH] guix: remap guix store paths to /usr
+
+---
+ libgcc/Makefile.in | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/libgcc/Makefile.in b/libgcc/Makefile.in
+index 851e7657d07..476c2becd1c 100644
+--- a/libgcc/Makefile.in
++++ b/libgcc/Makefile.in
+@@ -854,7 +854,7 @@ endif
+ # libgcc_eh.a, only LIB2ADDEH matters.  If we do, only LIB2ADDEHSTATIC and
+ # LIB2ADDEHSHARED matter.  (Usually all three are identical.)
+ 
+-c_flags := -fexceptions
++c_flags := -fexceptions $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;)
+ 
+ ifeq ($(enable_shared),yes)
+ 
+-- 
+2.37.0
+

contrib/guix/patches/glibc-2.24-fcommon.patch

@@ -0,0 +1,32 @@
+commit 264a4a0dbe1f4369db315080034b500bed66016c
+Author: fanquake <fanquake@gmail.com>
+Date:   Fri May 6 11:03:04 2022 +0100
+
+    build: use -fcommon to retain legacy behaviour with GCC 10
+    
+    GCC 10 started using -fno-common by default, which causes issues with
+    the powerpc builds using gibc 2.24. A patch was commited to glibc to fix
+    the issue, 18363b4f010da9ba459b13310b113ac0647c2fcc but is non-trvial
+    to backport, and was broken in at least one way, see the followup in
+    commit 7650321ce037302bfc2f026aa19e0213b8d02fe6.
+    
+    For now, retain the legacy GCC behaviour by passing -fcommon when
+    building glibc.
+    
+    https://gcc.gnu.org/onlinedocs/gcc/Code-Gen-Options.html.
+    https://sourceware.org/git/?p=glibc.git;a=commit;h=18363b4f010da9ba459b13310b113ac0647c2fcc
+    https://sourceware.org/git/?p=glibc.git;a=commit;h=7650321ce037302bfc2f026aa19e0213b8d02fe6
+
+diff --git a/Makeconfig b/Makeconfig
+index ee379f5852..63c4a2f234 100644
+--- a/Makeconfig
++++ b/Makeconfig
+@@ -824,7 +824,7 @@ ifeq	"$(strip $(+cflags))" ""
+ +cflags	:= $(default_cflags)
+ endif	# $(+cflags) == ""
+ 
+-+cflags += $(cflags-cpu) $(+gccwarn) $(+merge-constants) $(+math-flags)
+++cflags += $(cflags-cpu) $(+gccwarn) $(+merge-constants) $(+math-flags) -fcommon
+ +gcc-nowarn := -w
+ 
+ # Don't duplicate options if we inherited variables from the parent.

contrib/guix/patches/glibc-2.24-guix-prefix.patch

@@ -0,0 +1,25 @@
+Without ffile-prefix-map, the debug symbols will contain paths for the
+guix store which will include the hashes of each package. However, the
+hash for the same package will differ when on different architectures.
+In order to be reproducible regardless of the architecture used to build
+the package, map all guix store prefixes to something fixed, e.g. /usr.
+
+We might be able to drop this in favour of using --with-nonshared-cflags
+when we being using newer versions of glibc.
+
+--- a/Makeconfig
++++ b/Makeconfig
+@@ -950,6 +950,10 @@ object-suffixes-for-libc += .oS
+ # shared objects.  We don't want to use CFLAGS-os because users may, for
+ # example, make that processor-specific.
+ CFLAGS-.oS = $(CFLAGS-.o) $(PIC-ccflag)
++
++# Map Guix store paths to /usr
++CFLAGS-.oS += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;`
++
+ CPPFLAGS-.oS = $(CPPFLAGS-.o) -DPIC -DLIBC_NONSHARED=1
+ libtype.oS = lib%_nonshared.a
+ endif
+-- 
+2.35.1
+

contrib/guix/patches/glibc-2.27-dont-redefine-nss-database.patch

@@ -0,0 +1,87 @@
+commit 78a90c2f74a2012dd3eff302189e47ff6779a757
+Author: Andreas Schwab <schwab@linux-m68k.org>
+Date:   Fri Mar 2 23:07:14 2018 +0100
+
+    Fix multiple definitions of __nss_*_database (bug 22918)
+    
+    (cherry picked from commit eaf6753f8aac33a36deb98c1031d1bad7b593d2d)
+
+diff --git a/nscd/gai.c b/nscd/gai.c
+index d081747797..576fd0045b 100644
+--- a/nscd/gai.c
++++ b/nscd/gai.c
+@@ -45,3 +45,6 @@
+ #ifdef HAVE_LIBIDN
+ # include <libidn/idn-stub.c>
+ #endif
++
++/* Some variables normally defined in libc.  */
++service_user *__nss_hosts_database attribute_hidden;
+diff --git a/nss/nsswitch.c b/nss/nsswitch.c
+index d5e655974f..b0f0c11a3e 100644
+--- a/nss/nsswitch.c
++++ b/nss/nsswitch.c
+@@ -62,7 +62,7 @@ static service_library *nss_new_service (name_database *database,
+ 
+ /* Declare external database variables.  */
+ #define DEFINE_DATABASE(name)						      \
+-  extern service_user *__nss_##name##_database attribute_hidden;	      \
++  service_user *__nss_##name##_database attribute_hidden;		      \
+   weak_extern (__nss_##name##_database)
+ #include "databases.def"
+ #undef DEFINE_DATABASE
+diff --git a/nss/nsswitch.h b/nss/nsswitch.h
+index eccb535ef5..63573b9ebc 100644
+--- a/nss/nsswitch.h
++++ b/nss/nsswitch.h
+@@ -226,10 +226,10 @@ libc_hidden_proto (__nss_hostname_digits_dots)
+ #define MAX_NR_ADDRS    48
+ 
+ /* Prototypes for __nss_*_lookup2 functions.  */
+-#define DEFINE_DATABASE(arg)				    \
+-  service_user *__nss_##arg##_database attribute_hidden;    \
+-  int __nss_##arg##_lookup2 (service_user **, const char *, \
+-			     const char *, void **);	    \
++#define DEFINE_DATABASE(arg)						      \
++  extern service_user *__nss_##arg##_database attribute_hidden;		      \
++  int __nss_##arg##_lookup2 (service_user **, const char *,		      \
++			     const char *, void **);			      \
+   libc_hidden_proto (__nss_##arg##_lookup2)
+ #include "databases.def"
+ #undef DEFINE_DATABASE
+diff --git a/posix/tst-rfc3484-2.c b/posix/tst-rfc3484-2.c
+index f509534ca9..8c64ac59ff 100644
+--- a/posix/tst-rfc3484-2.c
++++ b/posix/tst-rfc3484-2.c
+@@ -58,6 +58,7 @@ _res_hconf_init (void)
+ #undef	USE_NSCD
+ #include "../sysdeps/posix/getaddrinfo.c"
+ 
++service_user *__nss_hosts_database attribute_hidden;
+ 
+ /* This is the beginning of the real test code.  The above defines
+    (among other things) the function rfc3484_sort.  */
+diff --git a/posix/tst-rfc3484-3.c b/posix/tst-rfc3484-3.c
+index ae44087a10..1c61aaf844 100644
+--- a/posix/tst-rfc3484-3.c
++++ b/posix/tst-rfc3484-3.c
+@@ -58,6 +58,7 @@ _res_hconf_init (void)
+ #undef	USE_NSCD
+ #include "../sysdeps/posix/getaddrinfo.c"
+ 
++service_user *__nss_hosts_database attribute_hidden;
+ 
+ /* This is the beginning of the real test code.  The above defines
+    (among other things) the function rfc3484_sort.  */
+diff --git a/posix/tst-rfc3484.c b/posix/tst-rfc3484.c
+index 7f191abbbc..8f45848e44 100644
+--- a/posix/tst-rfc3484.c
++++ b/posix/tst-rfc3484.c
+@@ -58,6 +58,7 @@ _res_hconf_init (void)
+ #undef	USE_NSCD
+ #include "../sysdeps/posix/getaddrinfo.c"
+ 
++service_user *__nss_hosts_database attribute_hidden;
+ 
+ /* This is the beginning of the real test code.  The above defines
+    (among other things) the function rfc3484_sort.  */

contrib/guix/patches/glibc-2.27-guix-prefix.patch

@@ -0,0 +1,25 @@
+Without ffile-prefix-map, the debug symbols will contain paths for the
+guix store which will include the hashes of each package. However, the
+hash for the same package will differ when on different architectures.
+In order to be reproducible regardless of the architecture used to build
+the package, map all guix store prefixes to something fixed, e.g. /usr.
+
+We might be able to drop this in favour of using --with-nonshared-cflags
+when we being using newer versions of glibc.
+
+--- a/Makeconfig
++++ b/Makeconfig
+@@ -992,6 +992,10 @@ object-suffixes :=
+ CPPFLAGS-.o = $(pic-default)
+ # libc.a must be compiled with -fPIE/-fpie for static PIE.
+ CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default)
++
++# Map Guix store paths to /usr
++CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;`
++
+ libtype.o := lib%.a
+ object-suffixes += .o
+ ifeq (yes,$(build-shared))
+-- 
+2.35.1
+

contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch

@@ -1,3 +1,7 @@
+Note that this has been modified from the original commit, to use __has_include
+instead of __has_include__, as the later was causing build failures with GCC 10.
+See also: http://lists.busybox.net/pipermail/buildroot/2020-July/590376.html.
+
 https://sourceware.org/git/?p=glibc.git;a=commit;h=0b9c84906f653978fb8768c7ebd0ee14a47e662e
 
 From 562c52cc81a4e456a62e6455feb32732049e9070 Mon Sep 17 00:00:00 2001
@@ -59,7 +63,7 @@ index d612ef4c6c..0b2042620b 100644
  #include <atomic.h>
  #include <sys/cachectl.h>
 -#include <asm/syscalls.h>
-+#if __has_include__ (<asm/syscalls.h>)
++#if __has_include (<asm/syscalls.h>)
 +# include <asm/syscalls.h>
 +#else
 +# include <asm/unistd.h>

contrib/guix/patches/lief-fix-ppc64-nx-default.patch

@@ -0,0 +1,29 @@
+Correct default for Binary::has_nx on ppc64
+
+From the Linux kernel source:
+
+    * This is the default if a program doesn't have a PT_GNU_STACK
+    * program header entry. The PPC64 ELF ABI has a non executable stack
+    * stack by default, so in the absence of a PT_GNU_STACK program header
+    * we turn execute permission off.
+
+This patch can be dropped the next time we update LIEF.
+
+diff --git a/src/ELF/Binary.cpp b/src/ELF/Binary.cpp
+index a90be1ab..fd2d9764 100644
+--- a/src/ELF/Binary.cpp
++++ b/src/ELF/Binary.cpp
+@@ -1084,7 +1084,12 @@ bool Binary::has_nx() const {
+                                        return segment->type() == SEGMENT_TYPES::PT_GNU_STACK;
+                                      });
+   if (it_stack == std::end(segments_)) {
+-    return false;
++    if (header().machine_type() == ARCH::EM_PPC64) {
++      // The PPC64 ELF ABI has a non-executable stack by default.
++      return true;
++    } else {
++      return false;
++    }
+   }
+ 
+   return !(*it_stack)->has(ELF_SEGMENT_FLAGS::PF_X);

contrib/linearize/linearize-data.py

@@ -20,49 +20,9 @@ from collections import namedtuple
 
 settings = {}
 
-def hex_switchEndian(s):
-    """ Switches the endianness of a hex string (in pairs of hex chars) """
-    pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
-    return b''.join(pairList[::-1]).decode()
-
-def uint32(x):
-    return x & 0xffffffff
-
-def bytereverse(x):
-    return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) |
-               (((x) >> 8) & 0x0000ff00) | ((x) >> 24) ))
-
-def bufreverse(in_buf):
-    out_words = []
-    for i in range(0, len(in_buf), 4):
-        word = struct.unpack('@I', in_buf[i:i+4])[0]
-        out_words.append(struct.pack('@I', bytereverse(word)))
-    return b''.join(out_words)
-
-def wordreverse(in_buf):
-    out_words = []
-    for i in range(0, len(in_buf), 4):
-        out_words.append(in_buf[i:i+4])
-    out_words.reverse()
-    return b''.join(out_words)
-
-def calc_hdr_hash(blk_hdr):
-    hash1 = hashlib.sha256()
-    hash1.update(blk_hdr)
-    hash1_o = hash1.digest()
-
-    hash2 = hashlib.sha256()
-    hash2.update(hash1_o)
-    hash2_o = hash2.digest()
-
-    return hash2_o
-
 def calc_hash_str(blk_hdr):
-    hash = calc_hdr_hash(blk_hdr)
-    hash = bufreverse(hash)
-    hash = wordreverse(hash)
-    hash_str = hash.hex()
-    return hash_str
+    blk_hdr_hash = hashlib.sha256(hashlib.sha256(blk_hdr).digest()).digest()
+    return blk_hdr_hash[::-1].hex()
 
 def get_blk_dt(blk_hdr):
     members = struct.unpack("<I", blk_hdr[68:68+4])
@@ -74,12 +34,12 @@ def get_blk_dt(blk_hdr):
 # When getting the list of block hashes, undo any byte reversals.
 def get_block_hashes(settings):
     blkindex = []
-    f = open(settings['hashlist'], "r", encoding="utf8")
-    for line in f:
-        line = line.rstrip()
-        if settings['rev_hash_bytes'] == 'true':
-            line = hex_switchEndian(line)
-        blkindex.append(line)
+    with open(settings['hashlist'], "r", encoding="utf8") as f:
+        for line in f:
+            line = line.rstrip()
+            if settings['rev_hash_bytes'] == 'true':
+                line = bytes.fromhex(line)[::-1].hex()
+            blkindex.append(line)
 
     print("Read " + str(len(blkindex)) + " hashes")
 
@@ -289,19 +249,18 @@ if __name__ == '__main__':
         print("Usage: linearize-data.py CONFIG-FILE")
         sys.exit(1)
 
-    f = open(sys.argv[1], encoding="utf8")
-    for line in f:
-        # skip comment lines
-        m = re.search(r'^\s*#', line)
-        if m:
-            continue
+    with open(sys.argv[1], encoding="utf8") as f:
+        for line in f:
+            # skip comment lines
+            m = re.search(r'^\s*#', line)
+            if m:
+                continue
 
-        # parse key=value lines
-        m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line)
-        if m is None:
-            continue
-        settings[m.group(1)] = m.group(2)
-    f.close()
+            # parse key=value lines
+            m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line)
+            if m is None:
+                continue
+            settings[m.group(1)] = m.group(2)
 
     # Force hash byte format setting to be lowercase to make comparisons easier.
     # Also place upfront in case any settings need to know about it.

contrib/linearize/linearize-hashes.py

@@ -17,11 +17,6 @@ import os.path
 
 settings = {}
 
-def hex_switchEndian(s):
-    """ Switches the endianness of a hex string (in pairs of hex chars) """
-    pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
-    return b''.join(pairList[::-1]).decode()
-
 class BitcoinRPC:
     def __init__(self, host, port, username, password):
         authpair = "%s:%s" % (username, password)
@@ -85,7 +80,7 @@ def get_block_hashes(settings, max_blocks_per_call=10000):
                 sys.exit(1)
             assert(resp_obj['id'] == x) # assume replies are in-sequence
             if settings['rev_hash_bytes'] == 'true':
-                resp_obj['result'] = hex_switchEndian(resp_obj['result'])
+                resp_obj['result'] = bytes.fromhex(resp_obj['result'])[::-1].hex()
             print(resp_obj['result'])
 
         height += num_blocks
@@ -103,19 +98,18 @@ if __name__ == '__main__':
         print("Usage: linearize-hashes.py CONFIG-FILE")
         sys.exit(1)
 
-    f = open(sys.argv[1], encoding="utf8")
-    for line in f:
-        # skip comment lines
-        m = re.search(r'^\s*#', line)
-        if m:
-            continue
+    with open(sys.argv[1], encoding="utf8") as f:
+        for line in f:
+            # skip comment lines
+            m = re.search(r'^\s*#', line)
+            if m:
+                continue
 
-        # parse key=value lines
-        m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line)
-        if m is None:
-            continue
-        settings[m.group(1)] = m.group(2)
-    f.close()
+            # parse key=value lines
+            m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line)
+            if m is None:
+                continue
+            settings[m.group(1)] = m.group(2)
 
     if 'host' not in settings:
         settings['host'] = '127.0.0.1'

contrib/macdeploy/README.md

@@ -15,13 +15,16 @@ When complete, it will have produced `Bitcoin-Core.dmg`.
 A free Apple Developer Account is required to proceed.
 
 Our current macOS SDK
-(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`) can be
-extracted from
+(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`)
+can be extracted from
 [Xcode_12.2.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip).
+
 Alternatively, after logging in to your account go to 'Downloads', then 'More'
-and search for [`Xcode_12.2`](https://developer.apple.com/download/all/?q=Xcode%2012.2).
+and search for [`Xcode 12.2`](https://developer.apple.com/download/all/?q=Xcode%2012.2).
+
 An Apple ID and cookies enabled for the hostname are needed to download this.
-The `sha256sum` of the archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`.
+
+The `sha256sum` of the downloaded XIP archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`.
 
 After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip`
 archive. This makes the SDK less-trivial to extract on non-macOS machines. One
@@ -55,7 +58,10 @@ previous stage) as the first argument.
 ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app'
 ```
 
+The `sha256sum` of the generated TAR.GZ archive should be `df75d30ecafc429e905134333aeae56ac65fac67cb4182622398fd717df77619`.
+
 ## Deterministic macOS DMG Notes
+
 Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple
 `binutils` (`ld`, `ar`, etc) and DMG authoring tools.
 
@@ -89,16 +95,7 @@ redistributed.
 
 [`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG.
 
-`xorrisofs` cannot compress DMGs, so afterwards, the DMG tool from the
-`libdmg-hfsplus` project is used to compress it. There are several bugs in this
-tool and its maintainer has seemingly abandoned the project.
-
-The DMG tool has the ability to create DMGs from scratch as well, but this functionality is
-broken. Only the compression feature is currently used. Ideally, the creation could be fixed
-and `xorrisofs` would no longer be necessary.
-
-Background images and other features can be added to DMG files by inserting a
-`.DS_Store` during creation.
+A background image is added to DMG files by inserting a `.DS_Store` during creation.
 
 As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in
 order to satisfy the new Gatekeeper requirements. Because this private key cannot be

contrib/macdeploy/detached-sig-apply.sh

@@ -1,27 +0,0 @@
-#!/bin/sh
-# Copyright (c) 2014-2021 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C
-set -e
-
-UNSIGNED="$1"
-SIGNATURE="$2"
-ROOTDIR=dist
-OUTDIR=signed-app
-SIGNAPPLE=signapple
-
-if [ -z "$UNSIGNED" ]; then
-  echo "usage: $0 <unsigned app> <signature>"
-  exit 1
-fi
-
-if [ -z "$SIGNATURE" ]; then
-  echo "usage: $0 <unsigned app> <signature>"
-  exit 1
-fi
-
-${SIGNAPPLE} apply "${UNSIGNED}" "${SIGNATURE}"
-mv ${ROOTDIR} ${OUTDIR}
-echo "Signed: ${OUTDIR}"

contrib/macdeploy/gen-sdk

@@ -8,6 +8,21 @@ import gzip
 import os
 import contextlib
 
+# monkey-patch Python 3.8 and older to fix wrong TAR header handling
+# see https://github.com/bitcoin/bitcoin/pull/24534
+# and https://github.com/python/cpython/pull/18080 for more info
+if sys.version_info < (3, 9):
+    _old_create_header = tarfile.TarInfo._create_header
+    def _create_header(info, format, encoding, errors):
+        buf = _old_create_header(info, format, encoding, errors)
+        # replace devmajor/devminor with binary zeroes
+        buf = buf[:329] + bytes(16) + buf[345:]
+        # recompute checksum
+        chksum = tarfile.calc_chksums(buf)[0]
+        buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:]
+        return buf
+    tarfile.TarInfo._create_header = staticmethod(_create_header)
+
 @contextlib.contextmanager
 def cd(path):
     """Context manager that restores PWD even if an exception was raised."""
@@ -75,14 +90,21 @@ def run():
                 tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name))
             if tarinfo.linkname and tarinfo.linkname.startswith("./"):
                 tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname))
+            # make metadata deterministic
+            tarinfo.mtime = 0
+            tarinfo.uid, tarinfo.uname =  0, ''
+            tarinfo.gid, tarinfo.gname = 0, ''
+            # don't use isdir() as there are also executable files present
+            tarinfo.mode = 0o0755 if tarinfo.mode & 0o0100 else 0o0644
             return tarinfo
         with cd(dir_to_add):
+            # recursion already adds entries in sorted order
             tarfp.add(".", recursive=True, filter=change_tarinfo_base)
 
     print("Creating output .tar.gz file...")
     with out_sdktgz_path.open("wb") as fp:
         with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf:
-            with tarfile.open(mode="w", fileobj=gzf) as tarfp:
+            with tarfile.open(mode="w", fileobj=gzf, format=tarfile.GNU_FORMAT) as tarfp:
                 print("Adding MacOSX SDK {} files...".format(sdk_version))
                 tarfp_add_with_base_change(tarfp, sdk_dir, out_name)
                 print("Adding libc++ headers...")

contrib/macdeploy/macdeployqtplus

@@ -211,7 +211,7 @@ def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]:
     return libraries
 
 def runInstallNameTool(action: str, *args):
-    installnametoolbin=os.getenv("INSTALLNAMETOOL", "install_name_tool")
+    installnametoolbin=os.getenv("INSTALL_NAME_TOOL", "install_name_tool")
     run([installnametoolbin, "-"+action] + list(args), check=True)
 
 def changeInstallName(oldName: str, newName: str, binaryPath: str, verbose: int):
@@ -544,6 +544,22 @@ ds.close()
 if platform.system() == "Darwin":
     subprocess.check_call(f"codesign --deep --force --sign - {target}", shell=True)
 
+print("+ Installing background.tiff +")
+
+bg_path = os.path.join('dist', '.background', 'background.tiff')
+os.mkdir(os.path.dirname(bg_path))
+
+tiff_path = os.path.join('contrib', 'macdeploy', 'background.tiff')
+shutil.copy2(tiff_path, bg_path)
+
+# ------------------------------------------------
+
+print("+ Generating symlink for /Applications +")
+
+os.symlink("/Applications", os.path.join('dist', "Applications"))
+
+# ------------------------------------------------
+
 if config.dmg is not None:
 
     print("+ Preparing .dmg disk image +")
@@ -567,19 +583,6 @@ if config.dmg is not None:
         print("Attaching temp image...")
     output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, universal_newlines=True, stdout=PIPE).stdout
 
-    m = re.search(r"/Volumes/(.+$)", output)
-    disk_root = m.group(0)
-
-    print("+ Applying fancy settings +")
-
-    bg_path = os.path.join(disk_root, ".background", os.path.basename('background.tiff'))
-    os.mkdir(os.path.dirname(bg_path))
-    if verbose:
-        print('background.tiff', "->", bg_path)
-    shutil.copy2('contrib/macdeploy/background.tiff', bg_path)
-
-    os.symlink("/Applications", os.path.join(disk_root, "Applications"))
-
     print("+ Finalizing .dmg disk image +")
 
     run(["hdiutil", "detach", f"/Volumes/{appname}"], universal_newlines=True)

contrib/message-capture/message-capture-parser.py

@@ -79,7 +79,7 @@ def to_jsonable(obj: Any) -> Any:
             val = getattr(obj, slot, None)
             if slot in HASH_INTS and isinstance(val, int):
                 ret[slot] = ser_uint256(val).hex()
-            elif slot in HASH_INT_VECTORS and isinstance(val[0], int):
+            elif slot in HASH_INT_VECTORS and all(isinstance(a, int) for a in val):
                 ret[slot] = [ser_uint256(a).hex() for a in val]
             else:
                 ret[slot] = to_jsonable(val)

contrib/seeds/.gitignore

@@ -1 +1,2 @@
 seeds_main.txt
+asmap-filled.dat

contrib/seeds/README.md

@@ -8,21 +8,11 @@ and remove old versions as necessary (at a minimum when GetDesirableServiceFlags
 changes its default return value, as those are the services which seeds are added
 to addrman with).
 
-The seeds compiled into the release are created from sipa's DNS seed data, like this:
+The seeds compiled into the release are created from sipa's DNS seed and AS map
+data. Run the following commands from the `/contrib/seeds` directory:
 
     curl https://bitcoin.sipa.be/seeds.txt.gz | gzip -dc > seeds_main.txt
-    python3 makeseeds.py < seeds_main.txt > nodes_main.txt
+    curl https://bitcoin.sipa.be/asmap-filled.dat > asmap-filled.dat
+    python3 makeseeds.py -a asmap-filled.dat < seeds_main.txt > nodes_main.txt
     cat nodes_main_manual.txt >> nodes_main.txt
     python3 generate-seeds.py . > ../../src/chainparamsseeds.h
-
-## Dependencies
-
-Ubuntu, Debian:
-
-    sudo apt-get install python3-dnspython
-
-and/or for other operating systems:
-
-    pip install dnspython
-
-See https://dnspython.readthedocs.io/en/latest/installation.html for more information.

contrib/seeds/asmap.py

@@ -0,0 +1,815 @@
+# Copyright (c) 2022 Pieter Wuille
+# Distributed under the MIT software license, see the accompanying
+# file LICENSE or http://www.opensource.org/licenses/mit-license.php.
+
+"""
+This module provides the ASNEntry and ASMap classes.
+"""
+
+import copy
+import ipaddress
+import random
+import unittest
+from enum import Enum
+from functools import total_ordering
+from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union, overload
+
+def net_to_prefix(net: Union[ipaddress.IPv4Network,ipaddress.IPv6Network]) -> List[bool]:
+    """
+    Convert an IPv4 or IPv6 network to a prefix represented as a list of bits.
+
+    IPv4 ranges are remapped to their IPv4-mapped IPv6 range (::ffff:0:0/96).
+    """
+    num_bits = net.prefixlen
+    netrange = int.from_bytes(net.network_address.packed, 'big')
+
+    # Map an IPv4 prefix into IPv6 space.
+    if isinstance(net, ipaddress.IPv4Network):
+        num_bits += 96
+        netrange += 0xffff00000000
+
+    # Strip unused bottom bits.
+    assert (netrange & ((1 << (128 - num_bits)) - 1)) == 0
+    return [((netrange >> (127 - i)) & 1) != 0 for i in range(num_bits)]
+
+def prefix_to_net(prefix: List[bool]) -> Union[ipaddress.IPv4Network,ipaddress.IPv6Network]:
+    """The reverse operation of net_to_prefix."""
+    # Convert to number
+    netrange = sum(b << (127 - i) for i, b in enumerate(prefix))
+    num_bits = len(prefix)
+    assert num_bits <= 128
+
+    # Return IPv4 range if in ::ffff:0:0/96
+    if num_bits >= 96 and (netrange >> 32) == 0xffff:
+        return ipaddress.IPv4Network((netrange & 0xffffffff, num_bits - 96), True)
+
+    # Return IPv6 range otherwise.
+    return ipaddress.IPv6Network((netrange, num_bits), True)
+
+# Shortcut for (prefix, ASN) entries.
+ASNEntry = Tuple[List[bool], int]
+
+# Shortcut for (prefix, old ASN, new ASN) entries.
+ASNDiff = Tuple[List[bool], int, int]
+
+class _VarLenCoder:
+    """
+    A class representing a custom variable-length binary encoder/decoder for
+    integers. Each object represents a different coder, with different parameters
+    minval and clsbits.
+
+    The encoding is easiest to describe using an example. Let's say minval=100 and
+    clsbits=[4,2,2,3]. In that case:
+    - x in [100..115]: encoded as [0] + [4-bit BE encoding of (x-100)].
+    - x in [116..119]: encoded as [1,0] + [2-bit BE encoding of (x-116)].
+    - x in [120..123]: encoded as [1,1,0] + [2-bit BE encoding of (x-120)].
+    - x in [124..131]: encoded as [1,1,1] + [3-bit BE encoding of (x-124)].
+
+    In general, every number is encoded as:
+    - First, k "1"-bits, where k is the class the number falls in (there is one class
+      per element of clsbits).
+    - Then, a "0"-bit, unless k is the highest class, in which case there is nothing.
+    - Lastly, clsbits[k] bits encoding in big endian the position in its class that
+      number falls into.
+    - Every class k consists of 2^clsbits[k] consecutive integers. k=0 starts at minval,
+      other classes start one past the last element of the class before it.
+    """
+
+    def __init__(self, minval: int, clsbits: List[int]):
+        """Construct a new _VarLenCoder."""
+        self._minval = minval
+        self._clsbits = clsbits
+        self._maxval = minval + sum(1 << b for b in clsbits) - 1
+
+    def can_encode(self, val: int) -> bool:
+        """Check whether value val is in the range this coder supports."""
+        return self._minval <= val <= self._maxval
+
+    def encode(self, val: int, ret: List[int]) -> None:
+        """Append encoding of val onto integer list ret."""
+
+        assert self._minval <= val <= self._maxval
+        val -= self._minval
+        bits = 0
+        for k, bits in enumerate(self._clsbits):
+            if val >> bits:
+                # If the value will not fit in class k, subtract its range from v,
+                # emit a "1" bit and continue with the next class.
+                val -= 1 << bits
+                ret.append(1)
+            else:
+                if k + 1 < len(self._clsbits):
+                    # Unless we're in the last class, emit a "0" bit.
+                    ret.append(0)
+                break
+        # And then encode v (now the position within the class) in big endian.
+        ret.extend((val >> (bits - 1 - b)) & 1 for b in range(bits))
+
+    def encode_size(self, val: int) -> int:
+        """Compute how many bits are needed to encode val."""
+        assert self._minval <= val <= self._maxval
+        val -= self._minval
+        ret = 0
+        bits = 0
+        for k, bits in enumerate(self._clsbits):
+            if val >> bits:
+                val -= 1 << bits
+                ret += 1
+            else:
+                ret += k + 1 < len(self._clsbits)
+                break
+        return ret + bits
+
+    def decode(self, stream, bitpos) -> Tuple[int,int]:
+        """Decode a number starting at bitpos in stream, returning value and new bitpos."""
+        val = self._minval
+        bits = 0
+        for k, bits in enumerate(self._clsbits):
+            bit = 0
+            if k + 1 < len(self._clsbits):
+                bit = stream[bitpos]
+                bitpos += 1
+            if not bit:
+                break
+            val += 1 << bits
+        for i in range(bits):
+            bit = stream[bitpos]
+            bitpos += 1
+            val += bit << (bits - 1 - i)
+        return val, bitpos
+
+# Variable-length encoders used in the binary asmap format.
+_CODER_INS = _VarLenCoder(0, [0, 0, 1])
+_CODER_ASN = _VarLenCoder(1, list(range(15, 25)))
+_CODER_MATCH = _VarLenCoder(2, list(range(1, 9)))
+_CODER_JUMP = _VarLenCoder(17, list(range(5, 31)))
+
+class _Instruction(Enum):
+    """One instruction in the binary asmap format."""
+    # A return instruction, encoded as [0], returns a constant ASN. It is followed by
+    # an integer using the ASN encoding.
+    RETURN = 0
+    # A jump instruction, encoded as [1,0] inspects the next unused bit in the input
+    # and either continues execution (if 0), or skips a specified number of bits (if 1).
+    # It is followed by an integer, and then two subprograms. The integer uses jump encoding
+    # and corresponds to the length of the first subprogram (so it can be skipped).
+    JUMP = 1
+    # A match instruction, encoded as [1,1,0] inspects 1 or more of the next unused bits
+    # in the input with its argument. If they all match, execution continues. If they do
+    # not, failure is returned. If a default instruction has been executed before, instead
+    # of failure the default instruction's argument is returned. It is followed by an
+    # integer in match encoding, and a subprogram. That value is at least 2 bits and at
+    # most 9 bits. An n-bit value signifies matching (n-1) bits in the input with the lower
+    # (n-1) bits in the match value.
+    MATCH = 2
+    # A default instruction, encoded as [1,1,1] sets the default variable to its argument,
+    # and continues execution. It is followed by an integer in ASN encoding, and a subprogram.
+    DEFAULT = 3
+    # Not an actual instruction, but a way to encode the empty program that fails. In the
+    # encoder, it is used more generally to represent the failure case inside MATCH instructions,
+    # which may (if used inside the context of a DEFAULT instruction) actually correspond to
+    # a successful return. In this usage, they're always converted to an actual MATCH or RETURN
+    # before the top level is reached (see make_default below).
+    END = 4
+
+class _BinNode:
+    """A class representing a (node of) the parsed binary asmap format."""
+
+    @overload
+    def __init__(self, ins: _Instruction): ...
+    @overload
+    def __init__(self, ins: _Instruction, arg1: int): ...
+    @overload
+    def __init__(self, ins: _Instruction, arg1: "_BinNode", arg2: "_BinNode"): ...
+    @overload
+    def __init__(self, ins: _Instruction, arg1: int, arg2: "_BinNode"): ...
+
+    def __init__(self, ins: _Instruction, arg1=None, arg2=None):
+        """
+        Construct a new asmap node. Possibilities are:
+        - _BinNode(_Instruction.RETURN, asn)
+        - _BinNode(_Instruction.JUMP, node_0, node_1)
+        - _BinNode(_Instruction.MATCH, val, node)
+        - _BinNode(_Instruction.DEFAULT, asn, node)
+        - _BinNode(_Instruction.END)
+        """
+        self.ins = ins
+        self.arg1 = arg1
+        self.arg2 = arg2
+        if ins == _Instruction.RETURN:
+            assert isinstance(arg1, int)
+            assert arg2 is None
+            self.size = _CODER_INS.encode_size(ins.value) + _CODER_ASN.encode_size(arg1)
+        elif ins == _Instruction.JUMP:
+            assert isinstance(arg1, _BinNode)
+            assert isinstance(arg2, _BinNode)
+            self.size = (_CODER_INS.encode_size(ins.value) + _CODER_JUMP.encode_size(arg1.size) +
+                         arg1.size + arg2.size)
+        elif ins == _Instruction.DEFAULT:
+            assert isinstance(arg1, int)
+            assert isinstance(arg2, _BinNode)
+            self.size = _CODER_INS.encode_size(ins.value) + _CODER_ASN.encode_size(arg1) + arg2.size
+        elif ins == _Instruction.MATCH:
+            assert isinstance(arg1, int)
+            assert isinstance(arg2, _BinNode)
+            self.size = (_CODER_INS.encode_size(ins.value) + _CODER_MATCH.encode_size(arg1)
+                         + arg2.size)
+        elif ins == _Instruction.END:
+            assert arg1 is None
+            assert arg2 is None
+            self.size = 0
+        else:
+            assert False
+
+    @staticmethod
+    def make_end() -> "_BinNode":
+        """Constructor for a _BinNode with just an END instruction."""
+        return _BinNode(_Instruction.END)
+
+    @staticmethod
+    def make_leaf(val: int) -> "_BinNode":
+        """Constructor for a _BinNode of just a RETURN instruction."""
+        assert val is not None and val > 0
+        return _BinNode(_Instruction.RETURN, val)
+
+    @staticmethod
+    def make_branch(node0: "_BinNode", node1: "_BinNode") -> "_BinNode":
+        """
+        Construct a _BinNode corresponding to running either the node0 or node1 subprogram,
+        based on the next input bit. It exploits shortcuts that are possible in the encoding,
+        and uses either a JUMP, MATCH, or END instruction.
+        """
+        if node0.ins == _Instruction.END and node1.ins == _Instruction.END:
+            return node0
+        if node0.ins == _Instruction.END:
+            if node1.ins == _Instruction.MATCH and node1.arg1 <= 0xFF:
+                return _BinNode(node1.ins, node1.arg1 + (1 << node1.arg1.bit_length()), node1.arg2)
+            return _BinNode(_Instruction.MATCH, 3, node1)
+        if node1.ins == _Instruction.END:
+            if node0.ins == _Instruction.MATCH and node0.arg1 <= 0xFF:
+                return _BinNode(node0.ins, node0.arg1 + (1 << (node0.arg1.bit_length() - 1)),
+                                node0.arg2)
+            return _BinNode(_Instruction.MATCH, 2, node0)
+        return _BinNode(_Instruction.JUMP, node0, node1)
+
+    @staticmethod
+    def make_default(val: int, sub: "_BinNode") -> "_BinNode":
+        """
+        Construct a _BinNode that corresponds to the specified subprogram, with the specified
+        default value. It exploits shortcuts that are possible in the encoding, and will use
+        either a DEFAULT or a RETURN instruction."""
+        assert val is not None and val > 0
+        if sub.ins == _Instruction.END:
+            return _BinNode(_Instruction.RETURN, val)
+        if sub.ins in (_Instruction.RETURN, _Instruction.DEFAULT):
+            return sub
+        return _BinNode(_Instruction.DEFAULT, val, sub)
+
+@total_ordering
+class ASMap:
+    """
+    A class whose objects represent a mapping from subnets to ASNs.
+
+    Internally the mapping is stored as a binary trie, but can be converted
+    from/to a list of ASNEntry objects, and from/to the binary asmap file format.
+
+    In the trie representation, nodes are represented as bare lists for efficiency
+    and ease of manipulation:
+    - [0] means an unassigned subnet (no ASN mapping for it is present)
+    - [int] means a subnet mapped entirely to the specified ASN.
+    - [node,node] means a subnet whose lower half and upper half have different
+    -             mappings, represented by new trie nodes.
+    """
+
+    def update(self, prefix: List[bool], asn: int) -> None:
+        """Update this ASMap object to map prefix to the specified asn."""
+        assert asn == 0 or _CODER_ASN.can_encode(asn)
+
+        def recurse(node: List, offset: int) -> None:
+            if offset == len(prefix):
+                # Reached the end of prefix; overwrite this node.
+                node.clear()
+                node.append(asn)
+                return
+            if len(node) == 1:
+                # Need to descend into a leaf node; split it up.
+                oldasn = node[0]
+                node.clear()
+                node.append([oldasn])
+                node.append([oldasn])
+            # Descend into the node.
+            recurse(node[prefix[offset]], offset + 1)
+            # If the result is two identical leaf children, merge them.
+            if len(node[0]) == 1 and len(node[1]) == 1 and node[0] == node[1]:
+                oldasn = node[0][0]
+                node.clear()
+                node.append(oldasn)
+        recurse(self._trie, 0)
+
+    def update_multi(self, entries: List[Tuple[List[bool], int]]) -> None:
+        """Apply multiple update operations, where longer prefixes take precedence."""
+        entries.sort(key=lambda entry: len(entry[0]))
+        for prefix, asn in entries:
+            self.update(prefix, asn)
+
+    def _set_trie(self, trie) -> None:
+        """Set trie directly. Internal use only."""
+        def recurse(node: List) -> None:
+            if len(node) < 2:
+                return
+            recurse(node[0])
+            recurse(node[1])
+            if len(node[0]) == 2:
+                return
+            if node[0] == node[1]:
+                if len(node[0]) == 0:
+                    node.clear()
+                else:
+                    asn = node[0][0]
+                    node.clear()
+                    node.append(asn)
+        recurse(trie)
+        self._trie = trie
+
+    def __init__(self, entries: Optional[Iterable[ASNEntry]] = None) -> None:
+        """Construct an ASMap object from an optional list of entries."""
+        self._trie = [0]
+        if entries is not None:
+            def entry_key(entry):
+                """Sort function that places shorter prefixes first."""
+                prefix, asn = entry
+                return len(prefix), prefix, asn
+            for prefix, asn in sorted(entries, key=entry_key):
+                self.update(prefix, asn)
+
+    def lookup(self, prefix: List[bool]) -> Optional[int]:
+        """Look up a prefix. Returns ASN, or 0 if unassigned, or None if indeterminate."""
+        node = self._trie
+        for bit in prefix:
+            if len(node) == 1:
+                break
+            node = node[bit]
+        if len(node) == 1:
+            return node[0]
+        return None
+
+    def _to_entries_flat(self, fill: bool = False) -> List[ASNEntry]:
+        """Convert an ASMap object to a list of non-overlapping (prefix, asn) objects."""
+        prefix : List[bool] = []
+
+        def recurse(node: List) -> List[ASNEntry]:
+            ret = []
+            if len(node) == 1:
+                if node[0] > 0:
+                    ret = [(list(prefix), node[0])]
+            elif len(node) == 2:
+                prefix.append(False)
+                ret = recurse(node[0])
+                prefix[-1] = True
+                ret += recurse(node[1])
+                prefix.pop()
+                if fill and len(ret) > 1:
+                    asns = set(x[1] for x in ret)
+                    if len(asns) == 1:
+                        ret = [(list(prefix), list(asns)[0])]
+            return ret
+        return recurse(self._trie)
+
+    def _to_entries_minimal(self, fill: bool = False) -> List[ASNEntry]:
+        """Convert a trie to a minimal list of ASNEntry objects, exploiting overlap."""
+        prefix : List[bool] = []
+
+        def recurse(node: List) -> (Tuple[Dict[Optional[int], List[ASNEntry]], bool]):
+            if len(node) == 1 and node[0] == 0:
+                return {None if fill else 0: []}, True
+            if len(node) == 1:
+                return {node[0]: [], None: [(list(prefix), node[0])]}, False
+            ret: Dict[Optional[int], List[ASNEntry]] = {}
+            prefix.append(False)
+            left, lhole = recurse(node[0])
+            prefix[-1] = True
+            right, rhole = recurse(node[1])
+            prefix.pop()
+            hole = not fill and (lhole or rhole)
+            def candidate(ctx: Optional[int], res0: Optional[List[ASNEntry]],
+                    res1: Optional[List[ASNEntry]]):
+                if res0 is not None and res1 is not None:
+                    if ctx not in ret or len(res0) + len(res1) < len(ret[ctx]):
+                        ret[ctx] = res0 + res1
+            for ctx in set(left) | set(right):
+                candidate(ctx, left.get(ctx), right.get(ctx))
+                candidate(ctx, left.get(None), right.get(ctx))
+                candidate(ctx, left.get(ctx), right.get(None))
+            if not hole:
+                for ctx in list(ret):
+                    if ctx is not None:
+                        candidate(None, [(list(prefix), ctx)], ret[ctx])
+            if None in ret:
+                ret = {ctx:entries for ctx, entries in ret.items()
+                       if ctx is None or len(entries) < len(ret[None])}
+            if hole:
+                ret = {ctx:entries for ctx, entries in ret.items() if ctx is None or ctx == 0}
+            return ret, hole
+        res, _ = recurse(self._trie)
+        return res[0] if 0 in res else res[None]
+
+    def __str__(self) -> str:
+        """Convert this ASMap object to a string containing Python code constructing it."""
+        return f"ASMap({self._trie})"
+
+    def to_entries(self, overlapping: bool = True, fill: bool = False) -> List[ASNEntry]:
+        """
+        Convert the mappings in this ASMap object to a list of ASNEntry objects.
+
+        Arguments:
+            overlapping: Permit the subnets in the resulting ASNEntry to overlap.
+                         Setting this can result in a shorter list.
+            fill:        Permit the resulting ASNEntry objects to cover subnets that
+                         are unassigned in this ASMap object. Setting this can
+                         result in a shorter list.
+        """
+        if overlapping:
+            return self._to_entries_minimal(fill)
+        return self._to_entries_flat(fill)
+
+    @staticmethod
+    def from_random(num_leaves: int = 10, max_asn: int = 6,
+                    unassigned_prob: float = 0.5) -> "ASMap":
+        """
+        Construct a random ASMap object, with specified:
+         - Number of leaves in its trie (at least 1)
+         - Maximum ASN value (at least 1)
+         - Probability for leaf nodes to be unassigned
+
+        The number of leaves in the resulting object may be less than what is
+        requested. This method is mostly intended for testing.
+        """
+        assert num_leaves >= 1
+        assert max_asn >= 1 or unassigned_prob == 1
+        assert _CODER_ASN.can_encode(max_asn)
+        assert 0.0 <= unassigned_prob <= 1.0
+        trie: List = []
+        leaves = [trie]
+        ret = ASMap()
+        for i in range(1, num_leaves):
+            idx = random.randrange(i)
+            leaf = leaves[idx]
+            lastleaf = leaves.pop()
+            if idx + 1 < i:
+                leaves[idx] = lastleaf
+            leaf.append([])
+            leaf.append([])
+            leaves.append(leaf[0])
+            leaves.append(leaf[1])
+        for leaf in leaves:
+            if random.random() >= unassigned_prob:
+                leaf.append(random.randrange(1, max_asn + 1))
+            else:
+                leaf.append(0)
+        #pylint: disable=protected-access
+        ret._set_trie(trie)
+        return ret
+
+    def _to_binnode(self, fill: bool = False) -> _BinNode:
+        """Convert a trie to a _BinNode object."""
+        def recurse(node: List) -> Tuple[Dict[Optional[int], _BinNode], bool]:
+            if len(node) == 1 and node[0] == 0:
+                return {(None if fill else 0): _BinNode.make_end()}, True
+            if len(node) == 1:
+                return {None: _BinNode.make_leaf(node[0]), node[0]: _BinNode.make_end()}, False
+            ret: Dict[Optional[int], _BinNode] = {}
+            left, lhole = recurse(node[0])
+            right, rhole = recurse(node[1])
+            hole = (lhole or rhole) and not fill
+
+            def candidate(ctx: Optional[int], arg1, arg2, func: Callable):
+                if arg1 is not None and arg2 is not None:
+                    cand = func(arg1, arg2)
+                    if ctx not in ret or cand.size < ret[ctx].size:
+                        ret[ctx] = cand
+
+            for ctx in set(left) | set(right):
+                candidate(ctx, left.get(ctx), right.get(ctx), _BinNode.make_branch)
+                candidate(ctx, left.get(None), right.get(ctx), _BinNode.make_branch)
+                candidate(ctx, left.get(ctx), right.get(None), _BinNode.make_branch)
+            if not hole:
+                for ctx in set(ret) - set([None]):
+                    candidate(None, ctx, ret[ctx], _BinNode.make_default)
+            if None in ret:
+                ret = {ctx:enc for ctx, enc in ret.items()
+                       if ctx is None or enc.size < ret[None].size}
+            if hole:
+                ret = {ctx:enc for ctx, enc in ret.items() if ctx is None or ctx == 0}
+            return ret, hole
+        res, _ = recurse(self._trie)
+        return res[0] if 0 in res else res[None]
+
+    @staticmethod
+    def _from_binnode(binnode: _BinNode) -> "ASMap":
+        """Construct an ASMap object from a _BinNode. Internal use only."""
+        def recurse(node: _BinNode, default: int) -> List:
+            if node.ins == _Instruction.RETURN:
+                return [node.arg1]
+            if node.ins == _Instruction.JUMP:
+                return [recurse(node.arg1, default), recurse(node.arg2, default)]
+            if node.ins == _Instruction.MATCH:
+                val = node.arg1
+                sub = recurse(node.arg2, default)
+                while val >= 2:
+                    bit = val & 1
+                    val >>= 1
+                    if bit:
+                        sub = [[default], sub]
+                    else:
+                        sub = [sub, [default]]
+                return sub
+            assert node.ins == _Instruction.DEFAULT
+            return recurse(node.arg2, node.arg1)
+        ret = ASMap()
+        if binnode.ins != _Instruction.END:
+            #pylint: disable=protected-access
+            ret._set_trie(recurse(binnode, 0))
+        return ret
+
+    def to_binary(self, fill: bool = False) -> bytes:
+        """
+        Convert this ASMap object to binary.
+
+        Argument:
+            fill: permit the resulting binary encoder to contain mappers for
+                  unassigned subnets in this ASMap object. Doing so may
+                  reduce the size of the encoding.
+        Returns:
+            A bytes object with the encoding of this ASMap object.
+        """
+        bits: List[int] = []
+
+        def recurse(node: _BinNode) -> None:
+            _CODER_INS.encode(node.ins.value, bits)
+            if node.ins == _Instruction.RETURN:
+                _CODER_ASN.encode(node.arg1, bits)
+            elif node.ins == _Instruction.JUMP:
+                _CODER_JUMP.encode(node.arg1.size, bits)
+                recurse(node.arg1)
+                recurse(node.arg2)
+            elif node.ins == _Instruction.DEFAULT:
+                _CODER_ASN.encode(node.arg1, bits)
+                recurse(node.arg2)
+            else:
+                assert node.ins == _Instruction.MATCH
+                _CODER_MATCH.encode(node.arg1, bits)
+                recurse(node.arg2)
+
+        binnode = self._to_binnode(fill)
+        if binnode.ins != _Instruction.END:
+            recurse(binnode)
+
+        val = 0
+        nbits = 0
+        ret = []
+        for bit in bits:
+            val += (bit << nbits)
+            nbits += 1
+            if nbits == 8:
+                ret.append(val)
+                val = 0
+                nbits = 0
+        if nbits:
+            ret.append(val)
+        return bytes(ret)
+
+    @staticmethod
+    def from_binary(bindata: bytes) -> Optional["ASMap"]:
+        """Decode an ASMap object from the provided binary encoding."""
+
+        bits: List[int] = []
+        for byte in bindata:
+            bits.extend((byte >> i) & 1 for i in range(8))
+
+        def recurse(bitpos: int) -> Tuple[_BinNode, int]:
+            insval, bitpos = _CODER_INS.decode(bits, bitpos)
+            ins = _Instruction(insval)
+            if ins == _Instruction.RETURN:
+                asn, bitpos = _CODER_ASN.decode(bits, bitpos)
+                return _BinNode(ins, asn), bitpos
+            if ins == _Instruction.JUMP:
+                jump, bitpos = _CODER_JUMP.decode(bits, bitpos)
+                left, bitpos1 = recurse(bitpos)
+                if bitpos1 != bitpos + jump:
+                    raise ValueError("Inconsistent jump")
+                right, bitpos = recurse(bitpos1)
+                return _BinNode(ins, left, right), bitpos
+            if ins == _Instruction.MATCH:
+                match, bitpos = _CODER_MATCH.decode(bits, bitpos)
+                sub, bitpos = recurse(bitpos)
+                return _BinNode(ins, match, sub), bitpos
+            assert ins == _Instruction.DEFAULT
+            asn, bitpos = _CODER_ASN.decode(bits, bitpos)
+            sub, bitpos = recurse(bitpos)
+            return _BinNode(ins, asn, sub), bitpos
+
+        if len(bits) == 0:
+            binnode = _BinNode(_Instruction.END)
+        else:
+            try:
+                binnode, bitpos = recurse(0)
+            except (ValueError, IndexError):
+                return None
+            if bitpos < len(bits) - 7:
+                return None
+            if not all(bit == 0 for bit in bits[bitpos:]):
+                return None
+
+        return ASMap._from_binnode(binnode)
+
+    def __lt__(self, other: "ASMap") -> bool:
+        return self._trie < other._trie
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, ASMap):
+            return self._trie == other._trie
+        return False
+
+    def extends(self, req: "ASMap") -> bool:
+        """Determine whether this matches req for all subranges where req is assigned."""
+        def recurse(actual: List, require: List) -> bool:
+            if len(require) == 1 and require[0] == 0:
+                return True
+            if len(require) == 1:
+                if len(actual) == 1:
+                    return bool(require[0] == actual[0])
+                return recurse(actual[0], require) and recurse(actual[1], require)
+            if len(actual) == 2:
+                return recurse(actual[0], require[0]) and recurse(actual[1], require[1])
+            return recurse(actual, require[0]) and recurse(actual, require[1])
+        assert isinstance(req, ASMap)
+        #pylint: disable=protected-access
+        return recurse(self._trie, req._trie)
+
+    def diff(self, other: "ASMap") -> List[ASNDiff]:
+        """Compute the diff from self to other."""
+        prefix: List[bool] = []
+        ret: List[ASNDiff] = []
+
+        def recurse(old_node: List, new_node: List):
+            if len(old_node) == 1 and len(new_node) == 1:
+                if old_node[0] != new_node[0]:
+                    ret.append((list(prefix), old_node[0], new_node[0]))
+            else:
+                old_left: List = old_node if len(old_node) == 1 else old_node[0]
+                old_right: List = old_node if len(old_node) == 1 else old_node[1]
+                new_left: List = new_node if len(new_node) == 1 else new_node[0]
+                new_right: List = new_node if len(new_node) == 1 else new_node[1]
+                prefix.append(False)
+                recurse(old_left, new_left)
+                prefix[-1] = True
+                recurse(old_right, new_right)
+                prefix.pop()
+        assert isinstance(other, ASMap)
+        #pylint: disable=protected-access
+        recurse(self._trie, other._trie)
+        return ret
+
+    def __copy__(self) -> "ASMap":
+        """Construct a copy of this ASMap object. Its state will not be shared."""
+        ret = ASMap()
+        #pylint: disable=protected-access
+        ret._set_trie(copy.deepcopy(self._trie))
+        return ret
+
+    def __deepcopy__(self, _) -> "ASMap":
+        # ASMap objects do not allow sharing of the _trie member, so we don't need the memoization.
+        return self.__copy__()
+
+
+class TestASMap(unittest.TestCase):
+    """Unit tests for this module."""
+
+    def test_ipv6_prefix_roundtrips(self) -> None:
+        """Test that random IPv6 network ranges roundtrip through prefix encoding."""
+        for _ in range(20):
+            net_bits = random.getrandbits(128)
+            for prefix_len in range(0, 129):
+                masked_bits = (net_bits >> (128 - prefix_len)) << (128 - prefix_len)
+                net = ipaddress.IPv6Network((masked_bits.to_bytes(16, 'big'), prefix_len))
+                prefix = net_to_prefix(net)
+                self.assertTrue(len(prefix) <= 128)
+                net2 = prefix_to_net(prefix)
+                self.assertEqual(net, net2)
+
+    def test_ipv4_prefix_roundtrips(self) -> None:
+        """Test that random IPv4 network ranges roundtrip through prefix encoding."""
+        for _ in range(100):
+            net_bits = random.getrandbits(32)
+            for prefix_len in range(0, 33):
+                masked_bits = (net_bits >> (32 - prefix_len)) << (32 - prefix_len)
+                net = ipaddress.IPv4Network((masked_bits.to_bytes(4, 'big'), prefix_len))
+                prefix = net_to_prefix(net)
+                self.assertTrue(32 <= len(prefix) <= 128)
+                net2 = prefix_to_net(prefix)
+                self.assertEqual(net, net2)
+
+    def test_asmap_roundtrips(self) -> None:
+        """Test case that verifies random ASMap objects roundtrip to/from entries/binary."""
+        # Iterate over the number of leaves the random test ASMap objects have.
+        for leaves in range(1, 20):
+            # Iterate over the number of bits in the AS numbers used.
+            for asnbits in range(0, 24):
+                # Iterate over the probability that leaves are unassigned.
+                for pct in range(101):
+                    # Construct a random ASMap object according to the above parameters.
+                    asmap = ASMap.from_random(num_leaves=leaves, max_asn=1 + (1 << asnbits),
+                                              unassigned_prob=0.01 * pct)
+                    # Run tests for to_entries and construction from those entries, both
+                    # for overlapping and non-overlapping ones.
+                    for overlapping in [False, True]:
+                        entries = asmap.to_entries(overlapping=overlapping, fill=False)
+                        random.shuffle(entries)
+                        asmap2 = ASMap(entries)
+                        assert asmap2 is not None
+                        self.assertEqual(asmap2, asmap)
+                        entries = asmap.to_entries(overlapping=overlapping, fill=True)
+                        random.shuffle(entries)
+                        asmap2 = ASMap(entries)
+                        assert asmap2 is not None
+                        self.assertTrue(asmap2.extends(asmap))
+
+                    # Run tests for to_binary and construction from binary.
+                    enc = asmap.to_binary(fill=False)
+                    asmap3 = ASMap.from_binary(enc)
+                    assert asmap3 is not None
+                    self.assertEqual(asmap3, asmap)
+                    enc = asmap.to_binary(fill=True)
+                    asmap3 = ASMap.from_binary(enc)
+                    assert asmap3 is not None
+                    self.assertTrue(asmap3.extends(asmap))
+
+    def test_patching(self) -> None:
+        """Test behavior of update, lookup, extends, and diff."""
+        #pylint: disable=too-many-locals,too-many-nested-blocks
+        # Iterate over the number of leaves the random test ASMap objects have.
+        for leaves in range(1, 20):
+            # Iterate over the number of bits in the AS numbers used.
+            for asnbits in range(0, 10):
+                # Iterate over the probability that leaves are unassigned.
+                for pct in range(0, 101):
+                    # Construct a random ASMap object according to the above parameters.
+                    asmap = ASMap.from_random(num_leaves=leaves, max_asn=1 + (1 << asnbits),
+                                              unassigned_prob=0.01 * pct)
+                    # Make a copy of that asmap object to which patches will be applied.
+                    # It starts off being equal to asmap.
+                    patched = copy.copy(asmap)
+                    # Keep a list of patches performed.
+                    patches: List[ASNEntry] = []
+                    # Initially there cannot be any difference.
+                    self.assertEqual(asmap.diff(patched), [])
+                    # Make 5 patches, each building on top of the previous ones.
+                    for _ in range(0, 5):
+                        # Construct a random path and new ASN to assign it to, apply it to patched,
+                        # and remember it in patches.
+                        pathlen = random.randrange(5)
+                        path = [random.getrandbits(1) != 0 for _ in range(pathlen)]
+                        newasn = random.randrange(1 + (1 << asnbits))
+                        patched.update(path, newasn)
+                        patches = [(path, newasn)] + patches
+
+                        # Compute the diff, and whether asmap extends patched, and the other way
+                        # around.
+                        diff = asmap.diff(patched)
+                        self.assertEqual(asmap == patched, len(diff) == 0)
+                        extends = asmap.extends(patched)
+                        back_extends = patched.extends(asmap)
+                        # Determine whether those extends results are consistent with the diff
+                        # result.
+                        self.assertEqual(extends, all(d[2] == 0 for d in diff))
+                        self.assertEqual(back_extends, all(d[1] == 0 for d in diff))
+                        # For every diff found:
+                        for path, old_asn, new_asn in diff:
+                            # Verify asmap and patched actually differ there.
+                            self.assertTrue(old_asn != new_asn)
+                            self.assertEqual(asmap.lookup(path), old_asn)
+                            self.assertEqual(patched.lookup(path), new_asn)
+                            for _ in range(2):
+                                # Extend the path far enough that it's smaller than any mapped
+                                # range, and check the lookup holds there too.
+                                spec_path = list(path)
+                                while len(spec_path) < 32:
+                                    spec_path.append(random.getrandbits(1) != 0)
+                                self.assertEqual(asmap.lookup(spec_path), old_asn)
+                                self.assertEqual(patched.lookup(spec_path), new_asn)
+                                # Search through the list of performed patches to find the last one
+                                # applying to the extended path (note that patches is in reverse
+                                # order, so the first match should work).
+                                found = False
+                                for patch_path, patch_asn in patches:
+                                    if spec_path[:len(patch_path)] == patch_path:
+                                        # When found, it must match whatever the result was patched
+                                        # to.
+                                        self.assertEqual(new_asn, patch_asn)
+                                        found = True
+                                        break
+                                # And such a patch must exist.
+                                self.assertTrue(found)
+
+if __name__ == '__main__':
+    unittest.main()

contrib/seeds/makeseeds.py

@@ -1,27 +1,28 @@
 #!/usr/bin/env python3
-# Copyright (c) 2013-2020 The Bitcoin Core developers
+# Copyright (c) 2013-2022 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 #
 # Generate seeds.txt from Pieter's DNS seeder
 #
 
+import argparse
+import collections
+import ipaddress
 import re
 import sys
-import dns.resolver
-import collections
+from typing import List, Dict, Union
+
+from asmap import ASMap, net_to_prefix
 
 NSEEDS=512
 
-MAX_SEEDS_PER_ASN=2
-
-MIN_BLOCKS = 337600
-
-# These are hosts that have been observed to be behaving strangely (e.g.
-# aggressively connecting to every node).
-with open("suspicious_hosts.txt", mode="r", encoding="utf-8") as f:
-    SUSPICIOUS_HOSTS = {s.strip() for s in f if s.strip()}
+MAX_SEEDS_PER_ASN = {
+    'ipv4': 2,
+    'ipv6': 10,
+}
 
+MIN_BLOCKS = 730000
 
 PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$")
 PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$")
@@ -37,13 +38,18 @@ PATTERN_AGENT = re.compile(
     r"0.20.(0|1|2|99)|"
     r"0.21.(0|1|2|99)|"
     r"22.(0|99)|"
-    r"23.99"
+    r"23.(0|99)|"
+    r"24.99"
     r")")
 
-def parseline(line):
+def parseline(line: str) -> Union[dict, None]:
+    """ Parses a line from `seeds_main.txt` into a dictionary of details for that line.
+    or `None`, if the line could not be parsed.
+    """
     sline = line.split()
     if len(sline) < 11:
-       return None
+        # line too short to be valid, skip it.
+        return None
     m = PATTERN_IPV4.match(sline[0])
     sortkey = None
     ip = None
@@ -107,98 +113,95 @@ def parseline(line):
         'sortkey': sortkey,
     }
 
-def dedup(ips):
-    '''deduplicate by address,port'''
+def dedup(ips: List[Dict]) -> List[Dict]:
+    """ Remove duplicates from `ips` where multiple ips share address and port. """
     d = {}
     for ip in ips:
         d[ip['ip'],ip['port']] = ip
     return list(d.values())
 
-def filtermultiport(ips):
-    '''Filter out hosts with more nodes per IP'''
+def filtermultiport(ips: List[Dict]) -> List[Dict]:
+    """ Filter out hosts with more nodes per IP"""
     hist = collections.defaultdict(list)
     for ip in ips:
         hist[ip['sortkey']].append(ip)
     return [value[0] for (key,value) in list(hist.items()) if len(value)==1]
 
-def lookup_asn(net, ip):
-    '''
-    Look up the asn for an IP (4 or 6) address by querying cymru.com, or None
-    if it could not be found.
-    '''
-    try:
-        if net == 'ipv4':
-            ipaddr = ip
-            prefix = '.origin'
-        else:                  # http://www.team-cymru.com/IP-ASN-mapping.html
-            res = str()                         # 2001:4860:b002:23::68
-            for nb in ip.split(':')[:4]:  # pick the first 4 nibbles
-                for c in nb.zfill(4):           # right padded with '0'
-                    res += c + '.'              # 2001 4860 b002 0023
-            ipaddr = res.rstrip('.')            # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3
-            prefix = '.origin6'
-
-        asn = int([x.to_text() for x in dns.resolver.resolve('.'.join(
-                   reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com',
-                   'TXT').response.answer][0].split('\"')[1].split(' ')[0])
-        return asn
-    except Exception as e:
-        sys.stderr.write(f'ERR: Could not resolve ASN for "{ip}": {e}\n')
-        return None
-
 # Based on Greg Maxwell's seed_filter.py
-def filterbyasn(ips, max_per_asn, max_per_net):
+def filterbyasn(asmap: ASMap, ips: List[Dict], max_per_asn: Dict, max_per_net: int) -> List[Dict]:
+    """ Prunes `ips` by
+    (a) trimming ips to have at most `max_per_net` ips from each net (e.g. ipv4, ipv6); and
+    (b) trimming ips to have at most `max_per_asn` ips from each asn in each net.
+    """
     # Sift out ips by type
     ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']]
     ips_onion = [ip for ip in ips if ip['net'] == 'onion']
 
     # Filter IPv46 by ASN, and limit to max_per_net per network
     result = []
-    net_count = collections.defaultdict(int)
-    asn_count = collections.defaultdict(int)
-    for ip in ips_ipv46:
+    net_count: Dict[str, int] = collections.defaultdict(int)
+    asn_count: Dict[int, int] = collections.defaultdict(int)
+
+    for i, ip in enumerate(ips_ipv46):
         if net_count[ip['net']] == max_per_net:
+            # do not add this ip as we already too many
+            # ips from this network
             continue
-        asn = lookup_asn(ip['net'], ip['ip'])
-        if asn is None or asn_count[asn] == max_per_asn:
+        asn = asmap.lookup(net_to_prefix(ipaddress.ip_network(ip['ip'])))
+        if not asn or asn_count[ip['net'], asn] == max_per_asn[ip['net']]:
+            # do not add this ip as we already have too many
+            # ips from this ASN on this network
             continue
-        asn_count[asn] += 1
+        asn_count[ip['net'], asn] += 1
         net_count[ip['net']] += 1
+        ip['asn'] = asn
         result.append(ip)
 
     # Add back Onions (up to max_per_net)
     result.extend(ips_onion[0:max_per_net])
     return result
 
-def ip_stats(ips):
-    hist = collections.defaultdict(int)
+def ip_stats(ips: List[Dict]) -> str:
+    """ Format and return pretty string from `ips`. """
+    hist: Dict[str, int] = collections.defaultdict(int)
     for ip in ips:
         if ip is not None:
             hist[ip['net']] += 1
 
-    return '%6d %6d %6d' % (hist['ipv4'], hist['ipv6'], hist['onion'])
+    return f"{hist['ipv4']:6d} {hist['ipv6']:6d} {hist['onion']:6d}"
+
+def parse_args():
+    argparser = argparse.ArgumentParser(description='Generate a list of bitcoin node seed ip addresses.')
+    argparser.add_argument("-a","--asmap", help='the location of the asmap asn database file (required)', required=True)
+    return argparser.parse_args()
 
 def main():
+    args = parse_args()
+
+    print(f'Loading asmap database "{args.asmap}"…', end='', file=sys.stderr, flush=True)
+    with open(args.asmap, 'rb') as f:
+        asmap = ASMap.from_binary(f.read())
+    print('Done.', file=sys.stderr)
+
+    print('Loading and parsing DNS seeds…', end='', file=sys.stderr, flush=True)
     lines = sys.stdin.readlines()
     ips = [parseline(line) for line in lines]
+    print('Done.', file=sys.stderr)
 
     print('\x1b[7m  IPv4   IPv6  Onion Pass                                               \x1b[0m', file=sys.stderr)
-    print('%s Initial' % (ip_stats(ips)), file=sys.stderr)
+    print(f'{ip_stats(ips):s} Initial', file=sys.stderr)
     # Skip entries with invalid address.
     ips = [ip for ip in ips if ip is not None]
-    print('%s Skip entries with invalid address' % (ip_stats(ips)), file=sys.stderr)
+    print(f'{ip_stats(ips):s} Skip entries with invalid address', file=sys.stderr)
     # Skip duplicates (in case multiple seeds files were concatenated)
     ips = dedup(ips)
-    print('%s After removing duplicates' % (ip_stats(ips)), file=sys.stderr)
-    # Skip entries from suspicious hosts.
-    ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS]
-    print('%s Skip entries from suspicious hosts' % (ip_stats(ips)), file=sys.stderr)
+    print(f'{ip_stats(ips):s} After removing duplicates', file=sys.stderr)
     # Enforce minimal number of blocks.
     ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS]
-    print('%s Enforce minimal number of blocks' % (ip_stats(ips)), file=sys.stderr)
+    print(f'{ip_stats(ips):s} Enforce minimal number of blocks', file=sys.stderr)
     # Require service bit 1.
     ips = [ip for ip in ips if (ip['service'] & 1) == 1]
-    print('%s Require service bit 1' % (ip_stats(ips)), file=sys.stderr)
+    print(f'{ip_stats(ips):s} Require service bit 1', file=sys.stderr)
     # Require at least 50% 30-day uptime for clearnet, 10% for onion.
     req_uptime = {
         'ipv4': 50,
@@ -206,25 +209,28 @@ def main():
         'onion': 10,
     }
     ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]]
-    print('%s Require minimum uptime' % (ip_stats(ips)), file=sys.stderr)
+    print(f'{ip_stats(ips):s} Require minimum uptime', file=sys.stderr)
     # Require a known and recent user agent.
     ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])]
-    print('%s Require a known and recent user agent' % (ip_stats(ips)), file=sys.stderr)
+    print(f'{ip_stats(ips):s} Require a known and recent user agent', file=sys.stderr)
     # Sort by availability (and use last success as tie breaker)
     ips.sort(key=lambda x: (x['uptime'], x['lastsuccess'], x['ip']), reverse=True)
     # Filter out hosts with multiple bitcoin ports, these are likely abusive
     ips = filtermultiport(ips)
-    print('%s Filter out hosts with multiple bitcoin ports' % (ip_stats(ips)), file=sys.stderr)
+    print(f'{ip_stats(ips):s} Filter out hosts with multiple bitcoin ports', file=sys.stderr)
     # Look up ASNs and limit results, both per ASN and globally.
-    ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS)
-    print('%s Look up ASNs and limit results per ASN and per net' % (ip_stats(ips)), file=sys.stderr)
+    ips = filterbyasn(asmap, ips, MAX_SEEDS_PER_ASN, NSEEDS)
+    print(f'{ip_stats(ips):s} Look up ASNs and limit results per ASN and per net', file=sys.stderr)
     # Sort the results by IP address (for deterministic output).
     ips.sort(key=lambda x: (x['net'], x['sortkey']))
     for ip in ips:
         if ip['net'] == 'ipv6':
-            print('[%s]:%i' % (ip['ip'], ip['port']))
+            print(f"[{ip['ip']}]:{ip['port']}", end="")
         else:
-            print('%s:%i' % (ip['ip'], ip['port']))
+            print(f"{ip['ip']}:{ip['port']}", end="")
+        if 'asn' in ip:
+            print(f" # AS{ip['asn']}", end="")
+        print()
 
 if __name__ == '__main__':
     main()

contrib/seeds/nodes_main_manual.txt

@@ -1,42 +1,77 @@
 
-# manually added 2021-03 for minimal torv3 bootstrap support
-2g5qfdkn2vvcbqhzcyvyiitg4ceukybxklraxjnu7atlhd22gdwywaid.onion:8333
-2jmtxvyup3ijr7u6uvu7ijtnojx4g5wodvaedivbv74w4vzntxbrhvad.onion:8333
-37m62wn7dz3uqpathpc4qfmgrbupachj52nt3jbtbjugpbu54kbud7yd.onion:8333
+# manually updated 2022-08 for minimal torv3 bootstrap support
 5g72ppm3krkorsfopcm2bi7wlv4ohhs4u4mlseymasn7g7zhdcyjpfid.onion:8333
-7cgwjuwi5ehvcay4tazy7ya6463bndjk6xzrttw5t3xbpq4p22q6fyid.onion:8333
-7pyrpvqdhmayxggpcyqn5l3m5vqkw3qubnmgwlpya2mdo6x7pih7r7id.onion:8333
 b64xcbleqmwgq2u46bh4hegnlrzzvxntyzbmucn3zt7cssm7y4ubv3id.onion:8333
-ejxefzf5fpst4mg2rib7grksvscl7p6fvjp6agzgfc2yglxnjtxc3aid.onion:8333
 fjdyxicpm4o42xmedlwl3uvk5gmqdfs5j37wir52327vncjzvtpfv7yd.onion:8333
 fpz6r5ppsakkwypjcglz6gcnwt7ytfhxskkfhzu62tnylcknh3eq6pad.onion:8333
-fzhn4uoxfbfss7h7d6ffbn266ca432ekbbzvqtsdd55ylgxn4jucm5qd.onion:8333
 gxo5anvfnffnftfy5frkgvplq3rpga2ie3tcblo2vl754fvnhgorn5yd.onion:8333
 ifdu5qvbofrt4ekui2iyb3kbcyzcsglazhx2hn4wfskkrx2v24qxriid.onion:8333
 itz3oxsihs62muvknc237xabl5f6w6rfznfhbpayrslv2j2ubels47yd.onion:8333
-lrjh6fywjqttmlifuemq3puhvmshxzzyhoqx7uoufali57eypuenzzid.onion:8333
+kpgvmscirrdqpekbqjsvw5teanhatztpp2gl6eee4zkowvwfxwenqaid.onion:8333
 m7cbpjolo662uel7rpaid46as2otcj44vvwg3gccodnvaeuwbm3anbyd.onion:8333
-opnyfyeiibe5qo5a3wbxzbb4xdiagc32bbce46owmertdknta5mi7uyd.onion:8333
-owjsdxmzla6d7lrwkbmetywqym5cyswpihciesfl5qdv2vrmwsgy4uqd.onion:8333
-q7kgmd7n7h27ds4fg7wocgniuqb3oe2zxp4nfe4skd5da6wyipibqzqd.onion:8333
+mwmfluek4au6mxxpw6fy7sjhkm65bdfc7izc7lpz3trewfdghyrzsbid.onion:8333
 rp7k2go3s5lyj3fnj6zn62ktarlrsft2ohlsxkyd7v3e3idqyptvread.onion:8333
-sys54sv4xv3hn3sdiv3oadmzqpgyhd4u4xphv4xqk64ckvaxzm57a7yd.onion:8333
-tddeij4qigtjr6jfnrmq6btnirmq5msgwcsdpcdjr7atftm7cxlqztid.onion:8333
-vi5bnbxkleeqi6hfccjochnn65lcxlfqs4uwgmhudph554zibiusqnad.onion:8333
-xqt25cobm5zqucac3634zfght72he6u3eagfyej5ellbhcdgos7t2had.onion:8333
 
-# manually added 2021-08 for minimal i2p bootstrap support
+# manually updated 2022-08 for minimal i2p bootstrap support
+255fhcp6ajvftnyo7bwz3an3t4a4brhopm3bamyh2iu5r3gnr2rq.b32.i2p:0
+27yrtht5b5bzom2w5ajb27najuqvuydtzb7bavlak25wkufec5mq.b32.i2p:0
+2el6enckmfyiwbfcwsygkwksovtynzsigmyv3bzyk7j7qqahooua.b32.i2p:0
+3gocb7wc4zvbmmebktet7gujccuux4ifk3kqilnxnj5wpdpqx2hq.b32.i2p:0
+3tns2oov4tnllntotazy6umzkq4fhkco3iu5rnkxtu3pbfzxda7q.b32.i2p:0
+4fcc23wt3hyjk3csfzcdyjz5pcwg5dzhdqgma6bch2qyiakcbboa.b32.i2p:0
+4osyqeknhx5qf3a73jeimexwclmt42cju6xdp7icja4ixxguu2hq.b32.i2p:0
+4umsi4nlmgyp4rckosg4vegd2ysljvid47zu7pqsollkaszcbpqq.b32.i2p:0
+52v6uo6crlrlhzphslyiqblirux6olgsaa45ixih7sq5np4jujaa.b32.i2p:0
+6j2ezegd3e2e2x3o3pox335f5vxfthrrigkdrbgfbdjchm5h4awa.b32.i2p:0
+6n36ljyr55szci5ygidmxqer64qr24f4qmnymnbvgehz7qinxnla.b32.i2p:0
+72yjs6mvlby3ky6mgpvvlemmwq5pfcznrzd34jkhclgrishqdxva.b32.i2p:0
+7r4ri53lby2i3xqbgpw3idvhzeku7ubhftlf72ldqkg5kde6dauq.b32.i2p:0
 a5qsnv3maw77mlmmzlcglu6twje6ttctd3fhpbfwcbpmewx6fczq.b32.i2p:0
-bitcornrd36coazsbzsz4pdebyzvaplmsalq4kpoljmn6cg6x5zq.b32.i2p:0
+aovep2pco7v2k4rheofrgytbgk23eg22dczpsjqgqtxcqqvmxk6a.b32.i2p:0
+bddbsmkas3z6fakorbkfjhv77i4hv6rysyjsvrdjukxolfghc23q.b32.i2p:0
+bitcoi656nll5hu6u7ddzrmzysdtwtnzcnrjd4rfdqbeey7dmn5a.b32.i2p:0
+brifkruhlkgrj65hffybrjrjqcgdgqs2r7siizb5b2232nruik3a.b32.i2p:0
 c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:0
-dhtq2p76tyhi442aidb3vd2bv7yxxjuddpb2jydnnrl2ons5bhha.b32.i2p:0
+day3hgxyrtwjslt54sikevbhxxs4qzo7d6vi72ipmscqtq3qmijq.b32.i2p:0
+di2zq6fr3fegf2jdcd7hdwyql4umr462gonsns2nxz5qg5vz4bka.b32.i2p:0
+e55k6wu46rzp4pg5pk5npgbr3zz45bc3ihtzu2xcye5vwnzdy7pq.b32.i2p:0
+eciohu5nq7vsvwjjc52epskuk75d24iccgzmhbzrwonw6lx4gdva.b32.i2p:0
+ejlnngarmhqvune74ko7kk55xtgbz5i5ncs4vmnvjpy3l7y63xaa.b32.i2p:0
+g47cqoppu26pr4n2cfaioqx7lbdi7mea7yqhlrkdz3wjwxjxdh2a.b32.i2p:0
 h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:0
-hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p:0
+hhfi4yqkg2twqiwezrfksftjjofbyx3ojkmlnfmcwntgnrjjhkya.b32.i2p:0
+hpiibrflqkbrcshfhmrtwfyeb7mds7a3obzwrgarejevddzamvsq.b32.i2p:0
+i4pyhsfdq4247dunel7paatdaq5gusi2hnybp2yf5wxwdnrgxaqq.b32.i2p:0
+iw6tgpmbdykffceku5da6nzf2bmz66fvp5fpcvemfu3df6aq6pga.b32.i2p:0
+jkfuajo4ayvo2rbv5qdj443q6adqmnormbhsf2f7rlp5t24xomda.b32.i2p:0
 jz3s4eurm5vzjresf4mwo7oni4bk36daolwxh4iqtewakylgkxmq.b32.i2p:0
-kokkmpquqlkptu5hkmzqlttsmtwxicldr4so7wqsufk6bwf32nma.b32.i2p:0
+liu75cvktv4icbctg72w7nxbk4eibt7wamizfdii4omz7gcke5vq.b32.i2p:0
+ljsquuu3y4xje6l32p32inn6r2y6ull6oocgup6jtjrohrqxbz6a.b32.i2p:0
+lrah7acdsgopybg43shadwwiv6igezaw64i6jb5muqdg7dmhj3la.b32.i2p:0
+lzuu6mjtu7vd55d2biphicihufipoa7vyym6xfnkmmlra3tiziia.b32.i2p:0
+m6bpynxkv2ktwxkg6p2gyudjfhdupb6kuzabeqdnckkdkf4kxjla.b32.i2p:0
+m6v454xd6p3bt5swujgmveklsp7lzbkqlqqfc2p36cjlwv5dbucq.b32.i2p:0
+mlgeizrroynuhpxbzeosajt5u4ddcvynxfmcbm6kwjpaufilxigq.b32.i2p:0
+ofubxr2ir7u2guzjwyrvujicivzmvinwa36nuzlrg7tnsmebal7a.b32.i2p:0
+okfxeoh6itu4f5f43dhbzvkqwfrvm5c66lj6lvjj4q2b35i4pk4q.b32.i2p:0
+oz2ia3flpm3du2tyusulrn7h7e2eo3juzkrmn34bvnrlcrugv7ia.b32.i2p:0
+qd6jlsevsexww3wefpqs7iglxb3f63y4e6ydulfzrvwflpicmdqa.b32.i2p:0
+qddg7myylinn4tw6kdjmmp6fsyetkosnrbp2gsjx77tmkqyqv6ua.b32.i2p:0
+rizfinyses2r3or4iubs5wx66gdy6mpf73w7uobfacm2l5cral3q.b32.i2p:0
+s5hhjtmlg53bko3nwwskas7xgsmeqzy6thtsj5aa64djyrljgqaq.b32.i2p:0
 sedndhv5vpcgdmykyi5st4yqhdxl3hpdtglta4do435wupahhx6q.b32.i2p:0
+tsl4dlpu2id252b6crbdnblruct664se6f2iw35fuqwa3te7wcoq.b32.i2p:0
+tugq6wa2ls2bv27pr2iy3da3k5ow3fzefbcvjcr22uc7w5vmevja.b32.i2p:0
+usztavbib756k5vqggzgkyswoj6mttihjvp3c2pa642t2mb4pvsa.b32.i2p:0
+vgu6llqbyjphml25umd5ztvyxrxuplz2g74fzbx75g3kkaetoyiq.b32.i2p:0
+wjrul5jwwb4vqdmkkrjbmly7osj6amecdpsac5xvaoqrti4nb3ha.b32.i2p:0
+wvktcp7hy4l6immhi5cxyz2dlsbhhvtcmskjemrnqehacnoap23q.b32.i2p:0
 wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:0
+xlqndzjoe5nr2nsxo6xwibh44ghyz4jfqevu62xykvemextpmjbq.b32.i2p:0
+yc4xwin5ujenvcr6ynwkz7lnmmq3nmzxvfguele6ovqqpxgjvonq.b32.i2p:0
+zdoabsg7ugzothyawodjhq54nvlofa746rxfkxpnjzj6nukmha6a.b32.i2p:0
 zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:0
+zysrlpii5ftrzivfcyhdrwpeyyqddbrdefnfu5q6otk5gtugmh2a.b32.i2p:0
 
 # manually added 2022-01 for minimal cjdns bootstrap support
 [fc32:17ea:e415:c3bf:9808:149d:b5a2:c9aa]:8333

contrib/seeds/nodes_test.txt

@@ -1,16 +1,89 @@
 # List of fixed seed nodes for testnet
 
-# Onion nodes
-35k2va6vyw4oo5ly2quvcszgdqr56kcnfgcqpnpcffut4jn3mhhwgbid.onion:18333
-blo2esfvk2rr7sr4jspmu3vt2vpgr5rigflsj645fnku7v4qmljurtid.onion:18333
-fuckcswupr5rmlvx2kqqrrosxvjyong4hatmuvxsvtcwe4dsh5rus7qd.onion:18333
-gblylyacjlitd2ywdmo2qqylwtdky7kgeqfvlhiw4zdag4x62tx54hyd.onion:18333
-gzwpduv33l7yze3bcdzj3inebiyjwddjnwvnjhh5wvnv4me76mjt2kad.onion:18333
-h3rphzofxzq52tb63mg5f6kc4my3fkcrgh3m5qryeatts43iljbawiid.onion:18333
-kf4qlhek34b3kgyxyodlmvgm4bxfrjsbjtgayyaiuyhr2eoyfgtm3bad.onion:18333
+# Onion nodes, last verified 2022-08 for minimal torv3 bootstrap support
+24j74ahq6ed4wmfrghdwroyfzimlkhnrb7zh4zw3vl2allzxbjrhaqid.onion:18333
+2fy74te65gm3c3gv3u5mhwdudvbdfh6k5fdz4gduimrltjjrxftbxrqd.onion:18333
+2lsncqdflwk272dhydrxf7ikfy23ppnmm54dnynyxiym6lqf3wowrmqd.onion:18333
+33o6qaidta7s2pmltet6vynd337vamgcifhh44rehwwxqpflcjt2njid.onion:18333
+3oo6bsc5mvf6a6ypmoaikilta6ka7mbdhdwhrnqhuhjlbaxyedvfvaqd.onion:18333
+3pe3fyklipy4sppkkgnhc22kcxtt57uler5kv72t676bbrwmcseo5qad.onion:18333
+4u4mcz2sfvxs7pwcwncswgmmcdzqtzjx7ztfo332jv4pqucb22ikdhad.onion:18333
+5v3i2kfqiqwp75gznjoptss7qgrcgseceqxpzpqkd34qeqzrg726i7id.onion:18333
+5zlrxk6q24t4vz5k4ie7gtuasdjavhoelhinzimxbfhc77u7vafipsid.onion:18333
+67s3af64ehw7xnxv422axm7tns4d6kutrftc6bjq375n74q3kj4pp7ad.onion:18333
+6a4ony53julvnufo632ktgmwvhupz63wbdwx7n7qudjy32qyq6gm3bqd.onion:18333
+6ftyg3nhc6tn2hyzls6zfdsfbroczhkxtdqumqb5q4yafhy5rdpapbid.onion:18333
+7554uw5djruh34j5ddx3iprzgqgzypcjtptwoldymfbgoywqcw2wiwyd.onion:18333
+766lozlabxaqjpbqsvt6sn3c65n6gkwwhoxyvggj7nfwnmw4cpaoccad.onion:18333
+7blv5abnytdf47yvbhxmykprmvjryqob65i2jmdwq3rrajcn2iiysbqd.onion:18333
+7v2ja4igx4v5y2jr6jrr6gaxohjhlzhvgwe4avlraxchozf7ea3kruqd.onion:18333
+7zgbmtzxow2oevd5aaqtsormw7ujv4zprl3oi2355immhq4gk7cyw5ad.onion:18333
+adstabjz7ec2y3jt4w2dvummowzv7g6m2f3kajeejffuaz7ojwj6epqd.onion:18333
+aesy6tfufadkut6flu2bsqgnw2422ur2ynjalguxlzuzuktg3zehttqd.onion:18333
+alxo32b5edi3bn2e224qrgytgxxpic4knyipvpdvctfsrvcaiq5lgeyd.onion:18333
+aoeart34umoonvd2kbqr3bc4sweu6a4msh2gp4skyqvei3shzcxbgmyd.onion:18333
+aprzvj7hgctsde4mkj3ewq35gvykspjvkqiygg7bpnw5tkvse2n7rhid.onion:18333
+awpk6z3xghx6ozouhodcydaqtr6uzzbnw4creuix7mkupxoxlmhhspad.onion:18333
+ayynqazucyh2jd5rehcfggmhunqpdwzlbhzbqgy6lj4ctz2ocj7chpid.onion:18333
+b2ika53aqckv4gs7wmog3byrea2vfzm5p7ye33digcsmvvnpbyqmzoyd.onion:18333
+be7zx3hh6dlahorlvsrrgqm4oahfrgqm2tbwnbd4u53ntu5f765n6hyd.onion:18333
+bluk62wj24bsvdwh47muo54hhwsatkftiqxevt5kba7hstjoex6ueeyd.onion:18333
+bubm6fiopfzkxqrfx6vqpioe5ahlhyubz57ogsqqy4ha5pnngiqlh6id.onion:18333
+d3czabzjj57lgrsr5gawkjd7v3gznrqa7zyizqmk4lryascavmipnyad.onion:18333
+ddj4cuvb32ve5chtp6jattcdnnmxmpoofjthzi7thgxxht7yqoetj3yd.onion:18333
+dqhhlssfwmh3g6zhwxpcfbw64xz5rfikcglinbhoxv5ajv4qzicjyeid.onion:18333
+drthcyb4x4rdfekw5g7xjogxi7aqoluilgulbgwvsme3nw3oibvchbad.onion:18333
+dwb47cmqa2tjpmvjaear7gdcars2lez6niefhi4qf22qehtyta6577qd.onion:18333
+e7tkrf54ng3q5vcn5gn77zwjwm74lkfav4mwdux3pvon6yvqg3tf46qd.onion:18333
+etuymy47s3quepvdaoo72i5e5mc7uovrzu5m4jf5q6mwlwizoxy4xgid.onion:18333
+fbimesnyhzubbzqc3uaufzkbyfmnkxvypoxaveaub7rzpzh2foxrn2yd.onion:18333
+fzbrwmgwmko7quelrhfuskt3ijabac76zx7g52dfrevmhdkj6ivh7qyd.onion:18333
+gy6nih4pmp5esyvvnhlj6qvk7zkbjuoswkxffyiip3dbkvsfxwz5zcqd.onion:18333
+ha62ziqzqdogd75zg7lfh4fqrg3bim3cpqzyupo43w5pw4fen6nr2pyd.onion:18333
+hacjjgj2mbqqrthzimmi6anvin7dljjhfl3ik6ebg3w3nmgsvr3ymmqd.onion:18333
+hbkp5xwpqo4qm75kpglfrclyiuuvdgv7mtiqfys7oqks4dmpqgpeoeid.onion:18333
+hqgoy62hoqjmz37brdfvoeov3cix5fixbqjoert4ydr6herg5oc3iwyd.onion:18333
+hvbmmzvqrpgps2x5u4ip4ksf3e5m2fneac754gtnhjn2rsevni6cz3ad.onion:18333
+hw3vzp32w4h6giplue6ix445oi6wt7gmeksrznb7tdfwhkgit7gnbbad.onion:18333
+iddr66ewkhenivapgianudjkwqcp6dxtssg7ixrdot5az6uh7m5tmjqd.onion:18333
+imya36iexiiiqrkwuxxcehnv4kg5shtirwd2vg4cnjy6lfjlph3fusqd.onion:18333
+iuhhuocns7entrzlxsxktyz2ibs7hqgiggv6sauzqkzka6laslwz7oqd.onion:18333
+ji5wmshokuc63eiulzlwj2zdvnligvrwfvvc76bice3tu43wfzvpmkyd.onion:18333
+jjfuyj7krgzkmpxvn3b2j2hwlzkmze3ezy3ifwk7dnswwawgmzqhjrqd.onion:18333
+jn2p4sgfphkxpow7kjrubrbqat77kkibzqkvuwhxyalcrazwmcqeaqyd.onion:18333
+jrveyz4us6sog6e6czsvr5mvvhgzjgv4idbe4idrolmqeudvt5a2dgid.onion:18333
+jsc4frvvnl2d3bhzyofsc72xpztgm23nl4fnb4dwkzsxr6fhij2q5iyd.onion:18333
+klymxdvje7kccv3tznabo3udopsftkmjemkbi2urqxjm4hefaudejjyd.onion:18333
+kwjxlauwjtecjfsiwopbl5pvn5n6z5rz76uk6osmlurd3uyuymcw7aid.onion:18333
+lc7upz2srw2yhpcvwg4afy64ylcoo6mfwlttqj5ovuglqnhnohpi5iqd.onion:18333
+lf3mpxfyjuovcqdvinl52pvdmmda6xqyfeiarlfamdjpgy3ouzmmlbyd.onion:18333
 mc7k47ndjvvhcgs54wmjzxvate4rtuybbjoryikdssjhcxlx27psbyqd.onion:18333
-mrhiniicugfo7mgrwv3wtolk3tptlcw2uq7ih6sq43fa4k4zbilut3yd.onion:18333
-uiudyws3qizgmepfoh7wwjmsoxoxut4qrmotjjhrn247xnjopr7sfcid.onion:18333
-zc2wvoqcezcrf64trji6jmhtss34a5ds5ntzdhqegzvex3ynrd7nxcad.onion:18333
-zd5m3dgdn46naj36pxvvcalfw2paecle6sdxq64ptwxtxjomkywpklqd.onion:18333
-
+mjbg3ggeuelmc7ixty3zjccyo2urg2uyherfqe7ytkm2ejkwlec7h6ad.onion:18333
+nkyqozv6kdwi423s7s2mezzguf5bafot2a3hv4ed2dbvtblisdmad4qd.onion:18333
+nvvqo4xxiwgb3y246jmcbuuveurfdq2zs3a5y7veqkeqv5jfhang7gyd.onion:18333
+o6vfovqxz3oxszfppczpjejwouobztjrgvfojc3emvhan3bkyskzhuad.onion:18333
+oaiw2lnhzgp5ry7ivzneuufmh7lfploquu2rjv5rozmlbefedsnxe5qd.onion:18333
+oln7ybci53wk4g5n42nipyixvyjxbludsbrfsmhnirb6tk7ovlikd5id.onion:18333
+otmfnhc6wrrbf2tpdy6zkisqc3r3urnsuowsnmatoto6yixaocnkseid.onion:18333
+ovc6sajbqfcbwv3wrq7ylklu6q6prvisz4jr4lyycn4kgukzjfe4mjad.onion:18333
+pm57didyzg5ljuvn5ufr5uun2iencuk3af2gzqc5zvgfh452c3rxtjyd.onion:18333
+pmismhpwug34gnqzbutranvx2wjwbshyqj4un2dyzyuvak2eh55psfyd.onion:18333
+polarisultijjhaku6z6u7jyboho5epdsg44ttebfaxmgau2z5sqolad.onion:18333
+qe2jbe447he6panfvpyqhyntf7346gmuf55bxrmdzggmgwyjsyknhxyd.onion:18333
+qz6yd5lsgdajcteoareeptwnipxsezyx5kks6ukpk5tvqinffzunqmyd.onion:18333
+rp6pn3b3oesyr2giolbysbjhqeugxntsu7crnkth4y33ok4zvcl7yrqd.onion:18333
+ujdchuw3hz5gkbouiv4p6pwbfdn7v4k6gluwvd4wiukqc7y7ow754uad.onion:18333
+vctlwaqgmu53eutz2hewuakcipfgtyljsd7czut4dd62xr3rp6fqezad.onion:18333
+vf5ur53tzmdtotvkndcgochklnuav7quqjvkc6mctqfvef6wnmn26mid.onion:18333
+wnxgjgjgplv5iu4mssyuunycvku4qnqr5t4q6cfdt47k7uwrfifuirad.onion:18333
+wpkbkdr7clw7zk3jkwiult6bf422j54u77ml4rgig2xq7icogyrcspid.onion:18333
+wzpdt24tdark26eugredddorik3tqwcj5ialtt2yim4ceiuiq7phkyqd.onion:18333
+xgapnikkbldoggjh5ewxkyauhuwnvf3xkspxroe3ojvfrk4lswkyx5yd.onion:18333
+xkvzdhcirontixbq6pjhru57bf4sgtqylvphk25csfrsy5p5ay3oc3yd.onion:18333
+xnipauenw5wnjb2zbx6v6umgvbb3g6xhf5kjo7pnyn5tdzvzaxtzicid.onion:18333
+yda7kwpii33j2qpq32ftf6lp22znknswipjwaccvsqj7l337jvfesnid.onion:18333
+z3j5foswuhpmtrg3kb56stkzmuoaesvd5jz3eztq46c4cidapglcyuad.onion:18333
+zcep44k7unwjm2wxty4ijh2e4fv5zgbrvwlctzyaqnrqhltjfzrtodad.onion:18333
+zmvizz7fd5hdue6wt3lwqumd6qwt4ijymmmotfzh75curq3mzjm53hyd.onion:18333
+zoaa3x7quyuijggii5zl4uyeioodudsgtr2uyv2qtdsslac5ukiwlxid.onion:18333
+zovauxlorl5eswumbsoxv2m5y3sm3qlk7657dcpr2uld7xf35en46sqd.onion:18333

contrib/seeds/suspicious_hosts.txt

@@ -1,16 +0,0 @@
-130.211.129.106
-148.251.238.178
-176.9.46.6
-178.63.107.226
-54.173.72.127
-54.174.10.182
-54.183.64.54
-54.194.231.211
-54.66.214.167
-54.66.220.137
-54.67.33.14
-54.77.251.214
-54.94.195.96
-54.94.200.247
-83.81.130.26
-88.198.17.7

contrib/signet/getcoins.py

@@ -129,7 +129,7 @@ if args.captcha != '': # Retrieve a captcha
 
     # Convert SVG image to PPM, and load it
     try:
-        rv = subprocess.run([args.imagemagick, 'svg:-', '-depth', '8', 'ppm:-'], input=res.content, check=True, capture_output=True)
+        rv = subprocess.run([args.imagemagick, 'svg:-', '-depth', '8', 'ppm:-'], input=res.content, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     except FileNotFoundError:
         raise SystemExit(f"The binary {args.imagemagick} could not be found. Please make sure ImageMagick (or a compatible fork) is installed and that the correct path is specified.")
 

contrib/signet/miner

@@ -4,7 +4,6 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 import argparse
-import base64
 import json
 import logging
 import math
@@ -15,14 +14,13 @@ import sys
 import time
 import subprocess
 
-from io import BytesIO
-
 PATH_BASE_CONTRIB_SIGNET = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
 PATH_BASE_TEST_FUNCTIONAL = os.path.abspath(os.path.join(PATH_BASE_CONTRIB_SIGNET, "..", "..", "test", "functional"))
 sys.path.insert(0, PATH_BASE_TEST_FUNCTIONAL)
 
-from test_framework.blocktools import WITNESS_COMMITMENT_HEADER, script_BIP34_coinbase_height # noqa: E402
-from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_hex, deser_string, hash256, ser_compact_size, ser_string, ser_uint256, tx_from_hex, uint256_from_str # noqa: E402
+from test_framework.blocktools import get_witness_script, script_BIP34_coinbase_height # noqa: E402
+from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_binary, from_hex, ser_string, ser_uint256, tx_from_hex # noqa: E402
+from test_framework.psbt import PSBT, PSBTMap, PSBT_GLOBAL_UNSIGNED_TX, PSBT_IN_FINAL_SCRIPTSIG, PSBT_IN_FINAL_SCRIPTWITNESS, PSBT_IN_NON_WITNESS_UTXO, PSBT_IN_SIGHASH_TYPE # noqa: E402
 from test_framework.script import CScriptOp # noqa: E402
 
 logging.basicConfig(
@@ -34,99 +32,12 @@ SIGNET_HEADER = b"\xec\xc7\xda\xa2"
 PSBT_SIGNET_BLOCK = b"\xfc\x06signetb"    # proprietary PSBT global field holding the block being signed
 RE_MULTIMINER = re.compile("^(\d+)(-(\d+))?/(\d+)$")
 
-# #### some helpers that could go into test_framework
-
-# like from_hex, but without the hex part
-def FromBinary(cls, stream):
-    """deserialize a binary stream (or bytes object) into an object"""
-    # handle bytes object by turning it into a stream
-    was_bytes = isinstance(stream, bytes)
-    if was_bytes:
-        stream = BytesIO(stream)
-    obj = cls()
-    obj.deserialize(stream)
-    if was_bytes:
-        assert len(stream.read()) == 0
-    return obj
-
-class PSBTMap:
-    """Class for serializing and deserializing PSBT maps"""
-
-    def __init__(self, map=None):
-        self.map = map if map is not None else {}
-
-    def deserialize(self, f):
-        m = {}
-        while True:
-            k = deser_string(f)
-            if len(k) == 0:
-                break
-            v = deser_string(f)
-            if len(k) == 1:
-                k = k[0]
-            assert k not in m
-            m[k] = v
-        self.map = m
-
-    def serialize(self):
-        m = b""
-        for k,v in self.map.items():
-            if isinstance(k, int) and 0 <= k and k <= 255:
-                k = bytes([k])
-            m += ser_compact_size(len(k)) + k
-            m += ser_compact_size(len(v)) + v
-        m += b"\x00"
-        return m
-
-class PSBT:
-    """Class for serializing and deserializing PSBTs"""
-
-    def __init__(self):
-        self.g = PSBTMap()
-        self.i = []
-        self.o = []
-        self.tx = None
-
-    def deserialize(self, f):
-        assert f.read(5) == b"psbt\xff"
-        self.g = FromBinary(PSBTMap, f)
-        assert 0 in self.g.map
-        self.tx = FromBinary(CTransaction, self.g.map[0])
-        self.i = [FromBinary(PSBTMap, f) for _ in self.tx.vin]
-        self.o = [FromBinary(PSBTMap, f) for _ in self.tx.vout]
-        return self
-
-    def serialize(self):
-        assert isinstance(self.g, PSBTMap)
-        assert isinstance(self.i, list) and all(isinstance(x, PSBTMap) for x in self.i)
-        assert isinstance(self.o, list) and all(isinstance(x, PSBTMap) for x in self.o)
-        assert 0 in self.g.map
-        tx = FromBinary(CTransaction, self.g.map[0])
-        assert len(tx.vin) == len(self.i)
-        assert len(tx.vout) == len(self.o)
-
-        psbt = [x.serialize() for x in [self.g] + self.i + self.o]
-        return b"psbt\xff" + b"".join(psbt)
-
-    def to_base64(self):
-        return base64.b64encode(self.serialize()).decode("utf8")
-
-    @classmethod
-    def from_base64(cls, b64psbt):
-        return FromBinary(cls, base64.b64decode(b64psbt))
-
-# #####
-
 def create_coinbase(height, value, spk):
     cb = CTransaction()
     cb.vin = [CTxIn(COutPoint(0, 0xffffffff), script_BIP34_coinbase_height(height), 0xffffffff)]
     cb.vout = [CTxOut(value, spk)]
     return cb
 
-def get_witness_script(witness_root, witness_nonce):
-    commitment = uint256_from_str(hash256(ser_uint256(witness_root) + ser_uint256(witness_nonce)))
-    return b"\x6a" + CScriptOp.encode_op_pushdata(WITNESS_COMMITMENT_HEADER + ser_uint256(commitment))
-
 def signet_txs(block, challenge):
     # assumes signet solution has not been added yet so does not need
     # to be removed
@@ -163,11 +74,11 @@ def signet_txs(block, challenge):
 
 def do_createpsbt(block, signme, spendme):
     psbt = PSBT()
-    psbt.g = PSBTMap( {0: signme.serialize(),
+    psbt.g = PSBTMap( {PSBT_GLOBAL_UNSIGNED_TX: signme.serialize(),
                        PSBT_SIGNET_BLOCK: block.serialize()
                      } )
-    psbt.i = [ PSBTMap( {0: spendme.serialize(),
-                         3: bytes([1,0,0,0])})
+    psbt.i = [ PSBTMap( {PSBT_IN_NON_WITNESS_UTXO: spendme.serialize(),
+                         PSBT_IN_SIGHASH_TYPE: bytes([1,0,0,0])})
              ]
     psbt.o = [ PSBTMap() ]
     return psbt.to_base64()
@@ -179,10 +90,10 @@ def do_decode_psbt(b64psbt):
     assert len(psbt.tx.vout) == 1
     assert PSBT_SIGNET_BLOCK in psbt.g.map
 
-    scriptSig = psbt.i[0].map.get(7, b"")
-    scriptWitness = psbt.i[0].map.get(8, b"\x00")
+    scriptSig = psbt.i[0].map.get(PSBT_IN_FINAL_SCRIPTSIG, b"")
+    scriptWitness = psbt.i[0].map.get(PSBT_IN_FINAL_SCRIPTWITNESS, b"\x00")
 
-    return FromBinary(CBlock, psbt.g.map[PSBT_SIGNET_BLOCK]), ser_string(scriptSig) + scriptWitness
+    return from_binary(CBlock, psbt.g.map[PSBT_SIGNET_BLOCK]), ser_string(scriptSig) + scriptWitness
 
 def finish_block(block, signet_solution, grind_cmd):
     block.vtx[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER + signet_solution)
@@ -222,7 +133,7 @@ def generate_psbt(tmpl, reward_spk, *, blocktime=None):
     cbwit = CTxInWitness()
     cbwit.scriptWitness.stack = [ser_uint256(witnonce)]
     block.vtx[0].wit.vtxinwit = [cbwit]
-    block.vtx[0].vout.append(CTxOut(0, get_witness_script(witroot, witnonce)))
+    block.vtx[0].vout.append(CTxOut(0, bytes(get_witness_script(witroot, witnonce))))
 
     signme, spendme = signet_txs(block, signet_spk_bin)
 
@@ -627,5 +538,3 @@ def main():
 
 if __name__ == "__main__":
     main()
-
-

contrib/testgen/README.md

@@ -2,7 +2,7 @@
 
 Utilities to generate test vectors for the data-driven Bitcoin tests.
 
-Usage:
+To use inside a scripted-diff (or just execute directly):
 
-    PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json
-    PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json
+    ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json
+    ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json

contrib/testgen/base58.py

@@ -1,115 +0,0 @@
-# Copyright (c) 2012-2020 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-'''
-Bitcoin base58 encoding and decoding.
-
-Based on https://bitcointalk.org/index.php?topic=1026.0 (public domain)
-'''
-import hashlib
-
-# for compatibility with following code...
-class SHA256:
-    new = hashlib.sha256
-
-if str != bytes:
-    # Python 3.x
-    def ord(c):
-        return c
-    def chr(n):
-        return bytes( (n,) )
-
-__b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
-__b58base = len(__b58chars)
-b58chars = __b58chars
-
-def b58encode(v):
-    """ encode v, which is a string of bytes, to base58.
-    """
-    long_value = 0
-    for (i, c) in enumerate(v[::-1]):
-        if isinstance(c, str):
-            c = ord(c)
-        long_value += (256**i) * c
-
-    result = ''
-    while long_value >= __b58base:
-        div, mod = divmod(long_value, __b58base)
-        result = __b58chars[mod] + result
-        long_value = div
-    result = __b58chars[long_value] + result
-
-    # Bitcoin does a little leading-zero-compression:
-    # leading 0-bytes in the input become leading-1s
-    nPad = 0
-    for c in v:
-        if c == 0:
-            nPad += 1
-        else:
-            break
-
-    return (__b58chars[0]*nPad) + result
-
-def b58decode(v, length = None):
-    """ decode v into a string of len bytes
-    """
-    long_value = 0
-    for i, c in enumerate(v[::-1]):
-        pos = __b58chars.find(c)
-        assert pos != -1
-        long_value += pos * (__b58base**i)
-
-    result = bytes()
-    while long_value >= 256:
-        div, mod = divmod(long_value, 256)
-        result = chr(mod) + result
-        long_value = div
-    result = chr(long_value) + result
-
-    nPad = 0
-    for c in v:
-        if c == __b58chars[0]:
-            nPad += 1
-            continue
-        break
-
-    result = bytes(nPad) + result
-    if length is not None and len(result) != length:
-        return None
-
-    return result
-
-def checksum(v):
-    """Return 32-bit checksum based on SHA256"""
-    return SHA256.new(SHA256.new(v).digest()).digest()[0:4]
-
-def b58encode_chk(v):
-    """b58encode a string, with 32-bit checksum"""
-    return b58encode(v + checksum(v))
-
-def b58decode_chk(v):
-    """decode a base58 string, check and remove checksum"""
-    result = b58decode(v)
-    if result is None:
-        return None
-    if result[-4:] == checksum(result[:-4]):
-        return result[:-4]
-    else:
-        return None
-
-def get_bcaddress_version(strAddress):
-    """ Returns None if strAddress is invalid.  Otherwise returns integer version of address. """
-    addr = b58decode_chk(strAddress)
-    if addr is None or len(addr)!=21:
-        return None
-    version = addr[0]
-    return ord(version)
-
-if __name__ == '__main__':
-    # Test case (from http://gitorious.org/bitcoin/python-base58.git)
-    assert get_bcaddress_version('15VjRaDX9zpbA8LVnbrCAFzrVzN7ixHNsC') == 0
-    _ohai = 'o hai'.encode('ascii')
-    _tmp = b58encode(_ohai)
-    assert _tmp == 'DYB3oMS'
-    assert b58decode(_tmp, 5) == _ohai
-    print("Tests passed")

contrib/testgen/gen_key_io_test_vectors.py

@@ -1,21 +1,21 @@
 #!/usr/bin/env python3
-# Copyright (c) 2012-2021 The Bitcoin Core developers
+# Copyright (c) 2012-2022 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 '''
 Generate valid and invalid base58/bech32(m) address and private key test vectors.
-
-Usage:
-    PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json
-    PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json
 '''
-# 2012 Wladimir J. van der Laan
-# Released under MIT License
-import os
+
 from itertools import islice
-from base58 import b58encode_chk, b58decode_chk, b58chars
+import os
 import random
-from segwit_addr import bech32_encode, decode_segwit_address, convertbits, CHARSET, Encoding
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '../../test/functional'))
+
+from test_framework.address import base58_to_byte, byte_to_base58, b58chars  # noqa: E402
+from test_framework.script import OP_0, OP_1, OP_2, OP_3, OP_16, OP_DUP, OP_EQUAL, OP_EQUALVERIFY, OP_HASH160, OP_CHECKSIG  # noqa: E402
+from test_framework.segwit_addr import bech32_encode, decode_segwit_address, convertbits, CHARSET, Encoding  # noqa: E402
 
 # key types
 PUBKEY_ADDRESS = 0
@@ -29,16 +29,6 @@ PRIVKEY_TEST = 239
 PRIVKEY_REGTEST = 239
 
 # script
-OP_0 = 0x00
-OP_1 = 0x51
-OP_2 = 0x52
-OP_3 = 0x53
-OP_16 = 0x60
-OP_DUP = 0x76
-OP_EQUAL = 0x87
-OP_EQUALVERIFY = 0x88
-OP_HASH160 = 0xa9
-OP_CHECKSIG = 0xac
 pubkey_prefix = (OP_DUP, OP_HASH160, 20)
 pubkey_suffix = (OP_EQUALVERIFY, OP_CHECKSIG)
 script_prefix = (OP_HASH160, 20)
@@ -114,8 +104,10 @@ def is_valid(v):
     '''Check vector v for validity'''
     if len(set(v) - set(b58chars)) > 0:
         return is_valid_bech32(v)
-    result = b58decode_chk(v)
-    if result is None:
+    try:
+        payload, version = base58_to_byte(v)
+        result = bytes([version]) + payload
+    except ValueError:  # thrown if checksum doesn't match
         return is_valid_bech32(v)
     for template in templates:
         prefix = bytearray(template[0])
@@ -135,18 +127,19 @@ def is_valid_bech32(v):
 def gen_valid_base58_vector(template):
     '''Generate valid base58 vector'''
     prefix = bytearray(template[0])
-    payload = bytearray(os.urandom(template[1]))
+    payload = rand_bytes(size=template[1])
     suffix = bytearray(template[2])
     dst_prefix = bytearray(template[4])
     dst_suffix = bytearray(template[5])
-    rv = b58encode_chk(prefix + payload + suffix)
+    assert len(prefix) == 1
+    rv = byte_to_base58(payload + suffix, prefix[0])
     return rv, dst_prefix + payload + dst_suffix
 
 def gen_valid_bech32_vector(template):
     '''Generate valid bech32 vector'''
     hrp = template[0]
     witver = template[1]
-    witprog = bytearray(os.urandom(template[2]))
+    witprog = rand_bytes(size=template[2])
     encoding = template[4]
     dst_prefix = bytearray(template[5])
     rv = bech32_encode(encoding, hrp, [witver] + convertbits(witprog, 8, 5))
@@ -176,21 +169,22 @@ def gen_invalid_base58_vector(template):
     corrupt_suffix = randbool(0.2)
 
     if corrupt_prefix:
-        prefix = os.urandom(1)
+        prefix = rand_bytes(size=1)
     else:
         prefix = bytearray(template[0])
 
     if randomize_payload_size:
-        payload = os.urandom(max(int(random.expovariate(0.5)), 50))
+        payload = rand_bytes(size=max(int(random.expovariate(0.5)), 50))
     else:
-        payload = os.urandom(template[1])
+        payload = rand_bytes(size=template[1])
 
     if corrupt_suffix:
-        suffix = os.urandom(len(template[2]))
+        suffix = rand_bytes(size=len(template[2]))
     else:
         suffix = bytearray(template[2])
 
-    val = b58encode_chk(prefix + payload + suffix)
+    assert len(prefix) == 1
+    val = byte_to_base58(payload + suffix, prefix[0])
     if random.randint(0,10)<1: # line corruption
         if randbool(): # add random character to end
             val += random.choice(b58chars)
@@ -206,7 +200,7 @@ def gen_invalid_bech32_vector(template):
     to_upper = randbool(0.1)
     hrp = template[0]
     witver = template[1]
-    witprog = bytearray(os.urandom(template[2]))
+    witprog = rand_bytes(size=template[2])
     encoding = template[3]
 
     if no_data:
@@ -236,6 +230,9 @@ def randbool(p = 0.5):
     '''Return True with P(p)'''
     return random.random() < p
 
+def rand_bytes(*, size):
+    return bytearray(random.getrandbits(8) for _ in range(size))
+
 def gen_invalid_vectors():
     '''Generate invalid test vectors'''
     # start with some manual edge-cases
@@ -250,9 +247,9 @@ def gen_invalid_vectors():
                 yield val,
 
 if __name__ == '__main__':
-    import sys
     import json
     iters = {'valid':gen_valid_vectors, 'invalid':gen_invalid_vectors}
+    random.seed(42)
     try:
         uiter = iters[sys.argv[1]]
     except IndexError:

contrib/valgrind.supp

@@ -64,12 +64,6 @@
    ...
    obj:*/libdb_cxx-*.so
 }
-{
-   Suppress leaks on init
-   Memcheck:Leak
-   ...
-   fun:_Z11AppInitMainR11NodeContext
-}
 {
    Suppress leaks on shutdown
    Memcheck:Leak
@@ -82,21 +76,6 @@
    ...
    obj:/usr/lib64/libgdk-3.so.0.2404.7
 }
-{
-   Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113
-   Memcheck:Leak
-   match-leak-kinds: reachable
-   fun:_Znwm
-   fun:_ZN7leveldbL10InitModuleEv
-}
-{
-   Suppress leveldb warning (leveldb::Env::Default()) - https://github.com/google/leveldb/issues/113
-   Memcheck:Leak
-   match-leak-kinds: reachable
-   fun:_Znwm
-   ...
-   fun:_ZN7leveldbL14InitDefaultEnvEv
-}
 {
    Suppress leveldb leak
    Memcheck:Leak
@@ -112,16 +91,6 @@
    ...
    fun:GetCoin
 }
-{
-   Suppress boost warning
-   Memcheck:Leak
-   fun:_Znwm
-   ...
-   fun:_ZN5boost9unit_test9framework5state17execute_test_treeEmjPKNS2_23random_generator_helperE
-   fun:_ZN5boost9unit_test9framework3runEmb
-   fun:_ZN5boost9unit_test14unit_test_mainEPFbvEiPPc
-   fun:main
-}
 {
    Suppress LogInstance still reachable memory warning
    Memcheck:Leak

contrib/verify-commits/trusted-git-root

@@ -1 +1 @@
-82bcf405f6db1d55b684a1f63a4aabad376cdad7
+577bd51a4b8de066466a445192c1c653872657e2

contrib/verify-commits/trusted-keys

@@ -1,8 +1,6 @@
 71A3B16735405025D447E8F274810B012346C9A6
-133EAC179436F14A5CF1B794860FEB804E669320
-32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC
 B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B
-CA03882CB1FC067B5D3ACFE4D300116E1C875A3D
 E777299FC265DD04793070EB944D35F9AC3DB76A
 D1DBF2C4B96F2DEBF4C16654410108112E7EA81F
 152812300785C96444D3334D17565732E08E5E41
+6B002C6EA3F91B1B0DF0C9BC8F617F1200A6D25C

contrib/verify-commits/verify-commits.py

@@ -82,11 +82,16 @@ def main():
     # get directory of this program and read data files
     dirname = os.path.dirname(os.path.abspath(__file__))
     print("Using verify-commits data from " + dirname)
-    verified_root = open(dirname + "/trusted-git-root", "r", encoding="utf8").read().splitlines()[0]
-    verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8").read().splitlines()[0]
-    revsig_allowed = open(dirname + "/allow-revsig-commits", "r", encoding="utf-8").read().splitlines()
-    unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf-8").read().splitlines()
-    incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf-8").read().splitlines()
+    with open(dirname + "/trusted-git-root", "r", encoding="utf8") as f:
+        verified_root = f.read().splitlines()[0]
+    with open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8") as f:
+        verified_sha512_root = f.read().splitlines()[0]
+    with open(dirname + "/allow-revsig-commits", "r", encoding="utf8") as f:
+        revsig_allowed = f.read().splitlines()
+    with open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf8") as f:
+        unclean_merge_allowed = f.read().splitlines()
+    with open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf8") as f:
+        incorrect_sha512_allowed = f.read().splitlines()
 
     # Set commit and branch and set variables
     current_commit = args.commit

depends/Makefile

@@ -42,8 +42,12 @@ NO_UPNP ?=
 NO_USDT ?=
 NO_NATPMP ?=
 MULTIPROCESS ?=
+LTO ?=
 FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources
 
+C_STANDARD ?= c11
+CXX_STANDARD ?= c++17
+
 BUILD = $(shell ./config.guess)
 HOST ?= $(BUILD)
 PATCHES_PATH = $(BASEDIR)/patches
@@ -140,8 +144,8 @@ include packages/packages.mk
 #     2. Before including packages/*.mk (excluding packages/packages.mk), since
 #        they rely on the build_id variables
 #
-build_id:=$(shell env CC='$(build_CC)' CXX='$(build_CXX)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
-$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' CXX='$(host_CXX)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
+build_id:=$(shell env CC='$(build_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(build_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
+$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(host_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
 
 qrencode_packages_$(NO_QR) = $(qrencode_$(host_os)_packages)
 
@@ -222,6 +226,9 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
             -e 's|@RANLIB@|$(host_RANLIB)|' \
             -e 's|@NM@|$(host_NM)|' \
             -e 's|@STRIP@|$(host_STRIP)|' \
+            -e 's|@OTOOL@|$(host_OTOOL)|' \
+            -e 's|@INSTALL_NAME_TOOL@|$(host_INSTALL_NAME_TOOL)|' \
+            -e 's|@DSYMUTIL@|$(host_DSYMUTIL)|' \
             -e 's|@build_os@|$(build_os)|' \
             -e 's|@host_os@|$(host_os)|' \
             -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \
@@ -239,6 +246,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
             -e 's|@no_usdt@|$(NO_USDT)|' \
             -e 's|@no_natpmp@|$(NO_NATPMP)|' \
             -e 's|@multiprocess@|$(MULTIPROCESS)|' \
+            -e 's|@lto@|$(LTO)|' \
             -e 's|@debug@|$(DEBUG)|' \
             $< > $@
 	touch $@
@@ -271,7 +279,7 @@ clean-all: clean
 	@rm -rf $(SOURCES_PATH) x86_64* i686* mips* arm* aarch64* powerpc* riscv32* riscv64* s390x*
 
 clean:
-	@rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD)
+	@rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD) *.log
 
 install: check-packages $(host_prefix)/share/config.site
 

depends/README.md

@@ -96,6 +96,8 @@ The following can be set when running make: `make FOO=bar`
 - `BASE_CACHE`: Built packages will be placed here
 - `SDK_PATH`: Path where SDKs can be found (used by macOS)
 - `FALLBACK_DOWNLOAD_PATH`: If a source file can't be fetched, try here before giving up
+- `C_STANDARD`: Set the C standard version used. Defaults to `c11`.
+- `CXX_STANDARD`: Set the C++ standard version used. Defaults to `c++17`.
 - `NO_QT`: Don't download/build/cache Qt and its dependencies
 - `NO_QR`: Don't download/build/cache packages needed for enabling qrencode
 - `NO_ZMQ`: Don't download/build/cache packages needed for enabling ZeroMQ
@@ -113,7 +115,11 @@ The following can be set when running make: `make FOO=bar`
 - `BUILD_ID_SALT`: Optional salt to use when generating build package ids
 - `FORCE_USE_SYSTEM_CLANG`: (EXPERTS ONLY) When cross-compiling for macOS, use Clang found in the
   system's `$PATH` rather than the default prebuilt release of Clang
-  from llvm.org. Clang 8 or later is required.
+  from llvm.org. Clang 8 or later is required
+- `LOG`: Use file-based logging for individual packages. During a package build its log file
+  resides in the `depends` directory, and the log file is printed out automatically in case
+  of build error. After successful build log files are moved along with package archives
+- `LTO`: Use LTO when building packages.
 
 If some packages are not built, for example `make NO_WALLET=1`, the appropriate
 options will be passed to bitcoin's configure. In this case, `--disable-wallet`.

depends/builders/darwin.mk

@@ -6,6 +6,7 @@ build_darwin_STRIP:=$(shell xcrun -f strip)
 build_darwin_OTOOL:=$(shell xcrun -f otool)
 build_darwin_NM:=$(shell xcrun -f nm)
 build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool)
+build_darwin_DSYMUTIL:=$(shell xcrun -f dsymutil)
 build_darwin_SHA256SUM=shasum -a 256
 build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o
 
@@ -19,6 +20,7 @@ darwin_LIBTOOL:=$(shell xcrun -f libtool)
 darwin_OTOOL:=$(shell xcrun -f otool)
 darwin_NM:=$(shell xcrun -f nm)
 darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool)
+darwin_DSYMUTIL:=$(shell xcrun -f dsymutil)
 darwin_native_binutils=
 darwin_native_toolchain=