Merge bitcoin/bitcoin#22858: [0.21] Final changes for 0.21.2

6c102a696e release: bump RC to 0 (-final) (fanquake)
f78570ef87 doc: mention bech32m/BIP350 in doc/descriptors.md (Pieter Wuille)

Pull request description:

  Backports:
  * #22837 - doc: mention bech32m/BIP350 in doc/descriptors.md

ACKs for top commit:
  laanwj:
    ACK 6c102a696e
  glozow:
    ACK 6c102a696e looks correct to me
  jnewbery:
    ACK 6c102a696e

Tree-SHA512: 8cdfee955e1abdbc6900df3d01982cb77ec57ef008b75487ac9a63d571cc21e4ae0f107849aa575d239a8662619553778955ffbebf57201de5afcd34298d57b4

.appveyor.yml

@@ -1,70 +1,63 @@
 version: '{branch}.{build}'
 skip_tags: true
-image: Visual Studio 2017
+image: Visual Studio 2019
 configuration: Release
 platform: x64
 clone_depth: 5
 environment:
-  APPVEYOR_SAVE_CACHE_ON_ERROR: true
-  CLCACHE_SERVER: 1
-  PACKAGES: berkeleydb boost-filesystem boost-signals2 boost-test libevent openssl rapidcheck zeromq double-conversion
   PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%'
   PYTHONUTF8: 1
-  QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.0/Qt5.9.7_ssl_x64_static_vs2017.zip'
-  QT_DOWNLOAD_HASH: 'D4D35B8112302B67E5610A03421BB3E43FE13F14D9A5F637C22AE60DCEC0E0F5'
-  QT_LOCAL_PATH: 'C:\Qt5.9.7_ssl_x64_static_vs2017'
-cache:
-- C:\tools\vcpkg\installed
-- C:\Users\appveyor\clcache -> .appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in
-- C:\Qt5.9.7_ssl_x64_static_vs2017
+  QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/qt598x64_vs2019_v1681/qt598_x64_vs2019_1681.zip'
+  QT_DOWNLOAD_HASH: '00cf7327818c07d74e0b1a4464ffe987c2728b00d49d4bf333065892af0515c3'
+  QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019'
+  VCPKG_TAG: '75522bb1f2e7d863078bcd06322348f053a9e33f'
 install:
-- cmd: pip install --quiet git+https://github.com/frerich/clcache.git@v4.2.0
 # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes.
 # - cmd: pip install zmq
-- cmd: echo set(VCPKG_BUILD_TYPE release) >> C:\tools\vcpkg\triplets\%PLATFORM%-windows-static.cmake
-- cmd: vcpkg remove --outdated --recurse
-- cmd: vcpkg install --triplet %PLATFORM%-windows-static %PACKAGES% > NUL
-before_build:
-- ps:  clcache -M 536870912
+# The powershell block below is to set up vcpkg to install the c++ dependencies. The pseudo code is:
+#    a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary,
+#    b. Append a setting to the vcpkg cmake config file to only do release builds of dependencies (skipping deubg builds saves ~5 mins).
+# Note originally this block also installed the dependencies using 'vcpkg install'. Dependencies are now installed
+# as part of the msbuild command using vcpkg mainfests.
 - ps: |
-      if(!(Test-Path -Path ($env:QT_LOCAL_PATH))) {
-        Write-Host "Downloading Qt binaries.";
-        Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip;
-        Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH...";
-        if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) {
-          Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH;
-          Write-Host "Qt binary download matched the expected hash.";
-        }
-        else {
-          Write-Host "ERROR: Qt binary download did not match the expected hash.";
-          Exit-AppveyorBuild;
-        }
+      cd c:\tools\vcpkg
+      $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead.
+      git -c advice.detachedHead=false checkout $env:VCPKG_TAG
+      .\bootstrap-vcpkg.bat > $null
+      Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)"
+      cd "$env:APPVEYOR_BUILD_FOLDER"
+before_build:
+# Powershell block below is to download and extract the Qt static libraries. The pseudo code is:
+#    a. Download the zip file with the prebuilt Qt static libraries.
+#    b. Check that the downloaded file matches the expected hash.
+#    c. Extract the zip file to the specific destination path expected by the msbuild projects.
+- ps: |
+      Write-Host "Downloading Qt binaries.";
+      Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip;
+      Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH...";
+      if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) {
+        Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH;
+        Write-Host "Qt binary download matched the expected hash.";
       }
       else {
-         Write-Host "Qt binaries already present.";
+        Write-Host "ERROR: Qt binary download did not match the expected hash.";
+        Exit-AppveyorBuild;
       }
 - cmd: python build_msvc\msvc-autogen.py
-- ps:  $files = (Get-ChildItem -Recurse | where {$_.extension -eq ".vcxproj"}).FullName
-- ps:  for (${i} = 0; ${i} -lt ${files}.length; ${i}++) {
-           ${content} = (Get-Content ${files}[${i}]);
-           ${content} = ${content}.Replace("</RuntimeLibrary>", "</RuntimeLibrary><DebugInformationFormat>None</DebugInformationFormat>");
-           ${content} = ${content}.Replace("<WholeProgramOptimization>true", "<WholeProgramOptimization>false");
-           Set-Content ${files}[${i}] ${content};
-       }
-- ps:  Start-Process clcache-server
-- ps:  fsutil behavior set disablelastaccess 0 # Enable Access time feature on Windows (for clcache)
 build_script:
-- cmd: msbuild /p:TrackFileAccess=false /p:CLToolExe=clcache.exe build_msvc\bitcoin.sln /m /v:n /nologo
+- cmd: msbuild /p:TrackFileAccess=false build_msvc\bitcoin.sln /m /v:q /nologo
 after_build:
-- ps:  fsutil behavior set disablelastaccess 1 # Disable Access time feature on Windows (better performance)
-- ps:  clcache -z
 #- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe
 test_script:
-- cmd: src\test_bitcoin.exe -k stdout -e stdout 2> NUL
-- cmd: src\bench_bitcoin.exe -evals=1 -scaling=0 > NUL
+- cmd: src\test_bitcoin.exe -l test_suite
+- cmd: src\bench_bitcoin.exe > NUL
 - ps:  python test\util\bitcoin-util-test.py
 - cmd: python test\util\rpcauth-test.py
-- cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast
+# Fee estimation test failing on appveyor with: WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted.
+# functional tests disabled for now. See
+# https://github.com/bitcoin/bitcoin/pull/18626#issuecomment-613396202
+# https://github.com/bitcoin/bitcoin/issues/18623
+# - cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation
 artifacts:
 #- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip
 deploy: off

.cirrus.yml

@@ -1,55 +1,146 @@
-task:
-  name: "FreeBsd 12.0 amd64  [GOAL: install]  [no depends, only system libs]"
-  freebsd_instance:
-    image: freebsd-12-0-release-amd64
-    cpu: 8
-    memory: 8G
-  timeout_in: 60m
-  env:
-    MAKEJOBS: "-j9"
-    CONFIGURE_OPTS: "--disable-dependency-tracking"
-    GOAL: "install"
-    TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
-    CCACHE_SIZE: "200M"
-    CCACHE_COMPRESS: 1
-    CCACHE_DIR: "/tmp/ccache_dir"
-  ccache_cache:
-    folder: "/tmp/ccache_dir"
-  install_script:
-    - pkg install -y autoconf automake boost-libs git gmake libevent libtool openssl pkgconf python3 ccache
-    - ./contrib/install_db4.sh $(pwd)
-    - ccache --max-size=${CCACHE_SIZE}
-  configure_script:
-    - ./autogen.sh
-    - ./configure ${CONFIGURE_OPTS} BDB_LIBS="-L$(pwd)/db4/lib -ldb_cxx-4.8" BDB_CFLAGS="-I$(pwd)/db4/include" || ( cat config.log && false)
-  make_script:
-    - gmake ${MAKEJOBS} ${GOAL} || ( echo "Build failure. Verbose build follows." && gmake ${GOAL} V=1 ; false )
-  check_script:
-    - gmake check ${MAKEJOBS} VERBOSE=1
-  functional_test_script:
-    - ./test/functional/test_runner.py --jobs 9 --ci --extended --exclude feature_dbcrash --combinedlogslen=1000 --quiet --failfast
-task:
-  name: "x86_64 Linux  [GOAL: install]  [bionic]  [Using ./ci/ system]"
-  container:
-    image: ubuntu:18.04
-    cpu: 8
-    memory: 8G
-  timeout_in: 60m
-  env:
-    MAKEJOBS: "-j9"
-    RUN_CI_ON_HOST: "1"
-    TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
-    CCACHE_SIZE: "200M"
-    CCACHE_DIR: "/tmp/ccache_dir"
+### Global defaults
+
+timeout_in: 120m  # https://cirrus-ci.org/faq/#instance-timed-out
+container:
+  # https://cirrus-ci.org/faq/#are-there-any-limits
+  # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel
+  cpu: 2
+  memory: 8G  # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers
+  kvm: true  # Use kvm to avoid spurious CI failures in the default virtualization cluster, see https://github.com/bitcoin/bitcoin/issues/20093
+env:
+  PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y"
+  MAKEJOBS: "-j4"
+  DANGER_RUN_CI_ON_HOST: "1"  # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system
+  TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
+  CCACHE_SIZE: "200M"
+  CCACHE_DIR: "/tmp/ccache_dir"
+
+### Global task template
+
+# https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks
+global_task_template: &GLOBAL_TASK_TEMPLATE
+  skip: $CIRRUS_REPO_FULL_NAME == "bitcoin-core/gui" && $CIRRUS_PR == ""  # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution
   ccache_cache:
     folder: "/tmp/ccache_dir"
   depends_built_cache:
     folder: "/tmp/cirrus-ci-build/depends/built"
   depends_sdk_cache:
     folder: "/tmp/cirrus-ci-build/depends/sdk-sources"
-  install_script:
-    - apt-get update
-    - apt-get -y install git bash ccache
-    - ccache --max-size=${CCACHE_SIZE}
+  depends_releases_cache:
+    folder: "/tmp/cirrus-ci-build/releases"
+  merge_base_script:
+    - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
+    - bash -c "$PACKAGE_MANAGER_INSTALL git"
+    - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH
+    - git config --global user.email "ci@ci.ci"
+    - git config --global user.name "ci"
+    - git merge FETCH_HEAD  # Merge base to detect silent merge conflicts
   ci_script:
     - ./ci/test_run_all.sh
+
+#task:
+#  name: "Windows"
+#  windows_container:
+#    image: cirrusci/windowsservercore:2019
+#  env:
+#    CIRRUS_SHELL: powershell
+#    PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%'
+#    PYTHONUTF8: 1
+#    QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip'
+#    QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21'
+#    QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019'
+#    VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed'
+#    VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74'
+#  install_script:
+#    - choco install python --version=3.7.7 -y
+
+task:
+  name: 'ARM  [GOAL: install]  [buster]  [unit tests, no functional tests]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: debian:buster
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_arm.sh"
+
+task:
+  name: 'Win64  [GOAL: deploy]  [unit tests, no gui, no boost::process, no functional tests]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:bionic
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_win64.sh"
+
+task:
+  name: 'x86_64 Linux  [GOAL: install]  [bionic]  [C++17, previous releases, uses qt5 dev package and some depends packages] [unsigned char]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:bionic
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh"
+
+task:
+  name: 'x86_64 Linux  [GOAL: install]  [focal]  [depends, sanitizers: thread (TSan), no gui]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+    cpu: 4  # Double CPU and Memory to avoid timeout
+    memory: 16G
+  env:
+    MAKEJOBS: "-j8"
+    FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
+
+task:
+  name: 'x86_64 Linux  [GOAL: install]  [focal]  [depends, sanitizers: memory (MSan)]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
+
+task:
+  name: 'x86_64 Linux  [GOAL: install]  [focal]  [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
+
+task:
+  name: 'x86_64 Linux  [GOAL: install]  [focal]  [no depends, only system libs, sanitizers: fuzzer,address,undefined]'
+  only_if: $CIRRUS_BRANCH == $CIRRUS_DEFAULT_BRANCH || $CIRRUS_BASE_BRANCH == $CIRRUS_DEFAULT_BRANCH
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
+
+task:
+  name: 'x86_64 Linux [GOAL: install]  [focal]  [multiprocess]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:focal
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_native_multiprocess.sh"
+
+task:
+  name: 'macOS 10.12  [GOAL: deploy] [no functional tests]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:bionic
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_mac.sh"
+
+task:
+  name: 'macOS 10.14 native [GOAL: install] [GUI] [no depends]'
+  brew_install_script:
+    - brew update
+    - brew install boost libevent berkeley-db4 qt@5 miniupnpc ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt
+  << : *GLOBAL_TASK_TEMPLATE
+  osx_instance:
+    # Use latest image, but hardcode version to avoid silent upgrades (and breaks)
+    image: catalina-xcode-12.1  # https://cirrus-ci.org/guide/macOS
+  env:
+    DANGER_RUN_CI_ON_HOST: "true"
+    CI_USE_APT_INSTALL: "no"
+    PACKAGE_MANAGER_INSTALL: "echo"  # Nothing to do
+    FILE_ENV: "./ci/test/00_setup_env_mac_host.sh"

.fuzzbuzz.yml

@@ -0,0 +1,16 @@
+base: ubuntu:16.04
+language: c++
+engine: libFuzzer
+environment:
+  - CXXFLAGS=-fcoverage-mapping -fno-omit-frame-pointer -fprofile-instr-generate -gline-tables-only -O1
+setup:
+  - sudo apt-get update
+  - sudo apt-get install -y autoconf bsdmainutils clang git libboost-all-dev libboost-program-options-dev libc++1 libc++abi1 libc++abi-dev libc++-dev libclang1 libclang-dev libdb5.3++ libevent-dev libllvm-ocaml-dev libomp5 libomp-dev libprotobuf-dev libqt5core5a libqt5dbus5 libqt5gui5 libssl-dev libtool llvm llvm-dev llvm-runtime pkg-config protobuf-compiler qttools5-dev qttools5-dev-tools software-properties-common
+  - ./autogen.sh
+  - CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined
+  - make
+  - git clone https://github.com/bitcoin-core/qa-assets
+auto_targets:
+  find_targets_command: find src/test/fuzz/ -executable -type f ! -name "*.cpp" ! -name "*.h"
+  base_corpus_dir: qa-assets/fuzz_seed_corpus/
+  memory_limit: none

.github/ISSUE_TEMPLATE.md

@@ -4,7 +4,11 @@ General bitcoin questions and/or support requests are best directed to the Bitco
 
 For reporting security issues, please read instructions at https://bitcoincore.org/en/contact/.
 
-If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running memtest and observe CPU temperature with a load-test tool such as linpack before creating an issue! -->
+If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running memtest and observe CPU temperature with a load-test tool such as linpack before creating an issue!
+
+Any report, issue or feature request related to the GUI should be reported at
+https://github.com/bitcoin-core/gui/issues/
+-->
 
 <!-- Describe the issue -->
 <!--- What behavior did you expect? -->

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,6 +1,6 @@
 ---
 name: Bug report
-about: Create a report to help us improve
+about: Create a report to help us improve (use this for suspected bugs only, if not sure, open a regular issue below)
 title: ''
 labels: Bug
 assignees: ''

.github/ISSUE_TEMPLATE/good_first_issue.md

@@ -0,0 +1,22 @@
+---
+name: Good first issue
+about: '(Regular devs only): Suggest a new good first issue'
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+<!-- Needs the label "good first issue" assigned manually before or after opening -->
+
+<!-- A good first issue is an uncontroversial issue, that has a relatively unique and obvious solution -->
+
+<!-- Motivate the issue and explain the solution briefly -->
+
+#### Useful skills:
+
+<!-- (For example, “C++11 std::thread”, “Qt5 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”.) -->
+
+#### Want to work on this issue?
+
+For guidance on contributing, please read [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md) before opening your pull request.

.github/ISSUE_TEMPLATE/gui_issue.md

@@ -0,0 +1,11 @@
+---
+name: An issue or feature request related to the GUI
+about: Any report, issue or feature request related to the GUI should be reported at https://github.com/bitcoin-core/gui/issues/
+title: Any report, issue or feature request related to the GUI should be reported at https://github.com/bitcoin-core/gui/issues/
+labels: GUI
+assignees: ''
+
+---
+
+Any report, issue or feature request related to the GUI should be reported at
+https://github.com/bitcoin-core/gui/issues/

.github/PULL_REQUEST_TEMPLATE.md

@@ -3,6 +3,10 @@
 
 Pull requests without a rationale and clear improvement may be closed
 immediately.
+
+GUI-related pull requests should be opened against
+https://github.com/bitcoin-core/gui
+first. See CONTRIBUTING.md
 -->
 
 <!--

.gitignore

@@ -5,10 +5,13 @@
 src/bitcoin
 src/bitcoind
 src/bitcoin-cli
+src/bitcoin-gui
+src/bitcoin-node
 src/bitcoin-tx
 src/bitcoin-wallet
+src/test/fuzz/*
+!src/test/fuzz/*.*
 src/test/test_bitcoin
-src/test/test_bitcoin_fuzzy
 src/qt/test/test_bitcoin-qt
 
 # autoreconf
@@ -28,6 +31,7 @@ build-aux/m4/ltversion.m4
 build-aux/missing
 build-aux/compile
 build-aux/test-driver
+config.cache
 config.log
 config.status
 configure
@@ -88,7 +92,7 @@ src/qt/bitcoin-qt.includes
 *.qm
 Makefile
 !depends/Makefile
-bitcoin-qt
+src/qt/bitcoin-qt
 Bitcoin-Qt.app
 background.tiff*
 
@@ -106,13 +110,19 @@ qrc_*.cpp
 .DS_Store
 build
 
+# Previous releases
+releases
+
 #lcov
 *.gcno
 *.gcda
 /*.info
 test_bitcoin.coverage/
 total.coverage/
+fuzz.coverage/
 coverage_percent.txt
+/cov_tool_wrapper.sh
+qa-assets/
 
 #build tests
 linux-coverage-build
@@ -120,6 +130,7 @@ linux-build
 win32-build
 test/config.ini
 test/cache/*
+test/.mypy_cache/
 
 !src/leveldb*/Makefile
 

.travis.yml

@@ -1,18 +1,4 @@
-# The test build matrix (stage: test) is constructed to test a wide range of
-# configurations, rather than a single pass/fail. This helps to catch build
-# failures and logic errors that present on platforms other than the ones the
-# author has tested.
-#
-# Some builders use the dependency-generator in `./depends`, rather than using
-# apt-get to install build dependencies. This guarantees that the tester is
-# using the same versions as Gitian, so the build results are nearly identical
-# to what would be found in a final release.
-#
-# In order to avoid rebuilding all dependencies for each build, the binaries
-# are cached and re-used when possible. Changes in the dependency-generator
-# will trigger cache-invalidation and rebuilds as necessary.
-#
-# These caches can be manually removed if necessary. This is one of the very
+# Travis caches can be manually removed if necessary. This is one of the very
 # few manual operations that is possible with Travis, and it can be done by a
 # Bitcoin Core GitHub member via the Travis web interface [0].
 #
@@ -25,37 +11,41 @@
 # [1] https://docs.travis-ci.com/user/caching/#build-phases
 # [2] https://docs.travis-ci.com/user/customizing-the-build#build-timeouts
 
-dist: xenial
+version: ~> 1.0
+
+dist: bionic
 os: linux
 language: minimal
+arch: amd64
 cache:
-  ccache: true
   directories:
     - $TRAVIS_BUILD_DIR/depends/built
     - $TRAVIS_BUILD_DIR/depends/sdk-sources
     - $TRAVIS_BUILD_DIR/ci/scratch/.ccache
+    - $TRAVIS_BUILD_DIR/releases/$HOST
 stages:
   - lint
   - test
-  - extended-lint
 env:
   global:
     - CI_RETRY_EXE="travis_retry"
-    - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. Please manually re-run this job by using the travis restart button or asking a bitcoin maintainer to restart. The next run should not time out because the build cache has been saved."
+    - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. See https://docs.travis-ci.com/user/customizing-the-build#build-timeouts . Please manually re-run this job by using the travis restart button. The next run should not time out because the build cache has been saved."
 before_install:
   - set -o errexit; source ./ci/test/00_setup_env.sh
   - set -o errexit; source ./ci/test/03_before_install.sh
 install:
   - set -o errexit; source ./ci/test/04_install.sh
 before_script:
-  - set -o errexit; source ./ci/test/05_before_script.sh
+  # Temporary workaround for https://github.com/bitcoin/bitcoin/issues/16368
+  - for i in {1..4}; do echo "$(sleep 500)" ; done &
+  - set -o errexit; source ./ci/test/05_before_script.sh &> "/dev/null"
 script:
   - export CONTINUE=1
   - if [ $SECONDS -gt 1200 ]; then export CONTINUE=0; fi  # Likely the depends build took very long
-  - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi  # Whitelisted repo (90 minutes build time)
+  - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi  # continue on repos with extended build time (90 minutes)
   - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi
-  - if [ $SECONDS -gt 2000 ]; then export CONTINUE=0; fi  # Likely the build took very long; The tests take about 1000s, so we should abort if we have less than 50*60-1000=2000s left
-  - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi  # Whitelisted repo (90 minutes build time)
+  - if [[ $SECONDS -gt 50*60-$EXPECTED_TESTS_DURATION_IN_SECONDS ]]; then export CONTINUE=0; fi
+  - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi  # continue on repos with extended build time (90 minutes)
   - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi
 after_script:
   - echo $TRAVIS_COMMIT_RANGE
@@ -65,7 +55,7 @@ jobs:
     - stage: lint
       name: 'lint'
       env:
-      cache: false
+      cache: pip
       language: python
       python: '3.5' # Oldest supported version according to doc/dependencies.md
       install:
@@ -75,66 +65,12 @@ jobs:
       script:
         - set -o errexit; source ./ci/lint/06_script.sh
 
-    - stage: extended-lint
-      name: 'extended lint [runtime >= 60 seconds]'
-      env:
-      cache: false
-      language: python
-      python: '3.5'
-      install:
-        - set -o errexit; source ./ci/extended_lint/04_install.sh
-      before_script:
-        - set -o errexit; source ./ci/lint/05_before_script.sh
-      script:
-        - set -o errexit; source ./ci/extended_lint/06_script.sh
+    - stage: test
+      name: '32-bit + dash  [GOAL: install]  [CentOS 7]  [gui]'
+      env: >-
+        FILE_ENV="./ci/test/00_setup_env_i686_centos.sh"
 
     - stage: test
-      name: 'ARM  [GOAL: install]  [no unit or functional tests]'
+      name: 'x86_64 Linux  [GOAL: install]  [xenial]  [no wallet]'
       env: >-
-        FILE_ENV="./ci/test/00_setup_env_arm.sh"
-
-    - stage: test
-      name: 'Win64  [GOAL: deploy]  [no gui or functional tests]'
-      env: >-
-        FILE_ENV="./ci/test/00_setup_env_win64.sh"
-
-    - stage: test
-      name: '32-bit + dash  [GOAL: install]  [GUI: BIP70 enabled]'
-      env: >-
-        FILE_ENV="./ci/test/00_setup_env_i686.sh"
-
-    - stage: test
-      name: 'x86_64 Linux  [GOAL: install]  [bionic]  [uses qt5 dev package instead of depends Qt to speed up build and avoid timeout] [unsigned char]'
-      env: >-
-        FILE_ENV="./ci/test/00_setup_env_amd64_qt5.sh"
-
-    - stage: test
-      name: 'x86_64 Linux  [GOAL: install]  [trusty]  [no functional tests, no depends, only system libs]'
-      env: >-
-        FILE_ENV="./ci/test/00_setup_env_amd64_trusty.sh"
-
-    - stage: test
-      name: 'x86_64 Linux  [GOAL: install]  [xenial]  [no depends, only system libs, sanitizers: thread (TSan), no wallet]'
-      env: >-
-        FILE_ENV="./ci/test/00_setup_env_amd64_tsan.sh"
-        TEST_RUNNER_EXTRA="--exclude feature_block"  # Not enough memory on travis machines
-
-    - stage: test
-      name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]'
-      env: >-
-        FILE_ENV="./ci/test/00_setup_env_amd64_asan.sh"
-
-    - stage: test
-      name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no depends, only system libs, sanitizers: fuzzer,address,undefined]'
-      env: >-
-        FILE_ENV="./ci/test/00_setup_env_amd64_fuzz.sh"
-
-    - stage: test
-      name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no wallet]'
-      env: >-
-        FILE_ENV="./ci/test/00_setup_env_amd64_nowallet.sh"
-
-    - stage: test
-      name: 'macOS 10.10  [GOAL: deploy] [no functional tests]'
-      env: >-
-        FILE_ENV="./ci/test/00_setup_env_mac.sh"
+        FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh"

.tx/config

@@ -1,7 +1,7 @@
 [main]
 host = https://www.transifex.com
 
-[bitcoin.qt-translation-019x]
+[bitcoin.qt-translation-021x]
 file_filter = src/qt/locale/bitcoin_<lang>.ts
 source_file = src/qt/locale/bitcoin_en.ts
 source_lang = en

CODEOWNERS

@@ -0,0 +1,136 @@
+# ==============================================================================
+#                          Bitcoin Core CODEOWNERS
+# ==============================================================================
+
+# Configuration of code ownership and review approvals for the bitcoin/bitcoin
+# repo.
+
+# Order is important; the last matching pattern takes the most precedence.
+# More info on how this file works can be found at:
+# https://help.github.com/articles/about-codeowners/
+
+# This file is called CODEOWNERS because it is a magic file for GitHub to
+# automatically suggest reviewers. In this project's case, the names below
+# should be thought of as code reviewers rather than owners. Regular
+# contributors are free to add their names to specific directories or files
+# provided that they are willing to provide a review when automatically
+# assigned.
+
+# Absence from this list should not be interpreted as a discouragement to
+# review a pull request. Peer review is always welcome and is a critical
+# component of the progress of the codebase. Information on peer review
+# guidelines can be found in the CONTRIBUTING.md doc.
+
+
+# Maintainers
+#   @laanwj
+#   @sipa
+#   @fanquake
+#   @jonasschnelli
+#   @marcofalke
+#   @meshcollider
+
+# Docs
+/doc/*[a-zA-Z-].md                          @harding
+/doc/Doxyfile.in                            @fanquake
+/doc/REST-interface.md                      @jonasschnelli
+/doc/benchmarking.md                        @ariard
+/doc/bitcoin-conf.md                        @hebasto
+/doc/build-freebsd.md                       @fanquake
+/doc/build-netbsd.md                        @fanquake
+/doc/build-openbsd.md                       @laanwj
+/doc/build-osx.md                           @fanquake
+/doc/build-unix.md                          @laanwj
+/doc/build-windows.md                       @sipsorcery
+/doc/dependencies.md                        @fanquake
+/doc/developer-notes.md                     @laanwj
+/doc/files.md                               @hebasto
+/doc/gitian-building.md                     @laanwj
+/doc/reduce-memory.md                       @fanquake
+/doc/reduce-traffic.md                      @jonasschnelli
+/doc/release-process.md                     @laanwj
+/doc/translation_strings_policy.md          @laanwj
+
+# Build aux
+/build-aux/m4/bitcoin_qt.m4                 @hebasto
+
+# MSVC build system
+/build_msvc/                                @sipsorcery
+
+# Settings
+/src/util/settings.*                        @ryanofsky
+
+# Fuzzing
+/src/test/fuzz/                             @practicalswift
+/doc/fuzzing.md                             @practicalswift
+
+# Test framework
+/test/functional/mempool_updatefromblock.py @hebasto
+/test/functional/feature_asmap.py           @jonatack
+/test/functional/interface_bitcoin_cli.py   @jonatack
+/test/functional/tool_wallet.py             @jonatack
+
+# Translations
+/src/util/translation.h                     @hebasto
+
+# Dev Tools
+/contrib/devtools/security-check.py         @fanquake
+/contrib/devtools/test-security-check.py    @fanquake
+/contrib/devtools/symbol-check.py           @fanquake
+
+# Gitian/Guix
+/contrib/gitian-build.py                    @hebasto
+/contrib/guix/                              @dongcarl
+
+# Compatibility
+/src/compat/glibc_*                         @fanquake
+
+# GUI
+/src/qt/forms/                              @hebasto
+
+# Wallet
+/src/wallet/                                @achow101
+
+# CLI
+/src/bitcoin-cli.cpp                        @jonatack
+
+# Coinstats
+/src/node/coinstats.*                       @fjahr
+
+# Index
+/src/index/                                 @fjahr
+
+# Descriptors
+*descriptor*                                @achow101 @sipa
+
+# Interfaces
+/src/interfaces/                            @ryanofsky
+
+# DB
+/src/txdb.*                                 @jamesob
+/src/dbwrapper.*                            @jamesob
+
+# Scripts/Linter
+*.sh                                        @practicalswift
+/test/lint/                                 @practicalswift
+/test/lint/lint-shell.sh                    @hebasto
+
+# Bech32
+/src/bech32.*                               @sipa
+/src/bench/bech32.*                         @sipa
+
+# PSBT
+/src/psbt*                                  @achow101
+/src/node/psbt*                             @achow101
+/doc/psbt.md                                @achow101
+
+# P2P
+/src/net_processing.*                       @sipa
+/src/protocol.*                             @sipa
+
+# Consensus
+/src/coins.*                                @sipa @jamesob
+/src/script/script.*                        @sipa
+/src/script/interpreter.*                   @sipa
+/src/validation.*                           @sipa
+/src/consensus/                             @sipa

CONTRIBUTING.md

@@ -6,34 +6,69 @@ welcome to contribute towards development in the form of peer review, testing
 and patches. This document explains the practical process and guidelines for
 contributing.
 
-Firstly in terms of structure, there is no particular concept of "Core
+First, in terms of structure, there is no particular concept of "Bitcoin Core
 developers" in the sense of privileged people. Open source often naturally
-revolves around meritocracy where longer term contributors gain more trust from
-the developer community. However, some hierarchy is necessary for practical
-purposes. As such there are repository "maintainers" who are responsible for
-merging pull requests as well as a "lead maintainer" who is responsible for the
-release cycle, overall merging, moderation and appointment of maintainers.
+revolves around a meritocracy where contributors earn trust from the developer
+community over time. Nevertheless, some hierarchy is necessary for practical
+purposes. As such, there are repository "maintainers" who are responsible for
+merging pull requests, as well as a "lead maintainer" who is responsible for the
+release cycle as well as overall merging, moderation and appointment of
+maintainers.
 
+Getting Started
+---------------
+
+New contributors are very welcome and needed.
+
+Reviewing and testing is highly valued and the most effective way you can contribute
+as a new contributor. It also will teach you much more about the code and
+process than opening pull requests. Please refer to the [peer review](#peer-review)
+section below.
+
+Before you start contributing, familiarize yourself with the Bitcoin Core build
+system and tests. Refer to the documentation in the repository on how to build
+Bitcoin Core and how to run the unit tests, functional tests, and fuzz tests.
+
+There are many open issues of varying difficulty waiting to be fixed.
 If you're looking for somewhere to start contributing, check out the
 [good first issue](https://github.com/bitcoin/bitcoin/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
-list.
+list or changes that are
+[up for grabs](https://github.com/bitcoin/bitcoin/issues?utf8=%E2%9C%93&q=label%3A%22Up+for+grabs%22).
+Some of them might no longer be applicable. So if you are interested, but
+unsure, you might want to leave a comment on the issue first.
+
+You may also participate in the weekly
+[Bitcoin Core PR Review Club](https://bitcoincore.reviews/) meeting.
+
+### Good First Issue Label
+
+The purpose of the `good first issue` label is to highlight which issues are
+suitable for a new contributor without a deep understanding of the codebase.
+
+However, good first issues can be solved by anyone. If they remain unsolved
+for a longer time, a frequent contributor might address them.
+
+You do not need to request permission to start working on an issue. However,
+you are encouraged to leave a comment if you are planning to work on it. This
+will help other contributors monitor which issues are actively being addressed
+and is also an effective way to request assistance if and when you need it.
 
 Communication Channels
 ----------------------
 
 Most communication about Bitcoin Core development happens on IRC, in the
-#bitcoin-core-dev channel on Freenode. The easiest way to participate on IRC is
+`#bitcoin-core-dev` channel on Freenode. The easiest way to participate on IRC is
 with the web client, [webchat.freenode.net](https://webchat.freenode.net/). Chat
 history logs can be found
 on [http://www.erisian.com.au/bitcoin-core-dev/](http://www.erisian.com.au/bitcoin-core-dev/)
 and [http://gnusha.org/bitcoin-core-dev/](http://gnusha.org/bitcoin-core-dev/).
 
-Discussion about code base improvements happens in GitHub issues and on pull
+Discussion about codebase improvements happens in GitHub issues and pull
 requests.
 
 The developer
 [mailing list](https://lists.linuxfoundation.org/mailman/listinfo/bitcoin-dev)
-should be used to discuss complicated or controversial changes before working on
+should be used to discuss complicated or controversial consensus or P2P protocol changes before working on
 a patch set.
 
 
@@ -41,22 +76,47 @@ Contributor Workflow
 --------------------
 
 The codebase is maintained using the "contributor workflow" where everyone
-without exception contributes patch proposals using "pull requests". This
+without exception contributes patch proposals using "pull requests" (PRs). This
 facilitates social contribution, easy testing and peer review.
 
 To contribute a patch, the workflow is as follows:
 
-  1. Fork repository
+  1. Fork repository ([only for the first time](https://help.github.com/en/articles/fork-a-repo))
   1. Create topic branch
   1. Commit patches
 
-The project coding conventions in the [developer notes](doc/developer-notes.md)
-must be adhered to.
+For GUI-related issues or pull requests, the https://github.com/bitcoin-core/gui repository should be used.
+For all other issues and pull requests, the https://github.com/bitcoin/bitcoin node repository should be used.
 
-In general [commits should be atomic](https://en.wikipedia.org/wiki/Atomic_commit#Atomic_commit_convention)
-and diffs should be easy to read. For this reason do not mix any formatting
+The master branch for all monotree repositories is identical.
+
+As a rule of thumb, everything that only modifies `src/qt` is a GUI-only pull
+request. However:
+
+* For global refactoring or other transversal changes the node repository
+  should be used.
+* For GUI-related build system changes, the node repository should be used
+  because the change needs review by the build systems reviewers.
+* Changes in `src/interfaces` need to go to the node repository because they
+  might affect other components like the wallet.
+
+For large GUI changes that include build system and interface changes, it is
+recommended to first open a pull request against the GUI repository. When there
+is agreement to proceed with the changes, a pull request with the build system
+and interfaces changes can be submitted to the node repository.
+
+The project coding conventions in the [developer notes](doc/developer-notes.md)
+must be followed.
+
+### Committing Patches
+
+In general, [commits should be atomic](https://en.wikipedia.org/wiki/Atomic_commit#Atomic_commit_convention)
+and diffs should be easy to read. For this reason, do not mix any formatting
 fixes or code moves with actual code changes.
 
+Make sure each individual commit is hygienic: that it builds successfully on its
+own without warnings, errors, regressions, or test failures.
+
 Commit messages should be verbose by default consisting of a short subject line
 (50 chars max), a blank line and detailed explanatory text as separate
 paragraph(s), unless the title alone is self-explanatory (like "Corrected typo
@@ -68,7 +128,7 @@ If a particular commit references another issue, please add the reference. For
 example: `refs #1234` or `fixes #4321`. Using the `fixes` or `closes` keywords
 will cause the corresponding issue to be closed when the pull request is merged.
 
-Commit messages should never contain any `@` mentions.
+Commit messages should never contain any `@` mentions (usernames prefixed with "@").
 
 Please refer to the [Git manual](https://git-scm.com/doc) for more information
 about Git.
@@ -76,6 +136,8 @@ about Git.
   - Push changes to your fork
   - Create pull request
 
+### Creating the Pull Request
+
 The title of the pull request should be prefixed by the component or area that
 the pull request affects. Valid areas as:
 
@@ -88,41 +150,53 @@ the pull request affects. Valid areas as:
   - `refactor` for structural changes that do not change behavior
   - `rpc`, `rest` or `zmq` for changes to the RPC, REST or ZMQ APIs
   - `script` for changes to the scripts and tools
-  - `test` for changes to the bitcoin unit tests or QA tests
+  - `test`, `qa` or `ci` for changes to the unit tests, QA tests or CI code
   - `util` or `lib` for changes to the utils or libraries
   - `wallet` for changes to the wallet code
-  - `build` for changes to the GNU Autotools, reproducible builds or CI code
+  - `build` for changes to the GNU Autotools or reproducible builds
 
 Examples:
 
     consensus: Add new opcode for BIP-XXXX OP_CHECKAWESOMESIG
-    net: Automatically create hidden service, listen on Tor
+    net: Automatically create onion service, listen on Tor
     qt: Add feed bump button
     log: Fix typo in log message
 
-Note that translations should not be submitted as pull requests, please see
+The body of the pull request should contain sufficient description of *what* the
+patch does, and even more importantly, *why*, with justification and reasoning.
+You should include references to any discussions (for example, other issues or
+mailing list discussions).
+
+The description for a new pull request should not contain any `@` mentions. The
+PR description will be included in the commit message when the PR is merged and
+any users mentioned in the description will be annoyingly notified each time a
+fork of Bitcoin Core copies the merge. Instead, make any username mentions in a
+subsequent comment to the PR.
+
+### Translation changes
+
+Note that translations should not be submitted as pull requests. Please see
 [Translation Process](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md)
 for more information on helping with translations.
 
+### Work in Progress Changes and Requests for Comments
+
 If a pull request is not to be considered for merging (yet), please
 prefix the title with [WIP] or use [Tasks Lists](https://help.github.com/articles/basic-writing-and-formatting-syntax/#task-lists)
 in the body of the pull request to indicate tasks are pending.
 
-The body of the pull request should contain enough description about what the
-patch does together with any justification/reasoning. You should include
-references to any discussions (for example other tickets or mailing list
-discussions).
+### Address Feedback
 
-At this stage one should expect comments and review from other contributors. You
+At this stage, one should expect comments and review from other contributors. You
 can add more commits to your pull request by committing them locally and pushing
 to your fork until you have satisfied all feedback.
 
-Note: Code review is a burdensome but important part of the development process, and as such, certain types of pull requests are rejected. In general, if the **improvements** do not warrant the **review effort** required, the PR has a high chance of being rejected. It is up to the PR author to convince the reviewers that the changes warrant the review effort, and if reviewers are "Concept NAK'ing" the PR, the author may need to present arguments and/or do research backing their suggested changes.
+Note: Code review is a burdensome but important part of the development process, and as such, certain types of pull requests are rejected. In general, if the **improvements** do not warrant the **review effort** required, the PR has a high chance of being rejected. It is up to the PR author to convince the reviewers that the changes warrant the review effort, and if reviewers are "Concept NACK'ing" the PR, the author may need to present arguments and/or do research backing their suggested changes.
 
-Squashing Commits
----------------------------
-If your pull request is accepted for merging, you may be asked by a maintainer
-to squash and or [rebase](https://git-scm.com/docs/git-rebase) your commits
+### Squashing Commits
+
+If your pull request contains fixup commits (commits that change the same line of code repeatedly) or too fine-grained
+commits, you may be asked to [squash](https://git-scm.com/docs/git-rebase#_interactive_mode) your commits
 before it will be merged. The basic squashing workflow is shown below.
 
     git checkout your_branch_name
@@ -133,13 +207,13 @@ before it will be merged. The basic squashing workflow is shown below.
     # Save and quit.
     git push -f # (force push to GitHub)
 
-Please update the resulting commit message if needed, it should read as a
-coherent message. In most cases this means that you should not just list the
-interim commits.
+Please update the resulting commit message, if needed. It should read as a
+coherent message. In most cases, this means not just listing the interim
+commits.
 
-If you have problems with squashing (or other workflows with `git`), you can
-alternatively enable "Allow edits from maintainers" in the right GitHub
-sidebar and ask for help in the pull request.
+If you have problems with squashing or other git workflows, you can enable
+"Allow edits from maintainers" in the right-hand sidebar of the GitHub web
+interface and ask for help in the pull request.
 
 Please refrain from creating several pull requests for the same change.
 Use the pull request that is already open (or was created earlier) to amend
@@ -149,6 +223,20 @@ the respective change set.
 The length of time required for peer review is unpredictable and will vary from
 pull request to pull request.
 
+### Rebasing Changes
+
+When a pull request conflicts with the target branch, you may be asked to rebase it on top of the current target branch.
+The `git rebase` command will take care of rebuilding your commits on top of the new base.
+
+This project aims to have a clean git history, where code changes are only made in non-merge commits. This simplifies
+auditability because merge commits can be assumed to not contain arbitrary code changes. Merge commits should be signed,
+and the resulting git tree hash must be deterministic and reproducible. The script in
+[/contrib/verify-commits](/contrib/verify-commits) checks that.
+
+After a rebase, reviewers are encouraged to sign off on the force push. This should be relatively straightforward with
+the `git range-diff` tool explained in the [productivity
+notes](/doc/productivity.md#diff-the-diffs-with-git-range-diff). To avoid needless review churn, maintainers will
+generally merge pull requests that received the most review attention first.
 
 Pull Request Philosophy
 -----------------------
@@ -173,9 +261,9 @@ in the future, they may be removed by the Repository Maintainer.
 Refactoring is a necessary part of any software project's evolution. The
 following guidelines cover refactoring pull requests for the project.
 
-There are three categories of refactoring, code only moves, code style fixes,
-code refactoring. In general refactoring pull requests should not mix these
-three kinds of activity in order to make refactoring pull requests easy to
+There are three categories of refactoring: code-only moves, code style fixes, and
+code refactoring. In general, refactoring pull requests should not mix these
+three kinds of activities in order to make refactoring pull requests easy to
 review and uncontroversial. In all cases, refactoring PRs must not change the
 behaviour of code within the pull request (bugs must be preserved as is).
 
@@ -209,8 +297,8 @@ In general, all pull requests must:
 
   - Have a clear use case, fix a demonstrable bug or serve the greater good of
     the project (for example refactoring for modularisation);
-  - Be well peer reviewed;
-  - Have unit tests and functional tests where appropriate;
+  - Be well peer-reviewed;
+  - Have unit tests, functional tests, and fuzz tests, where appropriate;
   - Follow code style guidelines ([C++](doc/developer-notes.md), [functional tests](test/functional/README.md));
   - Not break the existing test suite;
   - Where bugs are fixed, where possible, there should be unit tests
@@ -237,7 +325,7 @@ spread out over GitHub, mailing list and IRC discussions).
 #### Conceptual Review
 
 A review can be a conceptual review, where the reviewer leaves a comment
- * `Concept (N)ACK`, meaning "I do (not) agree in the general goal of this pull
+ * `Concept (N)ACK`, meaning "I do (not) agree with the general goal of this pull
    request",
  * `Approach (N)ACK`, meaning `Concept ACK`, but "I do (not) agree with the
    approach of this change".
@@ -247,30 +335,28 @@ NACKs without accompanying reasoning may be disregarded.
 
 #### Code Review
 
-After conceptual agreement on the change, code review can be provided. It is
-starting with `ACK BRANCH_COMMIT`, where `BRANCH_COMMIT` is the top of the
-topic branch. The review is followed by a description of how the reviewer did
-the review. The following
-language is used within pull-request comments:
+After conceptual agreement on the change, code review can be provided. A review
+begins with `ACK BRANCH_COMMIT`, where `BRANCH_COMMIT` is the top of the PR
+branch, followed by a description of how the reviewer did the review. The
+following language is used within pull request comments:
 
-  - "I have tested the code", involving
-    change-specific manual testing in addition to running the unit and functional
-    tests, and in case it is not obvious how the manual testing was done, it should
-    be described;
+  - "I have tested the code", involving change-specific manual testing in
+    addition to running the unit, functional, or fuzz tests, and in case it is
+    not obvious how the manual testing was done, it should be described;
   - "I have not tested the code, but I have reviewed it and it looks
     OK, I agree it can be merged";
-  - Nit refers to trivial, often non-blocking issues.
+  - A "nit" refers to a trivial, often non-blocking issue.
 
 Project maintainers reserve the right to weigh the opinions of peer reviewers
-using common sense judgement and also may weight based on meritocracy: Those
-that have demonstrated a deeper commitment and understanding towards the project
-(over time) or have clear domain expertise may naturally have more weight, as
-one would expect in all walks of life.
+using common sense judgement and may also weigh based on merit. Reviewers that
+have demonstrated a deeper commitment and understanding of the project over time
+or who have clear domain expertise may naturally have more weight, as one would
+expect in all walks of life.
 
-Where a patch set affects consensus critical code, the bar will be set much
+Where a patch set affects consensus-critical code, the bar will be much
 higher in terms of discussion and peer review requirements, keeping in mind that
 mistakes could be very costly to the wider community. This includes refactoring
-of consensus critical code.
+of consensus-critical code.
 
 Where a patch set proposes to change the Bitcoin consensus, it must have been
 discussed extensively on the mailing list and IRC, be accompanied by a widely
@@ -287,7 +373,7 @@ about:
 
   - It may be because of a feature freeze due to an upcoming release. During this time,
     only bug fixes are taken into consideration. If your pull request is a new feature,
-    it will not be prioritized until the release is over. Wait for release.
+    it will not be prioritized until after the release. Wait for the release.
   - It may be because the changes you are suggesting do not appeal to people. Rather than
     nits and critique, which require effort and means they care enough to spend time on your
     contribution, thundering silence is a good sign of widespread (mild) dislike of a given change
@@ -297,18 +383,45 @@ about:
     [developer notes](doc/developer-notes.md), is dangerous or insecure, is messily written, etc.
     Identify and address any of the issues you find. Then ask e.g. on IRC if someone could give
     their opinion on the concept itself.
-  - It may be because your code is too complex for all but a few people. And those people
+  - It may be because your code is too complex for all but a few people, and those people
     may not have realized your pull request even exists. A great way to find people who
     are qualified and care about the code you are touching is the
     [Git Blame feature](https://help.github.com/articles/tracing-changes-in-a-file/). Simply
-    find the person touching the code you are touching before you and see if you can find
-    them and give them a nudge. Don't be incessant about the nudging though.
+    look up who last modified the code you are changing and see if you can find
+    them and give them a nudge. Don't be incessant about the nudging, though.
   - Finally, if all else fails, ask on IRC or elsewhere for someone to give your pull request
-    a look. If you think you've been waiting an unreasonably long amount of time (month+) for
-    no particular reason (few lines changed, etc), this is totally fine. Try to return the favor
-    when someone else is asking for feedback on their code, and universe balances out.
+    a look. If you think you've been waiting for an unreasonably long time (say,
+    more than a month) for no particular reason (a few lines changed, etc.),
+    this is totally fine. Try to return the favor when someone else is asking
+    for feedback on their code, and the universe balances out.
+  - Remember that the best thing you can do while waiting is give review to others!
 
 
+Backporting
+-----------
+
+Security and bug fixes can be backported from `master` to release
+branches.
+If the backport is non-trivial, it may be appropriate to open an
+additional PR to backport the change, but only after the original PR
+has been merged.
+Otherwise, backports will be done in batches and
+the maintainers will use the proper `Needs backport (...)` labels
+when needed (the original author does not need to worry about it).
+
+A backport should contain the following metadata in the commit body:
+
+```
+Github-Pull: #<PR number>
+Rebased-From: <commit hash of the original commit>
+```
+
+Have a look at [an example backport PR](
+https://github.com/bitcoin/bitcoin/pull/16189).
+
+Also see the [backport.py script](
+https://github.com/bitcoin-core/bitcoin-maintainer-tools#backport).
+
 Release Policy
 --------------
 

COPYING

@@ -1,7 +1,7 @@
 The MIT License (MIT)
 
-Copyright (c) 2009-2019 The Bitcoin Core developers
-Copyright (c) 2009-2019 Bitcoin Developers
+Copyright (c) 2009-2020 The Bitcoin Core developers
+Copyright (c) 2009-2020 Bitcoin Developers
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile.am

@@ -1,7 +1,11 @@
-# Copyright (c) 2013-2016 The Bitcoin Core developers
+# Copyright (c) 2013-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+# Pattern rule to print variables, e.g. make print-top_srcdir
+print-%:
+	@echo $* = $($*)
+
 ACLOCAL_AMFLAGS = -I build-aux/m4
 SUBDIRS = src
 if ENABLE_MAN
@@ -21,7 +25,7 @@ BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT)
 BITCOIN_CLI_BIN=$(top_builddir)/src/$(BITCOIN_CLI_NAME)$(EXEEXT)
 BITCOIN_TX_BIN=$(top_builddir)/src/$(BITCOIN_TX_NAME)$(EXEEXT)
 BITCOIN_WALLET_BIN=$(top_builddir)/src/$(BITCOIN_WALLET_TOOL_NAME)$(EXEEXT)
-BITCOIN_WIN_INSTALLER=$(PACKAGE)-$(PACKAGE_VERSION)-win$(WINDOWS_BITS)-setup$(EXEEXT)
+BITCOIN_WIN_INSTALLER=$(PACKAGE)-$(PACKAGE_VERSION)-win64-setup$(EXEEXT)
 
 empty :=
 space := $(empty) $(empty)
@@ -37,15 +41,12 @@ OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus
 OSX_FANCY_PLIST=$(top_srcdir)/contrib/macdeploy/fancy.plist
 OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns
 OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed
-OSX_QT_TRANSLATIONS = da,de,es,hu,ru,uk,zh_CN,zh_TW
+OSX_QT_TRANSLATIONS = ar,bg,ca,cs,da,de,es,fa,fi,fr,gd,gl,he,hu,it,ja,ko,lt,lv,pl,pt,ru,sk,sl,sv,uk,zh_CN,zh_TW
+
+DIST_CONTRIB = \
+	       $(top_srcdir)/contrib/linearize/linearize-data.py \
+	       $(top_srcdir)/contrib/linearize/linearize-hashes.py
 
-DIST_DOCS = $(wildcard doc/*.md) $(wildcard doc/release-notes/*.md)
-DIST_CONTRIB = $(top_srcdir)/contrib/bitcoin-cli.bash-completion \
-	       $(top_srcdir)/contrib/bitcoin-tx.bash-completion \
-	       $(top_srcdir)/contrib/bitcoind.bash-completion \
-	       $(top_srcdir)/contrib/debian/copyright \
-	       $(top_srcdir)/contrib/init \
-	       $(top_srcdir)/contrib/install_db4.sh
 DIST_SHARE = \
   $(top_srcdir)/share/genbuild.sh \
   $(top_srcdir)/share/rpcauth
@@ -64,10 +65,10 @@ OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_FANCY_PLIST) $(OSX_INSTALLER_ICONS) \
   $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \
   $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh
 
-COVERAGE_INFO = baseline.info \
+COVERAGE_INFO = $(COV_TOOL_WRAPPER) baseline.info \
   test_bitcoin_filtered.info total_coverage.info \
   baseline_filtered.info functional_test.info functional_test_filtered.info \
-  test_bitcoin_coverage.info test_bitcoin.info
+  test_bitcoin_coverage.info test_bitcoin.info fuzz.info fuzz_filtered.info fuzz_coverage.info
 
 dist-hook:
 	-$(GIT) archive --format=tar HEAD -- src/clientversion.cpp | $(AMTAR) -C $(top_distdir) -xf -
@@ -79,7 +80,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TX_BIN) $(top_builddir)/release
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_WALLET_BIN) $(top_builddir)/release
-	@test -f $(MAKENSIS) && $(MAKENSIS) -V2 $(top_builddir)/share/setup.nsi || \
+	@test -f $(MAKENSIS) && echo 'OutFile "$@"' | cat $(top_builddir)/share/setup.nsi - | $(MAKENSIS) -V2 - || \
 	  echo error: could not build $@
 	@echo built $@
 
@@ -89,7 +90,7 @@ $(OSX_APP)/Contents/PkgInfo:
 
 $(OSX_APP)/Contents/Resources/empty.lproj:
 	$(MKDIR_P) $(@D)
-	@touch $@ 
+	@touch $@
 
 $(OSX_APP)/Contents/Info.plist: $(OSX_PLIST)
 	$(MKDIR_P) $(@D)
@@ -179,15 +180,41 @@ $(BITCOIN_WALLET_BIN): FORCE
 	$(MAKE) -C src $(@F)
 
 if USE_LCOV
-LCOV_FILTER_PATTERN=-p "/usr/include/" -p "/usr/lib/" -p "src/leveldb/" -p "src/bench/" -p "src/univalue" -p "src/crypto/ctaes" -p "src/secp256k1"
+LCOV_FILTER_PATTERN = \
+	-p "/usr/local/" \
+	-p "/usr/include/" \
+	-p "/usr/lib/" \
+	-p "/usr/lib64/" \
+	-p "src/leveldb/" \
+	-p "src/crc32c/" \
+	-p "src/bench/" \
+	-p "src/univalue" \
+	-p "src/crypto/ctaes" \
+	-p "src/secp256k1" \
+	-p "depends"
 
-baseline.info:
+DIR_FUZZ_SEED_CORPUS ?= qa-assets/fuzz_seed_corpus
+
+$(COV_TOOL_WRAPPER):
+	@echo 'exec $(COV_TOOL) "$$@"' > $(COV_TOOL_WRAPPER)
+	@chmod +x $(COV_TOOL_WRAPPER)
+
+baseline.info: $(COV_TOOL_WRAPPER)
 	$(LCOV) -c -i -d $(abs_builddir)/src -o $@
 
 baseline_filtered.info: baseline.info
 	$(abs_builddir)/contrib/filter-lcov.py $(LCOV_FILTER_PATTERN) $< $@
 	$(LCOV) -a $@ $(LCOV_OPTS) -o $@
 
+fuzz.info: baseline_filtered.info
+	@TIMEOUT=15 test/fuzz/test_runner.py $(DIR_FUZZ_SEED_CORPUS) -l DEBUG
+	$(LCOV) -c $(LCOV_OPTS) -d $(abs_builddir)/src --t fuzz-tests -o $@
+	$(LCOV) -z $(LCOV_OPTS) -d $(abs_builddir)/src
+
+fuzz_filtered.info: fuzz.info
+	$(abs_builddir)/contrib/filter-lcov.py $(LCOV_FILTER_PATTERN) $< $@
+	$(LCOV) -a $@ $(LCOV_OPTS) -o $@
+
 test_bitcoin.info: baseline_filtered.info
 	$(MAKE) -C src/ check
 	$(LCOV) -c $(LCOV_OPTS) -d $(abs_builddir)/src -t test_bitcoin -o $@
@@ -206,12 +233,19 @@ functional_test_filtered.info: functional_test.info
 	$(abs_builddir)/contrib/filter-lcov.py $(LCOV_FILTER_PATTERN) $< $@
 	$(LCOV) -a $@ $(LCOV_OPTS) -o $@
 
+fuzz_coverage.info: fuzz_filtered.info
+	$(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a fuzz_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt
+
 test_bitcoin_coverage.info: baseline_filtered.info test_bitcoin_filtered.info
 	$(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a test_bitcoin_filtered.info -o $@
 
 total_coverage.info: test_bitcoin_filtered.info functional_test_filtered.info
 	$(LCOV) -a $(LCOV_OPTS) baseline_filtered.info -a test_bitcoin_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt
 
+fuzz.coverage/.dirstamp: fuzz_coverage.info
+	$(GENHTML) -s $(LCOV_OPTS) $< -o $(@D)
+	@touch $@
+
 test_bitcoin.coverage/.dirstamp:  test_bitcoin_coverage.info
 	$(GENHTML) -s $(LCOV_OPTS) $< -o $(@D)
 	@touch $@
@@ -220,13 +254,15 @@ total.coverage/.dirstamp: total_coverage.info
 	$(GENHTML) -s $(LCOV_OPTS) $< -o $(@D)
 	@touch $@
 
+cov_fuzz: fuzz.coverage/.dirstamp
+
 cov: test_bitcoin.coverage/.dirstamp total.coverage/.dirstamp
 
 endif
 
 dist_noinst_SCRIPTS = autogen.sh
 
-EXTRA_DIST = $(DIST_SHARE) $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
+EXTRA_DIST = $(DIST_SHARE) $(DIST_CONTRIB) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
 
 EXTRA_DIST += \
     test/functional \
@@ -281,6 +317,8 @@ EXTRA_DIST += \
     test/util/data/txcreatescript3.json \
     test/util/data/txcreatescript4.hex \
     test/util/data/txcreatescript4.json \
+    test/util/data/txcreatescript5.hex \
+    test/util/data/txcreatescript6.hex \
     test/util/data/txcreatesignv1.hex \
     test/util/data/txcreatesignv1.json \
     test/util/data/txcreatesignv2.hex \
@@ -308,7 +346,17 @@ clean-docs:
 	rm -rf doc/doxygen
 
 clean-local: clean-docs
-	rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ test/tmp/ cache/ $(OSX_APP)
+	rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ fuzz.coverage/ test/tmp/ cache/ $(OSX_APP)
 	rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__
 	rm -rf osx_volname dist/ dpi36.background.tiff dpi72.background.tiff
 
+test-security-check:
+if TARGET_DARWIN
+	$(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO
+endif
+if TARGET_WINDOWS
+	$(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE
+endif
+if TARGET_LINUX
+	$(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF
+endif

README.md

@@ -12,7 +12,7 @@ with no central authority: managing transactions and issuing money are carried
 out collectively by the network. Bitcoin Core is the name of open source
 software which enables the use of this currency.
 
-For more information, as well as an immediately useable, binary version of
+For more information, as well as an immediately usable, binary version of
 the Bitcoin Core software, see https://bitcoincore.org/en/download/, or read the
 [original whitepaper](https://bitcoincore.org/bitcoin.pdf).
 
@@ -25,9 +25,14 @@ information or see https://opensource.org/licenses/MIT.
 Development Process
 -------------------
 
-The `master` branch is regularly built and tested, but is not guaranteed to be
+The `master` branch is regularly built (see `doc/build-*.md` for instructions) and tested, but it is not guaranteed to be
 completely stable. [Tags](https://github.com/bitcoin/bitcoin/tags) are created
-regularly to indicate new official, stable release versions of Bitcoin Core.
+regularly from release branches to indicate new official, stable release versions of Bitcoin Core.
+
+The https://github.com/bitcoin-core/gui repository is used exclusively for the
+development of the GUI. Its master branch is identical in all monotree
+repositories. Release branches and tags do not exist, so please do not fork
+that repository unless it is for development reasons.
 
 The contribution workflow is described in [CONTRIBUTING.md](CONTRIBUTING.md)
 and useful hints for developers can be found in [doc/developer-notes.md](doc/developer-notes.md).

autogen.sh

@@ -1,13 +1,13 @@
 #!/bin/sh
-# Copyright (c) 2013-2016 The Bitcoin Core developers
+# Copyright (c) 2013-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C
 set -e
-srcdir="$(dirname $0)"
+srcdir="$(dirname "$0")"
 cd "$srcdir"
-if [ -z ${LIBTOOLIZE} ] && GLIBTOOLIZE="$(command -v glibtoolize)"; then
+if [ -z "${LIBTOOLIZE}" ] && GLIBTOOLIZE="$(command -v glibtoolize)"; then
   LIBTOOLIZE="${GLIBTOOLIZE}"
   export LIBTOOLIZE
 fi

build-aux/m4/ax_boost_base.m4

@@ -33,7 +33,7 @@
 #   and this notice are preserved. This file is offered as-is, without any
 #   warranty.
 
-#serial 47
+#serial 48
 
 # example boost program (need to pass version)
 m4_define([_AX_BOOST_BASE_PROGRAM],
@@ -123,6 +123,7 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[
     dnl are almost assuredly the ones desired.
     AS_CASE([${host_cpu}],
       [i?86],[multiarch_libsubdir="lib/i386-${host_os}"],
+      [armv7l],[multiarch_libsubdir="lib/arm-${host_os}"],
       [multiarch_libsubdir="lib/${host_cpu}-${host_os}"]
     )
 

build-aux/m4/ax_boost_chrono.m4

@@ -1,118 +0,0 @@
-# ===========================================================================
-#     https://www.gnu.org/software/autoconf-archive/ax_boost_chrono.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_BOOST_CHRONO
-#
-# DESCRIPTION
-#
-#   Test for Chrono library from the Boost C++ libraries. The macro requires
-#   a preceding call to AX_BOOST_BASE. Further documentation is available at
-#   <http://randspringer.de/boost/index.html>.
-#
-#   This macro calls:
-#
-#     AC_SUBST(BOOST_CHRONO_LIB)
-#
-#   And sets:
-#
-#     HAVE_BOOST_CHRONO
-#
-# LICENSE
-#
-#   Copyright (c) 2012 Xiyue Deng <manphiz@gmail.com>
-#
-#   Copying and distribution of this file, with or without modification, are
-#   permitted in any medium without royalty provided the copyright notice
-#   and this notice are preserved. This file is offered as-is, without any
-#   warranty.
-
-#serial 5
-
-AC_DEFUN([AX_BOOST_CHRONO],
-[
-	AC_ARG_WITH([boost-chrono],
-	AS_HELP_STRING([--with-boost-chrono@<:@=special-lib@:>@],
-                   [use the Chrono library from boost - it is possible to specify a certain library for the linker
-                        e.g. --with-boost-chrono=boost_chrono-gcc-mt ]),
-        [
-        if test "$withval" = "no"; then
-			want_boost="no"
-        elif test "$withval" = "yes"; then
-            want_boost="yes"
-            ax_boost_user_chrono_lib=""
-        else
-		    want_boost="yes"
-		ax_boost_user_chrono_lib="$withval"
-		fi
-        ],
-        [want_boost="yes"]
-	)
-
-	if test "x$want_boost" = "xyes"; then
-        AC_REQUIRE([AC_PROG_CC])
-        AC_REQUIRE([AC_CANONICAL_BUILD])
-		CPPFLAGS_SAVED="$CPPFLAGS"
-		CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
-		export CPPFLAGS
-
-		LDFLAGS_SAVED="$LDFLAGS"
-		LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
-		export LDFLAGS
-
-        AC_CACHE_CHECK(whether the Boost::Chrono library is available,
-					   ax_cv_boost_chrono,
-        [AC_LANG_PUSH([C++])
-			 CXXFLAGS_SAVE=$CXXFLAGS
-
-			 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/chrono.hpp>]],
-                                   [[boost::chrono::system_clock::time_point* time = new boost::chrono::system_clock::time_point; delete time;]])],
-                   ax_cv_boost_chrono=yes, ax_cv_boost_chrono=no)
-			 CXXFLAGS=$CXXFLAGS_SAVE
-             AC_LANG_POP([C++])
-		])
-		if test "x$ax_cv_boost_chrono" = "xyes"; then
-			AC_SUBST(BOOST_CPPFLAGS)
-
-			AC_DEFINE(HAVE_BOOST_CHRONO,,[define if the Boost::Chrono library is available])
-            BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
-
-			LDFLAGS_SAVE=$LDFLAGS
-            if test "x$ax_boost_user_chrono_lib" = "x"; then
-                for libextension in `ls $BOOSTLIBDIR/libboost_chrono*.so* $BOOSTLIBDIR/libboost_chrono*.dylib* $BOOSTLIBDIR/libboost_chrono*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_chrono.*\)\.so.*$;\1;' -e 's;^lib\(boost_chrono.*\)\.dylib.*$;\1;' -e 's;^lib\(boost_chrono.*\)\.a.*$;\1;'` ; do
-                     ax_lib=${libextension}
-				    AC_CHECK_LIB($ax_lib, exit,
-                                 [BOOST_CHRONO_LIB="-l$ax_lib"; AC_SUBST(BOOST_CHRONO_LIB) link_chrono="yes"; break],
-                                 [link_chrono="no"])
-				done
-                if test "x$link_chrono" != "xyes"; then
-                for libextension in `ls $BOOSTLIBDIR/boost_chrono*.dll* $BOOSTLIBDIR/boost_chrono*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_chrono.*\)\.dll.*$;\1;' -e 's;^\(boost_chrono.*\)\.a.*$;\1;'` ; do
-                     ax_lib=${libextension}
-				    AC_CHECK_LIB($ax_lib, exit,
-                                 [BOOST_CHRONO_LIB="-l$ax_lib"; AC_SUBST(BOOST_CHRONO_LIB) link_chrono="yes"; break],
-                                 [link_chrono="no"])
-				done
-                fi
-
-            else
-               for ax_lib in $ax_boost_user_chrono_lib boost_chrono-$ax_boost_user_chrono_lib; do
-				      AC_CHECK_LIB($ax_lib, exit,
-                                   [BOOST_CHRONO_LIB="-l$ax_lib"; AC_SUBST(BOOST_CHRONO_LIB) link_chrono="yes"; break],
-                                   [link_chrono="no"])
-                  done
-
-            fi
-            if test "x$ax_lib" = "x"; then
-                AC_MSG_ERROR(Could not find a version of the Boost::Chrono library!)
-            fi
-			if test "x$link_chrono" = "xno"; then
-				AC_MSG_ERROR(Could not link against $ax_lib !)
-			fi
-		fi
-
-		CPPFLAGS="$CPPFLAGS_SAVED"
-	LDFLAGS="$LDFLAGS_SAVED"
-	fi
-])

build-aux/m4/ax_boost_process.m4

@@ -0,0 +1,121 @@
+# ===========================================================================
+#     https://www.gnu.org/software/autoconf-archive/ax_boost_process.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_BOOST_PROCESS
+#
+# DESCRIPTION
+#
+#   Test for Process library from the Boost C++ libraries. The macro
+#   requires a preceding call to AX_BOOST_BASE. Further documentation is
+#   available at <http://randspringer.de/boost/index.html>.
+#
+#   This macro calls:
+#
+#     AC_SUBST(BOOST_PROCESS_LIB)
+#
+#   And sets:
+#
+#     HAVE_BOOST_PROCESS
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
+#   Copyright (c) 2008 Michael Tindal
+#   Copyright (c) 2008 Daniel Casimiro <dan.casimiro@gmail.com>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved. This file is offered as-is, without any
+#   warranty.
+
+#serial 2
+
+AC_DEFUN([AX_BOOST_PROCESS],
+[
+	AC_ARG_WITH([boost-process],
+	AS_HELP_STRING([--with-boost-process@<:@=special-lib@:>@],
+                   [use the Process library from boost - it is possible to specify a certain library for the linker
+                        e.g. --with-boost-process=boost_process-gcc-mt ]),
+        [
+        if test "$withval" = "no"; then
+			want_boost_process="no"
+        elif test "$withval" = "yes"; then
+            want_boost_process="yes"
+            ax_boost_user_process_lib=""
+        else
+		    want_boost_process="yes"
+		ax_boost_user_process_lib="$withval"
+		fi
+        ],
+        [want_boost_process="yes"]
+	)
+
+	if test "x$want_boost_process" = "xyes"; then
+        AC_REQUIRE([AC_PROG_CC])
+        AC_REQUIRE([AC_CANONICAL_BUILD])
+		CPPFLAGS_SAVED="$CPPFLAGS"
+		CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+		export CPPFLAGS
+
+		LDFLAGS_SAVED="$LDFLAGS"
+		LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+		export LDFLAGS
+
+        AC_CACHE_CHECK(whether the Boost::Process library is available,
+					   ax_cv_boost_process,
+        [AC_LANG_PUSH([C++])
+			 CXXFLAGS_SAVE=$CXXFLAGS
+			 CXXFLAGS=
+
+             AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/process.hpp>]],
+                [[boost::process::child* child = new boost::process::child; delete child;]])],
+                ax_cv_boost_process=yes, ax_cv_boost_process=no)
+			 CXXFLAGS=$CXXFLAGS_SAVE
+             AC_LANG_POP([C++])
+		])
+		if test "x$ax_cv_boost_process" = "xyes"; then
+			AC_SUBST(BOOST_CPPFLAGS)
+
+			AC_DEFINE(HAVE_BOOST_PROCESS,,[define if the Boost::Process library is available])
+            BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
+
+			LDFLAGS_SAVE=$LDFLAGS
+            if test "x$ax_boost_user_process_lib" = "x"; then
+                for libextension in `ls -r $BOOSTLIBDIR/libboost_process* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'` ; do
+                     ax_lib=${libextension}
+				    AC_CHECK_LIB($ax_lib, exit,
+                                 [BOOST_PROCESS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROCESS_LIB) link_process="yes"; break],
+                                 [link_process="no"])
+				done
+                if test "x$link_process" != "xyes"; then
+                for libextension in `ls -r $BOOSTLIBDIR/boost_process* 2>/dev/null | sed 's,.*/,,' | sed -e 's,\..*,,'` ; do
+                     ax_lib=${libextension}
+				    AC_CHECK_LIB($ax_lib, exit,
+                                 [BOOST_PROCESS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROCESS_LIB) link_process="yes"; break],
+                                 [link_process="no"])
+				done
+                fi
+
+            else
+               for ax_lib in $ax_boost_user_process_lib boost_process-$ax_boost_user_process_lib; do
+				      AC_CHECK_LIB($ax_lib, exit,
+                                   [BOOST_PROCESS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROCESS_LIB) link_process="yes"; break],
+                                   [link_process="no"])
+                  done
+
+            fi
+            if test "x$ax_lib" = "x"; then
+                AC_MSG_ERROR(Could not find a version of the Boost::Process library!)
+            fi
+			if test "x$link_process" = "xno"; then
+				AC_MSG_ERROR(Could not link against $ax_lib !)
+			fi
+		fi
+
+		CPPFLAGS="$CPPFLAGS_SAVED"
+	LDFLAGS="$LDFLAGS_SAVED"
+	fi
+])

build-aux/m4/ax_boost_thread.m4

@@ -30,7 +30,7 @@
 #   and this notice are preserved. This file is offered as-is, without any
 #   warranty.
 
-#serial 32
+#serial 33
 
 AC_DEFUN([AX_BOOST_THREAD],
 [
@@ -67,13 +67,24 @@ AC_DEFUN([AX_BOOST_THREAD],
         [AC_LANG_PUSH([C++])
              CXXFLAGS_SAVE=$CXXFLAGS
 
-             if test "x$host_os" = "xsolaris" ; then
-                 CXXFLAGS="-pthreads $CXXFLAGS"
-             elif test "x$host_os" = "xmingw32" ; then
-                 CXXFLAGS="-mthreads $CXXFLAGS"
-             else
-                CXXFLAGS="-pthread $CXXFLAGS"
-             fi
+             case "x$host_os" in
+                 xsolaris )
+                     CXXFLAGS="-pthreads $CXXFLAGS"
+                     break;
+                     ;;
+                 xmingw32 )
+                     CXXFLAGS="-mthreads $CXXFLAGS"
+                     break;
+                     ;;
+                 *android* )
+                     break;
+                     ;;
+                 * )
+                     CXXFLAGS="-pthread $CXXFLAGS"
+                     break;
+                     ;;
+             esac
+
              AC_COMPILE_IFELSE([
                  AC_LANG_PROGRAM(
                      [[@%:@include <boost/thread/thread.hpp>]],
@@ -84,13 +95,23 @@ AC_DEFUN([AX_BOOST_THREAD],
              AC_LANG_POP([C++])
         ])
         if test "x$ax_cv_boost_thread" = "xyes"; then
-           if test "x$host_os" = "xsolaris" ; then
-              BOOST_CPPFLAGS="-pthreads $BOOST_CPPFLAGS"
-           elif test "x$host_os" = "xmingw32" ; then
-              BOOST_CPPFLAGS="-mthreads $BOOST_CPPFLAGS"
-           else
-              BOOST_CPPFLAGS="-pthread $BOOST_CPPFLAGS"
-           fi
+            case "x$host_os" in
+                xsolaris )
+                    BOOST_CPPFLAGS="-pthreads $BOOST_CPPFLAGS"
+                    break;
+                    ;;
+                xmingw32 )
+                    BOOST_CPPFLAGS="-mthreads $BOOST_CPPFLAGS"
+                    break;
+                    ;;
+                *android* )
+                    break;
+                    ;;
+                * )
+                    BOOST_CPPFLAGS="-pthread $BOOST_CPPFLAGS"
+                    break;
+                    ;;
+            esac
 
             AC_SUBST(BOOST_CPPFLAGS)
 
@@ -148,6 +169,9 @@ AC_DEFUN([AX_BOOST_THREAD],
                     xmingw32 )
                         break;
                         ;;
+                    *android* )
+                        break;
+                        ;;
                     * )
                         BOOST_THREAD_LIB="$BOOST_THREAD_LIB -lpthread"
                         break;

build-aux/m4/ax_cxx_compile_stdcxx.m4

@@ -1,5 +1,5 @@
 # ===========================================================================
-#   http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html
+#  https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html
 # ===========================================================================
 #
 # SYNOPSIS
@@ -33,21 +33,23 @@
 #   Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com>
 #   Copyright (c) 2015 Paul Norman <penorman@mac.com>
 #   Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu>
+#   Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com>
+#   Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com>
 #
 #   Copying and distribution of this file, with or without modification, are
 #   permitted in any medium without royalty provided the copyright notice
 #   and this notice are preserved.  This file is offered as-is, without any
 #   warranty.
 
-#serial 4
+#serial 11
 
 dnl  This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro
 dnl  (serial version number 13).
 
 AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl
-  m4_if([$1], [11], [],
-        [$1], [14], [],
-        [$1], [17], [m4_fatal([support for C++17 not yet implemented in AX_CXX_COMPILE_STDCXX])],
+  m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"],
+        [$1], [14], [ax_cxx_compile_alternatives="14 1y"],
+        [$1], [17], [ax_cxx_compile_alternatives="17 1z"],
         [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl
   m4_if([$2], [], [],
         [$2], [ext], [],
@@ -57,26 +59,13 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl
         [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true],
         [$3], [optional], [ax_cxx_compile_cxx$1_required=false],
         [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])])
-  m4_if([$4], [], [ax_cxx_compile_cxx$1_try_default=true],
-        [$4], [default], [ax_cxx_compile_cxx$1_try_default=true],
-        [$4], [nodefault], [ax_cxx_compile_cxx$1_try_default=false],
-        [m4_fatal([invalid fourth argument `$4' to AX_CXX_COMPILE_STDCXX])])
   AC_LANG_PUSH([C++])dnl
   ac_success=no
 
-  m4_if([$4], [nodefault], [], [dnl
-  AC_CACHE_CHECK(whether $CXX supports C++$1 features by default,
-  ax_cv_cxx_compile_cxx$1,
-  [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
-    [ax_cv_cxx_compile_cxx$1=yes],
-    [ax_cv_cxx_compile_cxx$1=no])])
-  if test x$ax_cv_cxx_compile_cxx$1 = xyes; then
-    ac_success=yes
-  fi])
-
   m4_if([$2], [noext], [], [dnl
   if test x$ac_success = xno; then
-    for switch in -std=gnu++$1 -std=gnu++0x; do
+    for alternative in ${ax_cxx_compile_alternatives}; do
+      switch="-std=gnu++${alternative}"
       cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch])
       AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch,
                      $cachevar,
@@ -102,22 +91,27 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl
     dnl HP's aCC needs +std=c++11 according to:
     dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf
     dnl Cray's crayCC needs "-h std=c++11"
-    for switch in -std=c++$1 -std=c++0x +std=c++$1 "-h std=c++$1"; do
-      cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch])
-      AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch,
-                     $cachevar,
-        [ac_save_CXX="$CXX"
-         CXX="$CXX $switch"
-         AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
-          [eval $cachevar=yes],
-          [eval $cachevar=no])
-         CXX="$ac_save_CXX"])
-      if eval test x\$$cachevar = xyes; then
-        CXX="$CXX $switch"
-        if test -n "$CXXCPP" ; then
-          CXXCPP="$CXXCPP $switch"
+    for alternative in ${ax_cxx_compile_alternatives}; do
+      for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do
+        cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch])
+        AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch,
+                       $cachevar,
+          [ac_save_CXX="$CXX"
+           CXX="$CXX $switch"
+           AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
+            [eval $cachevar=yes],
+            [eval $cachevar=no])
+           CXX="$ac_save_CXX"])
+        if eval test x\$$cachevar = xyes; then
+          CXX="$CXX $switch"
+          if test -n "$CXXCPP" ; then
+            CXXCPP="$CXXCPP $switch"
+          fi
+          ac_success=yes
+          break
         fi
-        ac_success=yes
+      done
+      if test x$ac_success = xyes; then
         break
       fi
     done
@@ -154,6 +148,11 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14],
   _AX_CXX_COMPILE_STDCXX_testbody_new_in_14
 )
 
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17],
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_14
+  _AX_CXX_COMPILE_STDCXX_testbody_new_in_17
+)
 
 dnl  Tests for new features in C++11
 
@@ -191,11 +190,13 @@ namespace cxx11
 
     struct Base
     {
+      virtual ~Base() {}
       virtual void f() {}
     };
 
     struct Derived : public Base
     {
+      virtual ~Derived() override {}
       virtual void f() override {}
     };
 
@@ -524,7 +525,7 @@ namespace cxx14
 
   }
 
-  namespace test_digit_seperators
+  namespace test_digit_separators
   {
 
     constexpr auto ten_million = 100'000'000;
@@ -566,3 +567,385 @@ namespace cxx14
 #endif  // __cplusplus >= 201402L
 
 ]])
+
+
+dnl  Tests for new features in C++17
+
+m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[
+
+// If the compiler admits that it is not ready for C++17, why torture it?
+// Hopefully, this will speed up the test.
+
+#ifndef __cplusplus
+
+#error "This is not a C++ compiler"
+
+#elif __cplusplus < 201703L
+
+#error "This is not a C++17 compiler"
+
+#else
+
+#include <initializer_list>
+#include <utility>
+#include <type_traits>
+
+namespace cxx17
+{
+
+  namespace test_constexpr_lambdas
+  {
+
+    constexpr int foo = [](){return 42;}();
+
+  }
+
+  namespace test::nested_namespace::definitions
+  {
+
+  }
+
+  namespace test_fold_expression
+  {
+
+    template<typename... Args>
+    int multiply(Args... args)
+    {
+      return (args * ... * 1);
+    }
+
+    template<typename... Args>
+    bool all(Args... args)
+    {
+      return (args && ...);
+    }
+
+  }
+
+  namespace test_extended_static_assert
+  {
+
+    static_assert (true);
+
+  }
+
+  namespace test_auto_brace_init_list
+  {
+
+    auto foo = {5};
+    auto bar {5};
+
+    static_assert(std::is_same<std::initializer_list<int>, decltype(foo)>::value);
+    static_assert(std::is_same<int, decltype(bar)>::value);
+  }
+
+  namespace test_typename_in_template_template_parameter
+  {
+
+    template<template<typename> typename X> struct D;
+
+  }
+
+  namespace test_fallthrough_nodiscard_maybe_unused_attributes
+  {
+
+    int f1()
+    {
+      return 42;
+    }
+
+    [[nodiscard]] int f2()
+    {
+      [[maybe_unused]] auto unused = f1();
+
+      switch (f1())
+      {
+      case 17:
+        f1();
+        [[fallthrough]];
+      case 42:
+        f1();
+      }
+      return f1();
+    }
+
+  }
+
+  namespace test_extended_aggregate_initialization
+  {
+
+    struct base1
+    {
+      int b1, b2 = 42;
+    };
+
+    struct base2
+    {
+      base2() {
+        b3 = 42;
+      }
+      int b3;
+    };
+
+    struct derived : base1, base2
+    {
+        int d;
+    };
+
+    derived d1 {{1, 2}, {}, 4};  // full initialization
+    derived d2 {{}, {}, 4};      // value-initialized bases
+
+  }
+
+  namespace test_general_range_based_for_loop
+  {
+
+    struct iter
+    {
+      int i;
+
+      int& operator* ()
+      {
+        return i;
+      }
+
+      const int& operator* () const
+      {
+        return i;
+      }
+
+      iter& operator++()
+      {
+        ++i;
+        return *this;
+      }
+    };
+
+    struct sentinel
+    {
+      int i;
+    };
+
+    bool operator== (const iter& i, const sentinel& s)
+    {
+      return i.i == s.i;
+    }
+
+    bool operator!= (const iter& i, const sentinel& s)
+    {
+      return !(i == s);
+    }
+
+    struct range
+    {
+      iter begin() const
+      {
+        return {0};
+      }
+
+      sentinel end() const
+      {
+        return {5};
+      }
+    };
+
+    void f()
+    {
+      range r {};
+
+      for (auto i : r)
+      {
+        [[maybe_unused]] auto v = i;
+      }
+    }
+
+  }
+
+  namespace test_lambda_capture_asterisk_this_by_value
+  {
+
+    struct t
+    {
+      int i;
+      int foo()
+      {
+        return [*this]()
+        {
+          return i;
+        }();
+      }
+    };
+
+  }
+
+  namespace test_enum_class_construction
+  {
+
+    enum class byte : unsigned char
+    {};
+
+    byte foo {42};
+
+  }
+
+  namespace test_constexpr_if
+  {
+
+    template <bool cond>
+    int f ()
+    {
+      if constexpr(cond)
+      {
+        return 13;
+      }
+      else
+      {
+        return 42;
+      }
+    }
+
+  }
+
+  namespace test_selection_statement_with_initializer
+  {
+
+    int f()
+    {
+      return 13;
+    }
+
+    int f2()
+    {
+      if (auto i = f(); i > 0)
+      {
+        return 3;
+      }
+
+      switch (auto i = f(); i + 4)
+      {
+      case 17:
+        return 2;
+
+      default:
+        return 1;
+      }
+    }
+
+  }
+
+  namespace test_template_argument_deduction_for_class_templates
+  {
+
+    template <typename T1, typename T2>
+    struct pair
+    {
+      pair (T1 p1, T2 p2)
+        : m1 {p1},
+          m2 {p2}
+      {}
+
+      T1 m1;
+      T2 m2;
+    };
+
+    void f()
+    {
+      [[maybe_unused]] auto p = pair{13, 42u};
+    }
+
+  }
+
+  namespace test_non_type_auto_template_parameters
+  {
+
+    template <auto n>
+    struct B
+    {};
+
+    B<5> b1;
+    B<'a'> b2;
+
+  }
+
+  namespace test_structured_bindings
+  {
+
+    int arr[2] = { 1, 2 };
+    std::pair<int, int> pr = { 1, 2 };
+
+    auto f1() -> int(&)[2]
+    {
+      return arr;
+    }
+
+    auto f2() -> std::pair<int, int>&
+    {
+      return pr;
+    }
+
+    struct S
+    {
+      int x1 : 2;
+      volatile double y1;
+    };
+
+    S f3()
+    {
+      return {};
+    }
+
+    auto [ x1, y1 ] = f1();
+    auto& [ xr1, yr1 ] = f1();
+    auto [ x2, y2 ] = f2();
+    auto& [ xr2, yr2 ] = f2();
+    const auto [ x3, y3 ] = f3();
+
+  }
+
+  namespace test_exception_spec_type_system
+  {
+
+    struct Good {};
+    struct Bad {};
+
+    void g1() noexcept;
+    void g2();
+
+    template<typename T>
+    Bad
+    f(T*, T*);
+
+    template<typename T1, typename T2>
+    Good
+    f(T1*, T2*);
+
+    static_assert (std::is_same_v<Good, decltype(f(g1, g2))>);
+
+  }
+
+  namespace test_inline_variables
+  {
+
+    template<class T> void f(T)
+    {}
+
+    template<class T> inline T g(T)
+    {
+      return T{};
+    }
+
+    template<> inline void f<>(int)
+    {}
+
+    template<> int g<>(int)
+    {
+      return 5;
+    }
+
+  }
+
+}  // namespace cxx17
+
+#endif  // __cplusplus < 201703L
+
+]])

build-aux/m4/ax_pthread.m4

@@ -1,5 +1,5 @@
 # ===========================================================================
-#        http://www.gnu.org/software/autoconf-archive/ax_pthread.html
+#        https://www.gnu.org/software/autoconf-archive/ax_pthread.html
 # ===========================================================================
 #
 # SYNOPSIS
@@ -55,6 +55,7 @@
 #
 #   Copyright (c) 2008 Steven G. Johnson <stevenj@alum.mit.edu>
 #   Copyright (c) 2011 Daniel Richard G. <skunk@iSKUNK.ORG>
+#   Copyright (c) 2019 Marc Stevens <marc.stevens@cwi.nl>
 #
 #   This program is free software: you can redistribute it and/or modify it
 #   under the terms of the GNU General Public License as published by the
@@ -67,7 +68,7 @@
 #   Public License for more details.
 #
 #   You should have received a copy of the GNU General Public License along
-#   with this program. If not, see <http://www.gnu.org/licenses/>.
+#   with this program. If not, see <https://www.gnu.org/licenses/>.
 #
 #   As a special exception, the respective Autoconf Macro's copyright owner
 #   gives unlimited permission to copy, distribute and modify the configure
@@ -82,7 +83,7 @@
 #   modified version of the Autoconf Macro, you may extend this special
 #   exception to the GPL to apply to your modified version as well.
 
-#serial 23
+#serial 27
 
 AU_ALIAS([ACX_PTHREAD], [AX_PTHREAD])
 AC_DEFUN([AX_PTHREAD], [
@@ -123,10 +124,12 @@ fi
 # (e.g. DEC) have both -lpthread and -lpthreads, where one of the
 # libraries is broken (non-POSIX).
 
-# Create a list of thread flags to try.  Items starting with a "-" are
-# C compiler flags, and other items are library names, except for "none"
-# which indicates that we try without any flags at all, and "pthread-config"
-# which is a program returning the flags for the Pth emulation library.
+# Create a list of thread flags to try. Items with a "," contain both
+# C compiler flags (before ",") and linker flags (after ","). Other items
+# starting with a "-" are C compiler flags, and remaining items are
+# library names, except for "none" which indicates that we try without
+# any flags at all, and "pthread-config" which is a program returning
+# the flags for the Pth emulation library.
 
 ax_pthread_flags="pthreads none -Kthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config"
 
@@ -194,14 +197,47 @@ case $host_os in
         # that too in a future libc.)  So we'll check first for the
         # standard Solaris way of linking pthreads (-mt -lpthread).
 
-        ax_pthread_flags="-mt,pthread pthread $ax_pthread_flags"
+        ax_pthread_flags="-mt,-lpthread pthread $ax_pthread_flags"
         ;;
 esac
 
+# Are we compiling with Clang?
+
+AC_CACHE_CHECK([whether $CC is Clang],
+    [ax_cv_PTHREAD_CLANG],
+    [ax_cv_PTHREAD_CLANG=no
+     # Note that Autoconf sets GCC=yes for Clang as well as GCC
+     if test "x$GCC" = "xyes"; then
+        AC_EGREP_CPP([AX_PTHREAD_CC_IS_CLANG],
+            [/* Note: Clang 2.7 lacks __clang_[a-z]+__ */
+#            if defined(__clang__) && defined(__llvm__)
+             AX_PTHREAD_CC_IS_CLANG
+#            endif
+            ],
+            [ax_cv_PTHREAD_CLANG=yes])
+     fi
+    ])
+ax_pthread_clang="$ax_cv_PTHREAD_CLANG"
+
+
 # GCC generally uses -pthread, or -pthreads on some platforms (e.g. SPARC)
 
+# Note that for GCC and Clang -pthread generally implies -lpthread,
+# except when -nostdlib is passed.
+# This is problematic using libtool to build C++ shared libraries with pthread:
+# [1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=25460
+# [2] https://bugzilla.redhat.com/show_bug.cgi?id=661333
+# [3] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=468555
+# To solve this, first try -pthread together with -lpthread for GCC
+
 AS_IF([test "x$GCC" = "xyes"],
-      [ax_pthread_flags="-pthread -pthreads $ax_pthread_flags"])
+      [ax_pthread_flags="-pthread,-lpthread -pthread -pthreads $ax_pthread_flags"])
+
+# Clang takes -pthread (never supported any other flag), but we'll try with -lpthread first
+
+AS_IF([test "x$ax_pthread_clang" = "xyes"],
+      [ax_pthread_flags="-pthread,-lpthread -pthread"])
+
 
 # The presence of a feature test macro requesting re-entrant function
 # definitions is, on some systems, a strong hint that pthreads support is
@@ -224,25 +260,86 @@ AS_IF([test "x$ax_pthread_check_macro" = "x--"],
       [ax_pthread_check_cond=0],
       [ax_pthread_check_cond="!defined($ax_pthread_check_macro)"])
 
-# Are we compiling with Clang?
 
-AC_CACHE_CHECK([whether $CC is Clang],
-    [ax_cv_PTHREAD_CLANG],
-    [ax_cv_PTHREAD_CLANG=no
-     # Note that Autoconf sets GCC=yes for Clang as well as GCC
-     if test "x$GCC" = "xyes"; then
-        AC_EGREP_CPP([AX_PTHREAD_CC_IS_CLANG],
-            [/* Note: Clang 2.7 lacks __clang_[a-z]+__ */
-#            if defined(__clang__) && defined(__llvm__)
-             AX_PTHREAD_CC_IS_CLANG
-#            endif
-            ],
-            [ax_cv_PTHREAD_CLANG=yes])
-     fi
-    ])
-ax_pthread_clang="$ax_cv_PTHREAD_CLANG"
+if test "x$ax_pthread_ok" = "xno"; then
+for ax_pthread_try_flag in $ax_pthread_flags; do
+
+        case $ax_pthread_try_flag in
+                none)
+                AC_MSG_CHECKING([whether pthreads work without any flags])
+                ;;
+
+                *,*)
+                PTHREAD_CFLAGS=`echo $ax_pthread_try_flag | sed "s/^\(.*\),\(.*\)$/\1/"`
+                PTHREAD_LIBS=`echo $ax_pthread_try_flag | sed "s/^\(.*\),\(.*\)$/\2/"`
+                AC_MSG_CHECKING([whether pthreads work with "$PTHREAD_CFLAGS" and "$PTHREAD_LIBS"])
+                ;;
+
+                -*)
+                AC_MSG_CHECKING([whether pthreads work with $ax_pthread_try_flag])
+                PTHREAD_CFLAGS="$ax_pthread_try_flag"
+                ;;
+
+                pthread-config)
+                AC_CHECK_PROG([ax_pthread_config], [pthread-config], [yes], [no])
+                AS_IF([test "x$ax_pthread_config" = "xno"], [continue])
+                PTHREAD_CFLAGS="`pthread-config --cflags`"
+                PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`"
+                ;;
+
+                *)
+                AC_MSG_CHECKING([for the pthreads library -l$ax_pthread_try_flag])
+                PTHREAD_LIBS="-l$ax_pthread_try_flag"
+                ;;
+        esac
+
+        ax_pthread_save_CFLAGS="$CFLAGS"
+        ax_pthread_save_LIBS="$LIBS"
+        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+        LIBS="$PTHREAD_LIBS $LIBS"
+
+        # Check for various functions.  We must include pthread.h,
+        # since some functions may be macros.  (On the Sequent, we
+        # need a special flag -Kthread to make this header compile.)
+        # We check for pthread_join because it is in -lpthread on IRIX
+        # while pthread_create is in libc.  We check for pthread_attr_init
+        # due to DEC craziness with -lpthreads.  We check for
+        # pthread_cleanup_push because it is one of the few pthread
+        # functions on Solaris that doesn't have a non-functional libc stub.
+        # We try pthread_create on general principles.
+
+        AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <pthread.h>
+#                       if $ax_pthread_check_cond
+#                        error "$ax_pthread_check_macro must be defined"
+#                       endif
+                        static void *some_global = NULL;
+                        static void routine(void *a)
+                          {
+                             /* To avoid any unused-parameter or
+                                unused-but-set-parameter warning.  */
+                             some_global = a;
+                          }
+                        static void *start_routine(void *a) { return a; }],
+                       [pthread_t th; pthread_attr_t attr;
+                        pthread_create(&th, 0, start_routine, 0);
+                        pthread_join(th, 0);
+                        pthread_attr_init(&attr);
+                        pthread_cleanup_push(routine, 0);
+                        pthread_cleanup_pop(0) /* ; */])],
+            [ax_pthread_ok=yes],
+            [])
+
+        CFLAGS="$ax_pthread_save_CFLAGS"
+        LIBS="$ax_pthread_save_LIBS"
+
+        AC_MSG_RESULT([$ax_pthread_ok])
+        AS_IF([test "x$ax_pthread_ok" = "xyes"], [break])
+
+        PTHREAD_LIBS=""
+        PTHREAD_CFLAGS=""
+done
+fi
 
-ax_pthread_clang_warning=no
 
 # Clang needs special handling, because older versions handle the -pthread
 # option in a rather... idiosyncratic way
@@ -261,11 +358,6 @@ if test "x$ax_pthread_clang" = "xyes"; then
         # -pthread does define _REENTRANT, and while the Darwin headers
         # ignore this macro, third-party headers might not.)
 
-        PTHREAD_CFLAGS="-pthread"
-        PTHREAD_LIBS=
-
-        ax_pthread_ok=yes
-
         # However, older versions of Clang make a point of warning the user
         # that, in an invocation where only linking and no compilation is
         # taking place, the -pthread option has no effect ("argument unused
@@ -320,78 +412,7 @@ if test "x$ax_pthread_clang" = "xyes"; then
 
 fi # $ax_pthread_clang = yes
 
-if test "x$ax_pthread_ok" = "xno"; then
-for ax_pthread_try_flag in $ax_pthread_flags; do
 
-        case $ax_pthread_try_flag in
-                none)
-                AC_MSG_CHECKING([whether pthreads work without any flags])
-                ;;
-
-                -mt,pthread)
-                AC_MSG_CHECKING([whether pthreads work with -mt -lpthread])
-                PTHREAD_CFLAGS="-mt"
-                PTHREAD_LIBS="-lpthread"
-                ;;
-
-                -*)
-                AC_MSG_CHECKING([whether pthreads work with $ax_pthread_try_flag])
-                PTHREAD_CFLAGS="$ax_pthread_try_flag"
-                ;;
-
-                pthread-config)
-                AC_CHECK_PROG([ax_pthread_config], [pthread-config], [yes], [no])
-                AS_IF([test "x$ax_pthread_config" = "xno"], [continue])
-                PTHREAD_CFLAGS="`pthread-config --cflags`"
-                PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`"
-                ;;
-
-                *)
-                AC_MSG_CHECKING([for the pthreads library -l$ax_pthread_try_flag])
-                PTHREAD_LIBS="-l$ax_pthread_try_flag"
-                ;;
-        esac
-
-        ax_pthread_save_CFLAGS="$CFLAGS"
-        ax_pthread_save_LIBS="$LIBS"
-        CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
-        LIBS="$PTHREAD_LIBS $LIBS"
-
-        # Check for various functions.  We must include pthread.h,
-        # since some functions may be macros.  (On the Sequent, we
-        # need a special flag -Kthread to make this header compile.)
-        # We check for pthread_join because it is in -lpthread on IRIX
-        # while pthread_create is in libc.  We check for pthread_attr_init
-        # due to DEC craziness with -lpthreads.  We check for
-        # pthread_cleanup_push because it is one of the few pthread
-        # functions on Solaris that doesn't have a non-functional libc stub.
-        # We try pthread_create on general principles.
-
-        AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <pthread.h>
-#                       if $ax_pthread_check_cond
-#                        error "$ax_pthread_check_macro must be defined"
-#                       endif
-                        static void routine(void *a) { a = 0; }
-                        static void *start_routine(void *a) { return a; }],
-                       [pthread_t th; pthread_attr_t attr;
-                        pthread_create(&th, 0, start_routine, 0);
-                        pthread_join(th, 0);
-                        pthread_attr_init(&attr);
-                        pthread_cleanup_push(routine, 0);
-                        pthread_cleanup_pop(0) /* ; */])],
-            [ax_pthread_ok=yes],
-            [])
-
-        CFLAGS="$ax_pthread_save_CFLAGS"
-        LIBS="$ax_pthread_save_LIBS"
-
-        AC_MSG_RESULT([$ax_pthread_ok])
-        AS_IF([test "x$ax_pthread_ok" = "xyes"], [break])
-
-        PTHREAD_LIBS=""
-        PTHREAD_CFLAGS=""
-done
-fi
 
 # Various other checks:
 if test "x$ax_pthread_ok" = "xyes"; then
@@ -438,7 +459,8 @@ if test "x$ax_pthread_ok" = "xyes"; then
         AC_CACHE_CHECK([for PTHREAD_PRIO_INHERIT],
             [ax_cv_PTHREAD_PRIO_INHERIT],
             [AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <pthread.h>]],
-                                             [[int i = PTHREAD_PRIO_INHERIT;]])],
+                                             [[int i = PTHREAD_PRIO_INHERIT;
+                                               return i;]])],
                             [ax_cv_PTHREAD_PRIO_INHERIT=yes],
                             [ax_cv_PTHREAD_PRIO_INHERIT=no])
             ])

build-aux/m4/bitcoin_qt.m4

@@ -72,32 +72,33 @@ AC_DEFUN([BITCOIN_QT_INIT],[
 
   AC_ARG_WITH([qtdbus],
     [AS_HELP_STRING([--with-qtdbus],
-    [enable DBus support (default is yes if qt is enabled and QtDBus is found)])],
+    [enable DBus support (default is yes if qt is enabled and QtDBus is found, except on Android)])],
     [use_dbus=$withval],
     [use_dbus=auto])
 
+  dnl Android doesn't support D-Bus and certainly doesn't use it for notifications
+  case $host in
+    *android*)
+      if test "x$use_dbus" != xyes; then
+        use_dbus=no
+      fi
+    ;;
+  esac
+
   AC_SUBST(QT_TRANSLATION_DIR,$qt_translation_path)
 ])
 
-dnl Find the appropriate version of Qt libraries and includes.
-dnl Inputs: $1: Whether or not pkg-config should be used. yes|no. Default: yes.
-dnl Inputs: $2: If $1 is "yes" and --with-gui=auto, which qt version should be
-dnl         tried first.
-dnl Outputs: See _BITCOIN_QT_FIND_LIBS_*
+dnl Find Qt libraries and includes.
+dnl
+dnl   BITCOIN_QT_CONFIGURE([MINIMUM-VERSION])
+dnl
+dnl Outputs: See _BITCOIN_QT_FIND_LIBS
 dnl Outputs: Sets variables for all qt-related tools.
 dnl Outputs: bitcoin_enable_qt, bitcoin_enable_qt_dbus, bitcoin_enable_qt_test
 AC_DEFUN([BITCOIN_QT_CONFIGURE],[
-  use_pkgconfig=$1
-
-  if test "x$use_pkgconfig" = x; then
-    use_pkgconfig=yes
-  fi
-
-  if test "x$use_pkgconfig" = xyes; then
-    BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS_WITH_PKGCONFIG])
-  else
-    BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS_WITHOUT_PKGCONFIG])
-  fi
+  qt_version=">= $1"
+  qt_lib_prefix="Qt5"
+  BITCOIN_QT_CHECK([_BITCOIN_QT_FIND_LIBS])
 
   dnl This is ugly and complicated. Yuck. Works as follows:
   dnl For Qt5, we can check a header to find out whether Qt is build
@@ -116,8 +117,10 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[
   if test "x$bitcoin_cv_static_qt" = xyes; then
     _BITCOIN_QT_FIND_STATIC_PLUGINS
     AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol if qt plugins are static])
-    _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal])
-    AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists])
+    if test "x$TARGET_OS" != xandroid; then
+      _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal])
+      AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists])
+    fi
     if test "x$TARGET_OS" = xwindows; then
       _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QWindowsIntegrationPlugin)],[-lqwindows])
       AC_DEFINE(QT_QPA_PLATFORM_WINDOWS, 1, [Define this symbol if the qt platform is windows])
@@ -128,13 +131,16 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[
       AX_CHECK_LINK_FLAG([[-framework IOKit]],[QT_LIBS="$QT_LIBS -framework IOKit"],[AC_MSG_ERROR(could not iokit framework)])
       _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QCocoaIntegrationPlugin)],[-lqcocoa])
       AC_DEFINE(QT_QPA_PLATFORM_COCOA, 1, [Define this symbol if the qt platform is cocoa])
+    elif test "x$TARGET_OS" = xandroid; then
+      QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lqtforandroid -ljnigraphics -landroid -lqtfreetype -lQt5EglSupport $QT_LIBS"
+      AC_DEFINE(QT_QPA_PLATFORM_ANDROID, 1, [Define this symbol if the qt platform is android])
     fi
   fi
   CPPFLAGS=$TEMP_CPPFLAGS
   CXXFLAGS=$TEMP_CXXFLAGS
   ])
 
-  if test "x$use_pkgconfig$qt_bin_path" = xyes; then
+  if test "x$qt_bin_path" = x; then
     qt_bin_path="`$PKG_CONFIG --variable=host_bins Qt5Core 2>/dev/null`"
   fi
 
@@ -208,7 +214,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[
 
 
   dnl enable qt support
-  AC_MSG_CHECKING(whether to build ]AC_PACKAGE_NAME[ GUI)
+  AC_MSG_CHECKING([whether to build ]AC_PACKAGE_NAME[ GUI])
   BITCOIN_QT_CHECK([
     bitcoin_enable_qt=yes
     bitcoin_enable_qt_test=yes
@@ -229,7 +235,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[
     bitcoin_enable_qt=no
   ])
   if test x$bitcoin_enable_qt = xyes; then
-    AC_MSG_RESULT([$bitcoin_enable_qt ($QT_LIB_PREFIX)])
+    AC_MSG_RESULT([$bitcoin_enable_qt ($qt_lib_prefix)])
   else
     AC_MSG_RESULT([$bitcoin_enable_qt])
   fi
@@ -250,57 +256,15 @@ dnl All macros below are internal and should _not_ be used from the main
 dnl configure.ac.
 dnl ----
 
-dnl Internal. Check included version of Qt against minimum specified in doc/dependencies.md
-dnl Requires: INCLUDES must be populated as necessary.
-dnl Output: bitcoin_cv_qt5=yes|no
-AC_DEFUN([_BITCOIN_QT_CHECK_QT5],[
-  AC_CACHE_CHECK(for Qt 5, bitcoin_cv_qt5,[
-  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
-      #include <QtCore/qconfig.h>
-      #ifndef QT_VERSION
-      #  include <QtCore/qglobal.h>
-      #endif
-    ]],
-    [[
-      #if QT_VERSION < 0x050501
-      choke
-      #endif
-    ]])],
-    [bitcoin_cv_qt5=yes],
-    [bitcoin_cv_qt5=no])
-])])
-
-dnl Internal. Check if the included version of Qt is greater than Qt58.
-dnl Requires: INCLUDES must be populated as necessary.
-dnl Output: bitcoin_cv_qt58=yes|no
-AC_DEFUN([_BITCOIN_QT_CHECK_QT58],[
-  AC_CACHE_CHECK(for > Qt 5.7, bitcoin_cv_qt58,[
-  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
-      #include <QtCore/qconfig.h>
-      #ifndef QT_VERSION
-      #  include <QtCore/qglobal.h>
-      #endif
-    ]],
-    [[
-      #if QT_VERSION_MINOR < 8
-      choke
-      #endif
-    ]])],
-    [bitcoin_cv_qt58=yes],
-    [bitcoin_cv_qt58=no])
-])])
-
-
 dnl Internal. Check if the linked version of Qt was built as static libs.
 dnl Requires: Qt5.
 dnl Requires: INCLUDES and LIBS must be populated as necessary.
 dnl Output: bitcoin_cv_static_qt=yes|no
-dnl Output: Defines QT_STATICPLUGIN if plugins are static.
 AC_DEFUN([_BITCOIN_QT_IS_STATIC],[
   AC_CACHE_CHECK(for static Qt, bitcoin_cv_static_qt,[
     AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
         #include <QtCore/qconfig.h>
-        #ifndef QT_VERSION OR QT_VERSION_STR
+        #ifndef QT_VERSION
         #  include <QtCore/qglobal.h>
         #endif
       ]],
@@ -312,9 +276,6 @@ AC_DEFUN([_BITCOIN_QT_IS_STATIC],[
       [bitcoin_cv_static_qt=yes],
       [bitcoin_cv_static_qt=no])
     ])
-  if test "x$bitcoin_cv_static_qt" = xyes; then
-    AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol for static Qt plugins])
-  fi
 ])
 
 dnl Internal. Check if the link-requirements for static plugins are met.
@@ -345,167 +306,52 @@ AC_DEFUN([_BITCOIN_QT_FIND_STATIC_PLUGINS],[
       if test -d "$qt_plugin_path/accessible"; then
         QT_LIBS="$QT_LIBS -L$qt_plugin_path/accessible"
       fi
-     if test "x$use_pkgconfig" = xyes; then
-     : dnl
-     m4_ifdef([PKG_CHECK_MODULES],[
-       if test x$bitcoin_cv_qt58 = xno; then
-         PKG_CHECK_MODULES([QTPLATFORM], [Qt5PlatformSupport], [QT_LIBS="$QTPLATFORM_LIBS $QT_LIBS"])
-       else
-         PKG_CHECK_MODULES([QTFONTDATABASE], [Qt5FontDatabaseSupport], [QT_LIBS="-lQt5FontDatabaseSupport $QT_LIBS"])
-         PKG_CHECK_MODULES([QTEVENTDISPATCHER], [Qt5EventDispatcherSupport], [QT_LIBS="-lQt5EventDispatcherSupport $QT_LIBS"])
-         PKG_CHECK_MODULES([QTTHEME], [Qt5ThemeSupport], [QT_LIBS="-lQt5ThemeSupport $QT_LIBS"])
-         PKG_CHECK_MODULES([QTDEVICEDISCOVERY], [Qt5DeviceDiscoverySupport], [QT_LIBS="-lQt5DeviceDiscoverySupport $QT_LIBS"])
-         PKG_CHECK_MODULES([QTACCESSIBILITY], [Qt5AccessibilitySupport], [QT_LIBS="-lQt5AccessibilitySupport $QT_LIBS"])
-         PKG_CHECK_MODULES([QTFB], [Qt5FbSupport], [QT_LIBS="-lQt5FbSupport $QT_LIBS"])
-                fi
-       if test "x$TARGET_OS" = xlinux; then
-         PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"])
-       elif test "x$TARGET_OS" = xdarwin; then
-         PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"])
-         PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport], [QT_LIBS="-lQt5GraphicsSupport $QT_LIBS"])
-         PKG_CHECK_MODULES([QTCGL], [Qt5CglSupport], [QT_LIBS="-lQt5CglSupport $QT_LIBS"])
-       fi
-     ])
-     else
-       if test "x$TARGET_OS" = xwindows; then
-         AC_CACHE_CHECK(for Qt >= 5.6, bitcoin_cv_need_platformsupport,[
-           AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
-               #include <QtCore/qconfig.h>
-               #ifndef QT_VERSION
-               #  include <QtCore/qglobal.h>
-               #endif
-             ]],
-             [[
-               #if QT_VERSION < 0x050600 || QT_VERSION_MINOR < 6
-               choke
-               #endif
-             ]])],
-           [bitcoin_cv_need_platformsupport=yes],
-           [bitcoin_cv_need_platformsupport=no])
-         ])
-         if test "x$bitcoin_cv_need_platformsupport" = xyes; then
-           if test x$bitcoin_cv_qt58 = xno; then
-             BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}PlatformSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXPlatformSupport not found)))
-           else
-             BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}FontDatabaseSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXFontDatabaseSupport not found)))
-             BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}EventDispatcherSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXEventDispatcherSupport not found)))
-             BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}ThemeSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXThemeSupport not found)))
-             BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}FbSupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXFbSupport not found)))
-             BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}DeviceDiscoverySupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXDeviceDiscoverySupport not found)))
-             BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}AccessibilitySupport],[main],,BITCOIN_QT_FAIL(lib$QT_LIB_PREFIXAccessibilitySupport not found)))
-             QT_LIBS="$QT_LIBS -lversion -ldwmapi -luxtheme"
-           fi
-         fi
-       fi
-     fi
-   fi
+      if test -d "$qt_plugin_path/platforms/android"; then
+        QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms/android -lqtfreetype -lEGL"
+      fi
+      PKG_CHECK_MODULES([QTFONTDATABASE], [Qt5FontDatabaseSupport], [QT_LIBS="-lQt5FontDatabaseSupport $QT_LIBS"])
+      PKG_CHECK_MODULES([QTEVENTDISPATCHER], [Qt5EventDispatcherSupport], [QT_LIBS="-lQt5EventDispatcherSupport $QT_LIBS"])
+      PKG_CHECK_MODULES([QTTHEME], [Qt5ThemeSupport], [QT_LIBS="-lQt5ThemeSupport $QT_LIBS"])
+      PKG_CHECK_MODULES([QTDEVICEDISCOVERY], [Qt5DeviceDiscoverySupport], [QT_LIBS="-lQt5DeviceDiscoverySupport $QT_LIBS"])
+      PKG_CHECK_MODULES([QTACCESSIBILITY], [Qt5AccessibilitySupport], [QT_LIBS="-lQt5AccessibilitySupport $QT_LIBS"])
+      PKG_CHECK_MODULES([QTFB], [Qt5FbSupport], [QT_LIBS="-lQt5FbSupport $QT_LIBS"])
+      if test "x$TARGET_OS" = xlinux; then
+        PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"])
+      elif test "x$TARGET_OS" = xdarwin; then
+        PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"])
+        PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport], [QT_LIBS="-lQt5GraphicsSupport $QT_LIBS"])
+        PKG_CHECK_MODULES([QTCGL], [Qt5CglSupport], [QT_LIBS="-lQt5CglSupport $QT_LIBS"])
+      fi
+    fi
 ])
 
 dnl Internal. Find Qt libraries using pkg-config.
-dnl Inputs: bitcoin_qt_want_version (from --with-gui=). The version to check
-dnl         first.
-dnl Inputs: $1: If bitcoin_qt_want_version is "auto", check for this version
-dnl         first.
 dnl Outputs: All necessary QT_* variables are set.
 dnl Outputs: have_qt_test and have_qt_dbus are set (if applicable) to yes|no.
-AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITH_PKGCONFIG],[
-  m4_ifdef([PKG_CHECK_MODULES],[
-    QT_LIB_PREFIX=Qt5
-    qt5_modules="Qt5Core Qt5Gui Qt5Network Qt5Widgets"
-    BITCOIN_QT_CHECK([
-      PKG_CHECK_MODULES([QT5], [$qt5_modules], [QT_INCLUDES="$QT5_CFLAGS"; QT_LIBS="$QT5_LIBS" have_qt=yes],[have_qt=no])
-
-      if test "x$have_qt" != xyes; then
-        have_qt=no
-        BITCOIN_QT_FAIL([Qt dependencies not found])
-      fi
-    ])
-    BITCOIN_QT_CHECK([
-      PKG_CHECK_MODULES([QT_TEST], [${QT_LIB_PREFIX}Test], [QT_TEST_INCLUDES="$QT_TEST_CFLAGS"; have_qt_test=yes], [have_qt_test=no])
-      if test "x$use_dbus" != xno; then
-        PKG_CHECK_MODULES([QT_DBUS], [${QT_LIB_PREFIX}DBus], [QT_DBUS_INCLUDES="$QT_DBUS_CFLAGS"; have_qt_dbus=yes], [have_qt_dbus=no])
-      fi
-    ])
-  ])
-  true; dnl
-])
-
-dnl Internal. Find Qt libraries without using pkg-config. Version is deduced
-dnl from the discovered headers.
-dnl Inputs: bitcoin_qt_want_version (from --with-gui=). The version to use.
-dnl         If "auto", the version will be discovered by _BITCOIN_QT_CHECK_QT5.
-dnl Outputs: All necessary QT_* variables are set.
-dnl Outputs: have_qt_test and have_qt_dbus are set (if applicable) to yes|no.
-AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITHOUT_PKGCONFIG],[
-  TEMP_CPPFLAGS="$CPPFLAGS"
-  TEMP_CXXFLAGS="$CXXFLAGS"
-  CXXFLAGS="$PIC_FLAGS $CXXFLAGS"
-  TEMP_LIBS="$LIBS"
+AC_DEFUN([_BITCOIN_QT_FIND_LIBS],[
   BITCOIN_QT_CHECK([
-    if test "x$qt_include_path" != x; then
-      QT_INCLUDES="-I$qt_include_path -I$qt_include_path/QtCore -I$qt_include_path/QtGui -I$qt_include_path/QtWidgets -I$qt_include_path/QtNetwork -I$qt_include_path/QtTest -I$qt_include_path/QtDBus"
-      CPPFLAGS="$QT_INCLUDES $CPPFLAGS"
-    fi
+    PKG_CHECK_MODULES([QT_CORE], [${qt_lib_prefix}Core $qt_version], [],
+                      [BITCOIN_QT_FAIL([${qt_lib_prefix}Core $qt_version not found])])
   ])
-
-  BITCOIN_QT_CHECK([AC_CHECK_HEADER([QtPlugin],,BITCOIN_QT_FAIL(QtCore headers missing))])
-  BITCOIN_QT_CHECK([AC_CHECK_HEADER([QApplication],, BITCOIN_QT_FAIL(QtGui headers missing))])
-  BITCOIN_QT_CHECK([AC_CHECK_HEADER([QLocalSocket],, BITCOIN_QT_FAIL(QtNetwork headers missing))])
+  BITCOIN_QT_CHECK([
+    PKG_CHECK_MODULES([QT_GUI], [${qt_lib_prefix}Gui $qt_version], [],
+                      [BITCOIN_QT_FAIL([${qt_lib_prefix}Gui $qt_version not found])])
+  ])
+  BITCOIN_QT_CHECK([
+    PKG_CHECK_MODULES([QT_WIDGETS], [${qt_lib_prefix}Widgets $qt_version], [],
+                      [BITCOIN_QT_FAIL([${qt_lib_prefix}Widgets $qt_version not found])])
+  ])
+  BITCOIN_QT_CHECK([
+    PKG_CHECK_MODULES([QT_NETWORK], [${qt_lib_prefix}Network $qt_version], [],
+                      [BITCOIN_QT_FAIL([${qt_lib_prefix}Network $qt_version not found])])
+  ])
+  QT_INCLUDES="$QT_CORE_CFLAGS $QT_GUI_CFLAGS $QT_WIDGETS_CFLAGS $QT_NETWORK_CFLAGS"
+  QT_LIBS="$QT_CORE_LIBS $QT_GUI_LIBS $QT_WIDGETS_LIBS $QT_NETWORK_LIBS"
 
   BITCOIN_QT_CHECK([
-    if test "x$bitcoin_qt_want_version" = xauto; then
-      _BITCOIN_QT_CHECK_QT5
-      _BITCOIN_QT_CHECK_QT58
-    fi
-    QT_LIB_PREFIX=Qt5
-  ])
-
-  BITCOIN_QT_CHECK([
-    LIBS=
-    if test "x$qt_lib_path" != x; then
-      LIBS="$LIBS -L$qt_lib_path"
-    fi
-
-    if test "x$TARGET_OS" = xwindows; then
-      AC_CHECK_LIB([imm32],      [main],, BITCOIN_QT_FAIL(libimm32 not found))
-    fi
-  ])
-
-  BITCOIN_QT_CHECK(AC_CHECK_LIB([z] ,[main],,AC_MSG_WARN([zlib not found. Assuming qt has it built-in])))
-  if test x$bitcoin_cv_qt58 = xno; then
-    BITCOIN_QT_CHECK(AC_SEARCH_LIBS([png_error] ,[qtpng png],,AC_MSG_WARN([libpng not found. Assuming qt has it built-in])))
-    BITCOIN_QT_CHECK(AC_SEARCH_LIBS([pcre16_exec], [qtpcre pcre16],,AC_MSG_WARN([libpcre16 not found. Assuming qt has it built-in])))
-  else
-    BITCOIN_QT_CHECK(AC_SEARCH_LIBS([png_error] ,[qtlibpng png],,AC_MSG_WARN([libpng not found. Assuming qt has it built-in])))
-    BITCOIN_QT_CHECK(AC_SEARCH_LIBS([pcre2_match_16], [qtpcre2 libqtpcre2],,AC_MSG_WARN([libqtpcre2 not found. Assuming qt has it built-in])))
-  fi
-  BITCOIN_QT_CHECK(AC_SEARCH_LIBS([hb_ot_tags_from_script] ,[qtharfbuzzng qtharfbuzz harfbuzz],,AC_MSG_WARN([libharfbuzz not found. Assuming qt has it built-in or support is disabled])))
-  BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Core]   ,[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Core not found)))
-  BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Gui]    ,[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Gui not found)))
-  BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Network],[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Network not found)))
-  BITCOIN_QT_CHECK(AC_CHECK_LIB([${QT_LIB_PREFIX}Widgets],[main],,BITCOIN_QT_FAIL(lib${QT_LIB_PREFIX}Widgets not found)))
-  QT_LIBS="$LIBS"
-  LIBS="$TEMP_LIBS"
-
-  BITCOIN_QT_CHECK([
-    LIBS=
-    if test "x$qt_lib_path" != x; then
-      LIBS="-L$qt_lib_path"
-    fi
-    AC_CHECK_LIB([${QT_LIB_PREFIX}Test],      [main],, have_qt_test=no)
-    AC_CHECK_HEADER([QTest],, have_qt_test=no)
-    QT_TEST_LIBS="$LIBS"
+    PKG_CHECK_MODULES([QT_TEST], [${qt_lib_prefix}Test $qt_version], [QT_TEST_INCLUDES="$QT_TEST_CFLAGS"; have_qt_test=yes], [have_qt_test=no])
     if test "x$use_dbus" != xno; then
-      LIBS=
-      if test "x$qt_lib_path" != x; then
-        LIBS="-L$qt_lib_path"
-      fi
-      AC_CHECK_LIB([${QT_LIB_PREFIX}DBus],      [main],, have_qt_dbus=no)
-      AC_CHECK_HEADER([QtDBus],, have_qt_dbus=no)
-      QT_DBUS_LIBS="$LIBS"
+      PKG_CHECK_MODULES([QT_DBUS], [${qt_lib_prefix}DBus $qt_version], [QT_DBUS_INCLUDES="$QT_DBUS_CFLAGS"; have_qt_dbus=yes], [have_qt_dbus=no])
     fi
   ])
-  CPPFLAGS="$TEMP_CPPFLAGS"
-  CXXFLAGS="$TEMP_CXXFLAGS"
-  LIBS="$TEMP_LIBS"
 ])

build-aux/m4/l_socket.m4

@@ -0,0 +1,36 @@
+# Illumos/SmartOS requires linking with -lsocket if
+# using getifaddrs & freeifaddrs
+
+m4_define([_CHECK_SOCKET_testbody], [[
+  #include <sys/types.h>
+  #include <ifaddrs.h>
+
+  int main() {
+    struct ifaddrs *ifaddr;
+    getifaddrs(&ifaddr);
+    freeifaddrs(ifaddr);
+  }
+]])
+
+AC_DEFUN([CHECK_SOCKET], [
+
+  AC_LANG_PUSH(C++)
+
+  AC_MSG_CHECKING([whether ifaddrs funcs can be used without link library])
+
+  AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_SOCKET_testbody])],[
+      AC_MSG_RESULT([yes])
+    ],[
+      AC_MSG_RESULT([no])
+      LIBS="$LIBS -lsocket"
+      AC_MSG_CHECKING([whether getifaddrs needs -lsocket])
+      AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_SOCKET_testbody])],[
+          AC_MSG_RESULT([yes])
+        ],[
+          AC_MSG_RESULT([no])
+          AC_MSG_FAILURE([cannot figure out how to use getifaddrs])
+        ])
+    ])
+
+  AC_LANG_POP
+])

build_msvc/.gitignore

@@ -8,7 +8,20 @@ packages/*
 */Release
 */x64
 *.vcxproj.user
-*.vcxproj
+
+# .vcxproj files that are auto-generated by the msvc-autogen.py script.
+libbitcoin_cli/libbitcoin_cli.vcxproj
+libbitcoin_common/libbitcoin_common.vcxproj
+libbitcoin_crypto/libbitcoin_crypto.vcxproj
+libbitcoin_server/libbitcoin_server.vcxproj
+libbitcoin_util/libbitcoin_util.vcxproj
+libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj
+libbitcoin_wallet/libbitcoin_wallet.vcxproj
+libbitcoin_zmq/libbitcoin_zmq.vcxproj
+bench_bitcoin/bench_bitcoin.vcxproj
+libtest_util/libtest_util.vcxproj
+
 */Win32
 libbitcoin_qt/QtGeneratedFiles/*
 test_bitcoin-qt/QtGeneratedFiles/*
+vcpkg_installed

build_msvc/README.md

@@ -3,7 +3,7 @@ Building Bitcoin Core with Visual Studio
 
 Introduction
 ---------------------
-Solution and project files to build the Bitcoin Core applications `msbuild` or Visual Studio can be found in the build_msvc directory. The build has been tested with Visual Studio 2017 and 2019.
+Solution and project files to build the Bitcoin Core applications `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2017 and 2019.
 
 Building with Visual Studio is an alternative to the Linux based [cross-compiler build](https://github.com/bitcoin/bitcoin/blob/master/doc/build-windows.md).
 
@@ -12,9 +12,9 @@ Quick Start
 The minimal steps required to build Bitcoin Core with the msbuild toolchain are below. More detailed instructions are contained in the following sections.
 
 ```
-vcpkg install --triplet x64-windows-static boost-filesystem boost-signals2 boost-test libevent openssl zeromq berkeleydb rapidcheck double-conversion
-py -3 build_msvc\msvc-autogen.py
-msbuild /m build_msvc\bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build
+cd build_msvc
+py -3 msvc-autogen.py
+msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build
 ```
 
 Dependencies
@@ -27,61 +27,45 @@ Options for installing the dependencies in a Visual Studio compatible manner are
 - Download the source code, build each dependency, add the required include paths, link libraries and binary tools to the Visual Studio project files.
 - Use [nuget](https://www.nuget.org/) packages with the understanding that any binary files have been compiled by an untrusted third party.
 
-The [external dependencies](https://github.com/bitcoin/bitcoin/blob/master/doc/dependencies.md) required for building are:
-
-- Berkeley DB
-- Boost
-- DoubleConversion
-- libevent
-- OpenSSL
-- Qt5
-- RapidCheck
-- ZeroMQ
+The [external dependencies](https://github.com/bitcoin/bitcoin/blob/master/doc/dependencies.md) required for building are listed in the `build_msvc/vcpkg.json` file. The `msbuild` project files are configured to automatically install the `vcpkg` dependencies.
 
 Qt
 ---------------------
-All the Bitcoin Core applications are configured to build with static linking. In order to build the Bitcoin Core Qt applications a static build of Qt is required.
+In order to build the Bitcoin Core a static build of Qt is required. The runtime library version (e.g. v141, v142) and platform type (x86 or x64) must also match.
 
-The runtime library version (e.g. v141, v142) and platform type (x86 or x64) must also match. OpenSSL must also be linked into the Qt binaries in order to provide full functionality of the Bitcoin Core Qt programs. An example of the configure command to build Qtv5.9.7 locally to link with Bitcoin Core is shown below (adjust paths accordingly), note it can be expected that the configure and subsequent build will fail numerous times until dependency issues are resolved.
+Some prebuilt x64 versions of Qt can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware these downloads are NOT officially sanctioned by Bitcoin Core and are provided for developer convenience only. They should NOT be used for builds that will be used in a production environment or with real funds.
 
-````
-..\Qtv5.9.7_src\configure -developer-build -confirm-license -debug-and-release -opensource -platform win32-msvc -opengl desktop -no-shared -static -no-static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -ltcg -make libs -make tools -no-libjpeg -nomake examples -no-compile-examples -no-dbus -no-libudev -no-qml-debug -no-icu -no-gtk -no-opengles3 -no-angle -no-sql-sqlite -no-sql-odbc -no-sqlite -no-libudev -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcanvas3d -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquickcontrols -skip qtquickcontrols2 -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -nomake tests -openssl-linked -IC:\Dev\github\vcpkg\installed\x64-windows-static\include -LC:\Dev\github\vcpkg\installed\x64-windows-static\lib OPENSSL_LIBS="-llibeay32 -lssleay32 -lgdi32 -luser32 -lwsock32 -ladvapi32" -prefix C:\Qt5.9.7_ssl_x64_static_vs2017
-````
+To determine which Qt prebuilt version to download open the `.appveyor.yml` file and note the `QT_DOWNLOAD_URL`. When extracting the zip file the destination path must be set to `C:\`. This is due to the way that Qt includes, libraries and tools use internal paths.
 
-A prebuilt version for x64 and Visual C++ runtime v141 (Visual Studio 2017) can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware this download is NOT an officially sanctioned Bitcoin Core distribution and is provided for developer convenience. It should NOT be used for builds that will be used in a production environment or with real funds.
-
-To build Bitcoin Core without Qt unload or disable the bitcoin-qt, libbitcoin_qt and test_bitcoin-qt projects.
+To build Bitcoin Core without Qt unload or disable the `bitcoin-qt`, `libbitcoin_qt` and `test_bitcoin-qt` projects.
 
 Building
 ---------------------
 The instructions below use `vcpkg` to install the dependencies.
 
-- Clone `vcpkg` from the [github repository](https://github.com/Microsoft/vcpkg) and install as per the instructions in the main README.md.
-- Install the required packages (replace x64 with x86 as required):
+- Install [`vcpkg`](https://github.com/Microsoft/vcpkg).
+
+- Use Python to generate `*.vcxproj` from Makefile
 
 ```
-    PS >.\vcpkg install --triplet x64-windows-static boost-filesystem boost-signals2 boost-test libevent openssl zeromq berkeleydb rapidcheck double-conversion
+PS >py -3 msvc-autogen.py
 ```
 
-- Use Python to generate *.vcxproj from Makefile
+- An optional step is to adjust the settings in the `build_msvc` directory and the `common.init.vcxproj` file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set.
+
+- To build from the command line with the Visual Studio 2017 toolchain use:
 
 ```
-    PS >py -3 msvc-autogen.py
+msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /p:PlatformToolset=v141 /t:build
 ```
 
-- An optional step is to adjust the settings in the build_msvc directory and the common.init.vcxproj file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set.
-
-- Build with Visual Studio 2017 or msbuild.
+- To build from the command line with the Visual Studio 2019 toolchain use:
 
 ```
 msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build
 ```
 
-- Build with Visual Studio 2019 or msbuild.
-
-```
-msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /p:PlatformToolset=v142 /t:build
-```
+- Alternatively open the `build_msvc/bitcoin.sln` file in Visual Studio.
 
 AppVeyor
 ---------------------

build_msvc/bench_bitcoin/bench_bitcoin.vcxproj.in

@@ -42,6 +42,9 @@
     <ProjectReference Include="..\libleveldb\libleveldb.vcxproj">
       <Project>{18430fef-6b61-4c53-b396-718e02850f1b}</Project>
     </ProjectReference>
+    <ProjectReference Include="..\libtest_util\libtest_util.vcxproj">
+      <Project>{1e065f03-3566-47d0-8fa9-daa72b084e7d}</Project>
+    </ProjectReference>
   </ItemGroup>
   <Target Name="RawBenchHeaderGen" BeforeTargets="PrepareForBuild">
     <PropertyGroup>

build_msvc/bitcoin-qt/bitcoin-qt.vcxproj

@@ -50,12 +50,13 @@
     </ProjectReference>
   </ItemGroup>
 
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
     <ClCompile>
       <AdditionalIncludeDirectories>$(QtIncludes);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <AdditionalDependencies>$(QtReleaseLibraries);%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalOptions>/ignore:4206 /LTCG:OFF</AdditionalOptions>
     </Link>
     <ResourceCompile>
       <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories>
@@ -63,12 +64,13 @@
     </ResourceCompile>
   </ItemDefinitionGroup>
 
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
     <ClCompile>
       <AdditionalIncludeDirectories>$(QtIncludes);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <AdditionalDependencies>$(QtDebugLibraries);%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalOptions>/ignore:4206</AdditionalOptions>
     </Link>
     <ResourceCompile>
       <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories>

build_msvc/bitcoin.sln

@@ -44,6 +44,10 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_qt", "libbitcoin
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-qt", "bitcoin-qt\bitcoin-qt.vcxproj", "{7E99172D-7FF2-4CB6-B736-AC9B76ED412A}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libtest_util", "libtest_util\libtest_util.vcxproj", "{868474FD-35F6-4400-8EED-30A33E7521D4}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_bitcoin-qt", "test_bitcoin-qt\test_bitcoin-qt.vcxproj", "{51201D5E-D939-4854-AE9D-008F03FF518E}"
+EndProject
 Global
     GlobalSection(SolutionConfigurationPlatforms) = preSolution
         Debug|x64 = Debug|x64
@@ -220,11 +224,29 @@ Global
         {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x64.Build.0 = Release|x64
         {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x86.ActiveCfg = Release|Win32
         {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x86.Build.0 = Release|Win32
+        {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x64.ActiveCfg = Debug|x64
+        {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x64.Build.0 = Debug|x64
+        {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x86.ActiveCfg = Debug|Win32
+        {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x86.Build.0 = Debug|Win32
+        {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x64.ActiveCfg = Release|x64
+        {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x64.Build.0 = Release|x64
+        {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x86.ActiveCfg = Release|Win32
+        {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x86.Build.0 = Release|Win32
+        {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.ActiveCfg = Debug|x64
+        {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.Build.0 = Debug|x64
+        {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x86.ActiveCfg = Debug|Win32
+        {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x86.Build.0 = Debug|Win32
+        {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.ActiveCfg = Release|x64
+        {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.Build.0 = Release|x64
+        {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x86.ActiveCfg = Release|Win32
+        {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x86.Build.0 = Release|Win32
     EndGlobalSection
     GlobalSection(SolutionProperties) = preSolution
         HideSolutionNode = FALSE
     EndGlobalSection
     GlobalSection(ExtensibilityGlobals) = postSolution
-        SolutionGuid = {8AA72EDA-2CD4-4564-B1E4-688B760EEEE9}
+                        SolutionGuid = {8AA72EDA-2CD4-4564-B1E4-688B760EEEE9}
+                SolutionGuid = {8607C0F4-F33D-41B8-8D51-18E366A0F8DF}
+        SolutionGuid = {58AAB032-7274-49BD-845E-5EF4DBB69B70}
     EndGlobalSection
 EndGlobal

build_msvc/bitcoin_config.h

@@ -1,3 +1,7 @@
+// Copyright (c) 2018-2020 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
 #ifndef BITCOIN_BITCOIN_CONFIG_H
 #define BITCOIN_BITCOIN_CONFIG_H
 
@@ -8,16 +12,16 @@
 #define CLIENT_VERSION_BUILD 0
 
 /* Version is release */
-#define CLIENT_VERSION_IS_RELEASE false
+#define CLIENT_VERSION_IS_RELEASE true
 
 /* Major version */
 #define CLIENT_VERSION_MAJOR 0
 
 /* Minor version */
-#define CLIENT_VERSION_MINOR 18
+#define CLIENT_VERSION_MINOR 21
 
 /* Build revision */
-#define CLIENT_VERSION_REVISION 99
+#define CLIENT_VERSION_REVISION 2
 
 /* Copyright holder(s) before %s replacement */
 #define COPYRIGHT_HOLDERS "The %s developers"
@@ -37,20 +41,14 @@
 /* Define to 1 to enable ZMQ functions */
 #define ENABLE_ZMQ 1
 
-/* parameter and return value type for __fdelt_chk */
-/* #undef FDELT_TYPE */
-
 /* define if the Boost library is available */
 #define HAVE_BOOST /**/
 
-/* define if the Boost::Chrono library is available */
-#define HAVE_BOOST_CHRONO /**/
-
 /* define if the Boost::Filesystem library is available */
 #define HAVE_BOOST_FILESYSTEM /**/
 
-/* define if the Boost::PROGRAM_OPTIONS library is available */
-#define HAVE_BOOST_PROGRAM_OPTIONS /**/
+/* define if the Boost::Process library is available */
+#define HAVE_BOOST_PROCESS /**/
 
 /* define if the Boost::System library is available */
 #define HAVE_BOOST_SYSTEM /**/
@@ -98,10 +96,6 @@
    */
 #define HAVE_DECL_DAEMON 0
 
-/* Define to 1 if you have the declaration of `EVP_MD_CTX_new', and to 0 if
-   you don't. */
-//#define HAVE_DECL_EVP_MD_CTX_NEW 1
-
 /* Define to 1 if you have the declaration of `htobe16', and to 0 if you
    don't. */
 #define HAVE_DECL_HTOBE16 0
@@ -146,18 +140,6 @@
    don't. */
 #define HAVE_DECL_STRNLEN 1
 
-/* Define to 1 if you have the declaration of `__builtin_clz', and to 0 if you
-   don't. */
-//#define HAVE_DECL___BUILTIN_CLZ 1
-
-/* Define to 1 if you have the declaration of `__builtin_clzl', and to 0 if
-   you don't. */
-//#define HAVE_DECL___BUILTIN_CLZL 1
-
-/* Define to 1 if you have the declaration of `__builtin_clzll', and to 0 if
-   you don't. */
-//#define HAVE_DECL___BUILTIN_CLZLL 1
-
 /* Define to 1 if you have the <dlfcn.h> header file. */
 /* #undef HAVE_DLFCN_H */
 
@@ -183,75 +165,6 @@
 /* Define to 1 if you have the <inttypes.h> header file. */
 #define HAVE_INTTYPES_H 1
 
-/* Define to 1 if you have the `advapi32' library (-ladvapi32). */
-#define HAVE_LIBADVAPI32 1
-
-/* Define to 1 if you have the `comctl32' library (-lcomctl32). */
-#define HAVE_LIBCOMCTL32 1
-
-/* Define to 1 if you have the `comdlg32' library (-lcomdlg32). */
-#define HAVE_LIBCOMDLG32 1
-
-/* Define to 1 if you have the `crypt32' library (-lcrypt32). */
-#define HAVE_LIBCRYPT32 1
-
-/* Define to 1 if you have the `gdi32' library (-lgdi32). */
-#define HAVE_LIBGDI32 1
-
-/* Define to 1 if you have the `imm32' library (-limm32). */
-#define HAVE_LIBIMM32 1
-
-/* Define to 1 if you have the `iphlpapi' library (-liphlpapi). */
-#define HAVE_LIBIPHLPAPI 1
-
-/* Define to 1 if you have the `kernel32' library (-lkernel32). */
-#define HAVE_LIBKERNEL32 1
-
-/* Define to 1 if you have the `mingwthrd' library (-lmingwthrd). */
-#define HAVE_LIBMINGWTHRD 1
-
-/* Define to 1 if you have the `mswsock' library (-lmswsock). */
-#define HAVE_LIBMSWSOCK 1
-
-/* Define to 1 if you have the `ole32' library (-lole32). */
-#define HAVE_LIBOLE32 1
-
-/* Define to 1 if you have the `oleaut32' library (-loleaut32). */
-#define HAVE_LIBOLEAUT32 1
-
-/* Define to 1 if you have the `rpcrt4' library (-lrpcrt4). */
-#define HAVE_LIBRPCRT4 1
-
-/* Define to 1 if you have the `rt' library (-lrt). */
-/* #undef HAVE_LIBRT */
-
-/* Define to 1 if you have the `shell32' library (-lshell32). */
-#define HAVE_LIBSHELL32 1
-
-/* Define to 1 if you have the `shlwapi' library (-lshlwapi). */
-#define HAVE_LIBSHLWAPI 1
-
-/* Define to 1 if you have the `ssp' library (-lssp). */
-#define HAVE_LIBSSP 1
-
-/* Define to 1 if you have the `user32' library (-luser32). */
-#define HAVE_LIBUSER32 1
-
-/* Define to 1 if you have the `uuid' library (-luuid). */
-#define HAVE_LIBUUID 1
-
-/* Define to 1 if you have the `winmm' library (-lwinmm). */
-#define HAVE_LIBWINMM 1
-
-/* Define to 1 if you have the `winspool' library (-lwinspool). */
-#define HAVE_LIBWINSPOOL 1
-
-/* Define to 1 if you have the `ws2_32' library (-lws2_32). */
-#define HAVE_LIBWS2_32 1
-
-/* Define to 1 if you have the `z ' library (-lz ). */
-#define HAVE_LIBZ_ 1
-
 /* Define this symbol if you have malloc_info */
 /* #undef HAVE_MALLOC_INFO */
 
@@ -330,12 +243,6 @@
 /* Define if the visibility attribute is supported. */
 #define HAVE_VISIBILITY_ATTRIBUTE 1
 
-/* Define this symbol if boost sleep works */
-/* #undef HAVE_WORKING_BOOST_SLEEP */
-
-/* Define this symbol if boost sleep_for works */
-#define HAVE_WORKING_BOOST_SLEEP_FOR 1
-
 /* Define to the sub-directory where libtool stores uninstalled libraries. */
 #define LT_OBJDIR ".libs/"
 
@@ -346,7 +253,7 @@
 #define PACKAGE_NAME "Bitcoin Core"
 
 /* Define to the full name and version of this package. */
-#define PACKAGE_STRING "Bitcoin Core 0.18.99"
+#define PACKAGE_STRING "Bitcoin Core 0.21.2"
 
 /* Define to the one symbol short name of this package. */
 #define PACKAGE_TARNAME "bitcoin"
@@ -355,7 +262,7 @@
 #define PACKAGE_URL "https://bitcoincore.org/"
 
 /* Define to the version of this package. */
-#define PACKAGE_VERSION "0.18.99"
+#define PACKAGE_VERSION "0.21.2"
 
 /* Define to necessary symbol if this constant uses a non-standard name on
    your system. */
@@ -385,9 +292,6 @@
 /* Define this symbol to build in assembly routines */
 //#define USE_ASM 1
 
-/* Define this symbol if coverage is enabled */
-/* #undef USE_COVERAGE */
-
 /* Define if dbus support should be compiled in */
 /* #undef USE_DBUS */
 

build_msvc/common.init.vcxproj

@@ -4,9 +4,19 @@
 
   <PropertyGroup Label="Globals">
     <VCProjectVersion>16.0</VCProjectVersion>
+    <UseNativeEnvironment>true</UseNativeEnvironment>
+   </PropertyGroup>
+
+   <PropertyGroup Label="Vcpkg">
+    <VcpkgEnabled>true</VcpkgEnabled>
+    <VcpkgEnableManifest>true</VcpkgEnableManifest>
+    <VcpkgManifestInstall>true</VcpkgManifestInstall>
+    <VcpkgUseStatic>true</VcpkgUseStatic>
+    <VcpkgAutoLink>true</VcpkgAutoLink>
+    <VcpkgConfiguration>$(Configuration)</VcpkgConfiguration>
     <VcpkgTriplet Condition="'$(Platform)'=='Win32'">x86-windows-static</VcpkgTriplet>
     <VcpkgTriplet Condition="'$(Platform)'=='x64'">x64-windows-static</VcpkgTriplet>
-   </PropertyGroup>
+  </PropertyGroup>
 
   <PropertyGroup Condition="'$(WindowsTargetPlatformVersion)'=='' and !Exists('$(WindowsSdkDir)\DesignTime\CommonConfiguration\Neutral\Windows.props')">
      <WindowsTargetPlatformVersion_10 Condition="'$(WindowsTargetPlatformVersion_10)' == ''">$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion)</WindowsTargetPlatformVersion_10>
@@ -35,66 +45,46 @@
     </ProjectConfiguration>
   </ItemGroup>
 
-  <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
-    <LinkIncremental>true</LinkIncremental>
-    <WholeProgramOptimization>false</WholeProgramOptimization>
-    <UseDebugLibraries>true</UseDebugLibraries>
-    <PlatformToolset>v141</PlatformToolset>
-    <CharacterSet>Unicode</CharacterSet>
-    <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir>
-    <IntDir>$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
-  </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
     <LinkIncremental>false</LinkIncremental>
-    <WholeProgramOptimization>true</WholeProgramOptimization>
     <UseDebugLibraries>false</UseDebugLibraries>
-    <PlatformToolset>v141</PlatformToolset>
+    <PlatformToolset>v142</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+    <GenerateManifest>No</GenerateManifest>
+    <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir>
+    <IntDir>$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
+    <LinkIncremental>true</LinkIncremental>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v142</PlatformToolset>
     <CharacterSet>Unicode</CharacterSet>
     <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\</OutDir>
     <IntDir>$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
   </PropertyGroup>
 
-<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+<ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
     <ClCompile>
-      <Optimization>MaxSpeed</Optimization>
+      <Optimization>Disabled</Optimization>
+      <WholeProgramOptimization>false</WholeProgramOptimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <DebugInformationFormat>None</DebugInformationFormat>
     </ClCompile>
     <Link>
-      <EnableCOMDATFolding>true</EnableCOMDATFolding>
-      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>false</EnableCOMDATFolding>
+      <OptimizeReferences>false</OptimizeReferences>
+      <AdditionalOptions>/LTCG:OFF</AdditionalOptions>
     </Link>
   </ItemDefinitionGroup>
 
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
-    <ClCompile>
-      <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <SDLCheck>true</SDLCheck>
-      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
-      <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
-    </ClCompile>
-  </ItemDefinitionGroup>
-
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
-    <ClCompile>
-      <Optimization>MaxSpeed</Optimization>
-      <FunctionLevelLinking>true</FunctionLevelLinking>
-      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <SDLCheck>true</SDLCheck>
-      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
-    </ClCompile>
-    <Link>
-      <EnableCOMDATFolding>true</EnableCOMDATFolding>
-      <OptimizeReferences>true</OptimizeReferences>
-    </Link>
-  </ItemDefinitionGroup>
-
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
     <ClCompile>
       <Optimization>Disabled</Optimization>
+      <WholeProgramOptimization>false</WholeProgramOptimization>
       <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <SDLCheck>true</SDLCheck>
       <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
@@ -106,16 +96,15 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <PrecompiledHeader>NotUsing</PrecompiledHeader>
-      <AdditionalOptions>/utf-8 %(AdditionalOptions)</AdditionalOptions>
-      <DisableSpecificWarnings>4018;4221;4244;4267;4334;4715;4805;</DisableSpecificWarnings>
+      <AdditionalOptions>/utf-8 /std:c++17 %(AdditionalOptions)</AdditionalOptions>
+      <DisableSpecificWarnings>4018;4221;4244;4267;4334;4715;4805;4834</DisableSpecificWarnings>
       <TreatWarningAsError>true</TreatWarningAsError>
-      <PreprocessorDefinitions>ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
-      <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalDependencies>crypt32.lib;Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>Iphlpapi.lib;ws2_32.lib;Shlwapi.lib;kernel32.lib;user32.lib;gdi32.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
     <Lib>
       <AdditionalOptions>/ignore:4221</AdditionalOptions>

build_msvc/common.qt.init.vcxproj

@@ -2,7 +2,7 @@
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 
   <PropertyGroup Label="QtGlobals">
-    <QtBaseDir>C:\Qt5.9.7_ssl_x64_static_vs2017</QtBaseDir>
+    <QtBaseDir>C:\Qt5.9.8_x64_static_vs2019</QtBaseDir>
     <QtPluginsLibraryDir>$(QtBaseDir)\plugins</QtPluginsLibraryDir>
     <QtLibraryDir>$(QtBaseDir)\lib</QtLibraryDir>
     <QtIncludeDir>$(QtBaseDir)\include</QtIncludeDir>

build_msvc/common.vcxproj

@@ -4,7 +4,7 @@
   <Target Name="CopyBuildArtifacts" Condition="'$(ConfigurationType)' != 'StaticLibrary'">
     <ItemGroup>
       <BuildArtifacts Include="$(OutDir)$(TargetName)$(TargetExt)"></BuildArtifacts>
-      <BuildArtifacts Include="$(OutDir)$(TargetName).pdb"></BuildArtifacts>
+      <BuildArtifacts Include="$(OutDir)$(TargetName).pdb" Condition="Exists('$(OutDir)$(TargetName).pdb')"></BuildArtifacts>
     </ItemGroup>
     <Copy SourceFiles="@(BuildArtifacts)" SkipUnchangedFiles="true" DestinationFolder="..\..\src\" Condition="'$(OutDir)' != ''"></Copy>
   </Target>

build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj

@@ -35,6 +35,7 @@
     <ClCompile Include="..\..\src\qt\paymentserver.cpp" />
     <ClCompile Include="..\..\src\qt\peertablemodel.cpp" />
     <ClCompile Include="..\..\src\qt\platformstyle.cpp" />
+    <ClCompile Include="..\..\src\qt\psbtoperationsdialog.cpp" />
     <ClCompile Include="..\..\src\qt\qrimagewidget.cpp" />
     <ClCompile Include="..\..\src\qt\qvalidatedlineedit.cpp" />
     <ClCompile Include="..\..\src\qt\qvaluecombobox.cpp" />
@@ -87,6 +88,7 @@
     <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_paymentserver.cpp" />
     <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_peertablemodel.cpp" />
     <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_platformstyle.cpp" />
+    <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_psbtoperationsdialog.cpp" />
     <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_qrimagewidget.cpp" />
     <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_qvalidatedlineedit.cpp" />
     <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_qvaluecombobox.cpp" />
@@ -102,6 +104,7 @@
     <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactiondesc.cpp" />
     <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactiondescdialog.cpp" />
     <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactionfilterproxy.cpp" />
+    <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactionoverviewwidget.cpp" />
     <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactionrecord.cpp" />
     <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactiontablemodel.cpp" />
     <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_transactionview.cpp" />

build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in

@@ -8,6 +8,7 @@
     <ConfigurationType>StaticLibrary</ConfigurationType>
   </PropertyGroup>
   <ItemGroup>
+    <ClCompile Include="..\..\src\util\url.cpp" />
 @SOURCE_FILES@
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />

build_msvc/libleveldb/libleveldb.vcxproj

@@ -24,8 +24,6 @@
     <ClCompile Include="..\..\src\leveldb\db\version_set.cc" />
     <ClCompile Include="..\..\src\leveldb\db\write_batch.cc" />
     <ClCompile Include="..\..\src\leveldb\helpers\memenv\memenv.cc" />
-    <ClCompile Include="..\..\src\leveldb\port\port_posix_sse.cc" />
-    <ClCompile Include="..\..\src\leveldb\port\port_win.cc" />
     <ClCompile Include="..\..\src\leveldb\table\block.cc" />
     <ClCompile Include="..\..\src\leveldb\table\block_builder.cc" />
     <ClCompile Include="..\..\src\leveldb\table\filter_block.cc" />
@@ -42,7 +40,7 @@
     <ClCompile Include="..\..\src\leveldb\util\comparator.cc" />
     <ClCompile Include="..\..\src\leveldb\util\crc32c.cc" />
     <ClCompile Include="..\..\src\leveldb\util\env.cc" />
-    <ClCompile Include="..\..\src\leveldb\util\env_win.cc" />
+    <ClCompile Include="..\..\src\leveldb\util\env_windows.cc" />
     <ClCompile Include="..\..\src\leveldb\util\filter_policy.cc" />
     <ClCompile Include="..\..\src\leveldb\util\hash.cc" />
     <ClCompile Include="..\..\src\leveldb\util\histogram.cc" />
@@ -51,11 +49,11 @@
     <ClCompile Include="..\..\src\leveldb\util\status.cc" />
   </ItemGroup>
   <ItemDefinitionGroup>
-    <ClCompile>
-	 <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;LEVELDB_PLATFORM_WINDOWS;LEVELDB_ATOMIC_PRESENT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-	 <DisableSpecificWarnings>4244;4267;4312;</DisableSpecificWarnings>
-	 <AdditionalIncludeDirectories>..\..\src\leveldb;..\..\src\leveldb\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-	</ClCompile>
+     <ClCompile>
+       <PreprocessorDefinitions>HAVE_CRC32C=0;HAVE_SNAPPY=0;__STDC_LIMIT_MACROS;LEVELDB_IS_BIG_ENDIAN=0;_UNICODE;UNICODE;_CRT_NONSTDC_NO_DEPRECATE;LEVELDB_PLATFORM_WINDOWS;LEVELDB_ATOMIC_PRESENT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+       <DisableSpecificWarnings>4244;4267;4312;4722;</DisableSpecificWarnings>
+       <AdditionalIncludeDirectories>..\..\src\leveldb;..\..\src\leveldb\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+     </ClCompile>
   </ItemDefinitionGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />

build_msvc/libsecp256k1/libsecp256k1.vcxproj

@@ -12,7 +12,7 @@
   </ItemGroup>
     <ItemDefinitionGroup>
     <ClCompile>
-      <PreprocessorDefinitions>ENABLE_MODULE_ECDH;ENABLE_MODULE_RECOVERY;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ENABLE_MODULE_ECDH;ENABLE_MODULE_RECOVERY;ENABLE_MODULE_EXTRAKEYS;ENABLE_MODULE_SCHNORRSIG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
        <AdditionalIncludeDirectories>..\..\src\secp256k1;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
      </ClCompile>
   </ItemDefinitionGroup>

build_msvc/libsecp256k1_config.h

@@ -26,4 +26,7 @@
 #define USE_FIELD_10X26 1
 #define USE_SCALAR_8X32 1
 
+#define ECMULT_GEN_PREC_BITS 4
+#define ECMULT_WINDOW_SIZE 15
+
 #endif /* BITCOIN_LIBSECP256K1_CONFIG_H */

build_msvc/libtest_util/libtest_util.vcxproj.in

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="..\common.init.vcxproj" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{868474FD-35F6-4400-8EED-30A33E7521D4}</ProjectGuid>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+  </PropertyGroup>
+  <ItemGroup>
+@SOURCE_FILES@
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <Import Project="..\common.vcxproj" />
+</Project>

build_msvc/msvc-autogen.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+# Copyright (c) 2016-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 import os
 import re
@@ -18,6 +21,7 @@ libs = [
     'libbitcoin_wallet',
     'libbitcoin_zmq',
     'bench_bitcoin',
+    'libtest_util',
 ]
 
 ignore_list = [

build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj

@@ -1,16 +1,14 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="..\common.init.vcxproj" />
   <Import Project="..\common.qt.init.vcxproj" />
   <PropertyGroup Label="Globals">
     <ProjectGuid>{51201D5E-D939-4854-AE9D-008F03FF518E}</ProjectGuid>
-  </PropertyGroup>
-  <PropertyGroup Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
     <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
   </PropertyGroup>
   <ItemGroup>
-    <ClCompile Include="..\..\src\test\setup_common.cpp" />
+    <ClCompile Include="..\..\src\test\util\setup_common.cpp" />
     <ClCompile Include="..\..\src\qt\test\addressbooktests.cpp" />
     <ClCompile Include="..\..\src\qt\test\apptests.cpp" />
     <ClCompile Include="..\..\src\qt\test\compattests.cpp" />
@@ -66,29 +64,32 @@
     </ProjectReference>
   </ItemGroup>
 
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+
+  <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
     <ClCompile>
       <AdditionalIncludeDirectories>..\libbitcoin_qt\$(GeneratedFilesOutDir)\..\;$(QtIncludeDir)\QtTest;$(QtIncludes);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>$(QtReleaseLibaries);%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>$(QtLibraryDir)\Qt5Test.lib;$(QtReleaseLibraries);%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalOptions>/ignore:4206 /LTCG:OFF</AdditionalOptions>
     </Link>
   </ItemDefinitionGroup>
 
-    <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
     <ClCompile>
       <AdditionalIncludeDirectories>..\libbitcoin_qt\$(GeneratedFilesOutDir)\..\;$(QtIncludeDir)\QtTest;$(QtIncludes);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
       <AdditionalDependencies>$(QtDebugLibraries);%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalOptions>/ignore:4206</AdditionalOptions>
     </Link>
   </ItemDefinitionGroup>
-
   <ItemGroup>
     <MocTestFiles Include="..\..\src\qt\test\addressbooktests.h" />
     <MocTestFiles Include="..\..\src\qt\test\apptests.h" />
     <MocTestFiles Include="..\..\src\qt\test\compattests.h" />
-    <MocTestFiles Include="..\..\src\qt\test\paymentservertests.h" />
     <MocTestFiles Include="..\..\src\qt\test\rpcnestedtests.h" />
     <MocTestFiles Include="..\..\src\qt\test\uritests.h" />
     <MocTestFiles Include="..\..\src\qt\test\wallettests.h" />
@@ -106,8 +107,6 @@
     <RemoveDir Directories="$(GeneratedFilesOutDir)\moc\*" />
     <RemoveDir Directories="$(GeneratedFilesOutDir)\moc" />
   </Target>
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <PropertyGroup>
     <BuildDependsOn>
        moccode;
@@ -120,4 +119,4 @@
         $(CleanDependsOn);
     </CleanDependsOn>
   </PropertyGroup>
- </Project>
+</Project>

build_msvc/test_bitcoin/test_bitcoin.vcxproj

@@ -9,13 +9,13 @@
     <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
   </PropertyGroup>
   <ItemGroup>
-    <ClCompile Include="..\..\src\test\*_tests.cpp" />
     <ClCompile Include="..\..\src\test\*_properties.cpp" />
+    <ClCompile Include="..\..\src\test\*_tests.cpp" />
     <ClCompile Include="..\..\src\test\gen\*_gen.cpp" />
-    <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" />
-    <ClCompile Include="..\..\src\test\setup_common.cpp" />
     <ClCompile Include="..\..\src\test\main.cpp" />
+    <ClCompile Include="..\..\src\test\util\*.cpp" />
     <ClCompile Include="..\..\src\wallet\test\*_fixture.cpp" />
+    <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj">
@@ -42,6 +42,9 @@
     <ProjectReference Include="..\libbitcoin_zmq\libbitcoin_zmq.vcxproj">
       <Project>{792d487f-f14c-49fc-a9de-3fc150f31c3f}</Project>
     </ProjectReference>
+    <ProjectReference Include="..\libtest_util\libtest_util.vcxproj">
+      <Project>{1e065f03-3566-47d0-8fa9-daa72b084e7d}</Project>
+    </ProjectReference>
     <ProjectReference Include="..\libunivalue\libunivalue.vcxproj">
       <Project>{5724ba7d-a09a-4ba8-800b-c4c1561b3d69}</Project>
     </ProjectReference>
@@ -58,7 +61,9 @@
     </PropertyGroup>
     <ItemGroup>
       <JsonTestFile Include="..\..\src\test\data\*.json" />
+      <RawTestFile Include="..\..\src\test\data\*.raw" />
     </ItemGroup>
+    <HeaderFromHexdump RawFilePath="%(RawTestFile.FullPath)" HeaderFilePath="%(RawTestFile.FullPath).h" SourceHeader="static unsigned const char %(RawTestFile.Filename)_raw[] = {" SourceFooter="};" />
     <HeaderFromHexdump RawFilePath="%(JsonTestFile.FullPath)" HeaderFilePath="%(JsonTestFile.FullPath).h" SourceHeader="namespace json_tests{ static unsigned const char %(JsonTestFile.Filename)[] = {" SourceFooter="};}" />
   </Target>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />

build_msvc/testconsensus/testconsensus.cpp

@@ -1,3 +1,7 @@
+// Copyright (c) 2018-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
 #include <iostream>
 
 // bitcoin includes.

build_msvc/vcpkg.json

@@ -0,0 +1,20 @@
+{
+  "name": "bitcoin-core",
+  "version-string": "1",
+  "dependencies": [
+    "berkeleydb",
+    "boost-filesystem",
+    "boost-multi-index",
+    "boost-process",
+    "boost-signals2",
+    "boost-test",
+    "boost-thread",
+    "sqlite3",
+    "double-conversion",
+    {
+      "name": "libevent",
+      "features": ["thread"]
+    },
+    "zeromq"
+  ]
+}

ci/README.md

@@ -1,18 +1,24 @@
-## ci scripts
+## CI Scripts
 
 This directory contains scripts for each build step in each build stage.
 
-Currently three stages `lint`, `extended_lint` and `test` are defined. Each stage has its own lifecycle, similar to the
-[Travis CI lifecycle](https://docs.travis-ci.com/user/job-lifecycle#the-job-lifecycle). Every script in here is named
-and numbered according to which stage and lifecycle step it belongs to.
+### Running a Stage Locally
 
-### Running a stage locally
+Be aware that the tests will be built and run in-place, so please run at your own risk.
+If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.
+
+The ci needs to perform various sysadmin tasks such as installing packages or writing to the user's home directory.
+While most of the actions are done inside a docker container, this is not possible for all. Thus, cache directories,
+such as the depends cache, previous release binaries, or ccache, are mounted as read-write into the docker container. While it should be fine to run
+the ci system locally on you development box, the ci scripts can generally be assumed to have received less review and
+testing compared to other parts of the codebase. If you want to keep the work tree clean, you might want to run the ci
+system in a virtual machine with a Linux operating system of your choice.
 
 To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage
 requires `docker` to be installed. To install all requirements on Ubuntu, run
 
 ```
-sudo apt install docker.io ccache bash git
+sudo apt install docker.io bash
 ```
 
 To run the default test stage,
@@ -27,5 +33,33 @@ To run the test stage with a specific configuration,
 FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
 ```
 
-Be aware that the tests will be build and run in-place, so please run at your own risk.
-If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.
+### Configurations
+
+The test files (`FILE_ENV`) are constructed to test a wide range of
+configurations, rather than a single pass/fail. This helps to catch build
+failures and logic errors that present on platforms other than the ones the
+author has tested.
+
+Some builders use the dependency-generator in `./depends`, rather than using
+the system package manager to install build dependencies. This guarantees that
+the tester is using the same versions as the release builds, which also use
+`./depends`.
+
+If no `FILE_ENV` has been specified or values are left out, `00_setup_env.sh`
+is used as the default configuration with fallback values.
+
+It is also possible to force a specific configuration without modifying the
+file. For example,
+
+```
+MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
+```
+
+The files starting with `0n` (`n` greater than 0) are the scripts that are run
+in order.
+
+### Cache
+
+In order to avoid rebuilding all dependencies for each build, the binaries are
+cached and re-used when possible. Changes in the dependency-generator will
+trigger cache-invalidation and rebuilds as necessary.

ci/extended_lint/04_install.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C
-
-CPPCHECK_VERSION=1.86
-curl -s https://codeload.github.com/danmar/cppcheck/tar.gz/${CPPCHECK_VERSION} | tar -zxf - --directory /tmp/
-(cd /tmp/cppcheck-${CPPCHECK_VERSION}/ && make CFGDIR=/tmp/cppcheck-${CPPCHECK_VERSION}/cfg/ > /dev/null)
-export PATH="$PATH:/tmp/cppcheck-${CPPCHECK_VERSION}/"

ci/extended_lint/06_script.sh

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C
-
-test/lint/extended-lint-all.sh

ci/lint/04_install.sh

@@ -1,14 +1,20 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018 The Bitcoin Core developers
+# Copyright (c) 2018-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C
 
-travis_retry pip3 install codespell==1.15.0
-travis_retry pip3 install flake8==3.7.8
+travis_retry sudo apt update && sudo apt install -y clang-format-9
+sudo update-alternatives --install /usr/bin/clang-format      clang-format      $(which clang-format-9     ) 100
+sudo update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(which clang-format-diff-9) 100
 
-SHELLCHECK_VERSION=v0.6.0
-curl -s "https://storage.googleapis.com/shellcheck/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/
+travis_retry pip3 install codespell==1.17.1
+travis_retry pip3 install flake8==3.8.3
+travis_retry pip3 install yq
+travis_retry pip3 install mypy==0.781
+
+SHELLCHECK_VERSION=v0.7.1
+curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/
 export PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}"

ci/lint/05_before_script.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018 The Bitcoin Core developers
+# Copyright (c) 2018-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 

ci/lint/06_script.sh

@@ -1,19 +1,24 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018 The Bitcoin Core developers
+# Copyright (c) 2018-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C
 
 if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then
-  test/lint/commit-script-check.sh $TRAVIS_COMMIT_RANGE
+  # TRAVIS_BRANCH will be present in a Travis environment. For builds triggered
+  # by a pull request this is the name of the branch targeted by the pull request.
+  # https://docs.travis-ci.com/user/environment-variables/
+  COMMIT_RANGE="$TRAVIS_BRANCH..HEAD"
+  test/lint/commit-script-check.sh $COMMIT_RANGE
 fi
 
 test/lint/git-subtree-check.sh src/crypto/ctaes
 test/lint/git-subtree-check.sh src/secp256k1
 test/lint/git-subtree-check.sh src/univalue
 test/lint/git-subtree-check.sh src/leveldb
+test/lint/git-subtree-check.sh src/crc32c
 test/lint/check-doc.py
 test/lint/check-rpc-mappings.py .
 test/lint/lint-all.sh

ci/retry/README.md

@@ -32,8 +32,8 @@ Help:
         -v, --verbose                    Verbose output
         -t, --tries=#                    Set max retries: Default 10
         -s, --sleep=secs                 Constant sleep amount (seconds)
-        -m, --min=secs                   Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3
-        -x, --max=secs                   Exponenetial Backoff: maximum sleep amount (seconds): Default 60
+        -m, --min=secs                   Exponential Backoff: minimum sleep amount (seconds): Default 0.3
+        -x, --max=secs                   Exponential Backoff: maximum sleep amount (seconds): Default 60
         -f, --fail="script +cmds"        Fail Script: run in case of final failure
 
 ### Examples

ci/retry/retry

@@ -17,7 +17,7 @@ __log_out() {
   echo "$1" 1>&2
 }
 
-# Paramters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND
+# Parameters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND
 retry()
 {
   local max_tries="$1"; shift
@@ -83,8 +83,8 @@ Usage: $retry [options] -- execute command
     -v, --verbose                    Verbose output
     -t, --tries=#                    Set max retries: Default 10
     -s, --sleep=secs                 Constant sleep amount (seconds)
-    -m, --min=secs                   Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3
-    -x, --max=secs                   Exponenetial Backoff: maximum sleep amount (seconds): Default 60
+    -m, --min=secs                   Exponential Backoff: minimum sleep amount (seconds): Default 0.3
+    -x, --max=secs                   Exponential Backoff: maximum sleep amount (seconds): Default 60
     -f, --fail="script +cmds"        Fail Script: run in case of final failure
 EOF
   }

ci/test/00_setup_env.sh

@@ -1,11 +1,17 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019 The Bitcoin Core developers
+# Copyright (c) 2019-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
+# The root dir.
+# The ci system copies this folder.
+# This is where the depends build is done.
+BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
+export BASE_ROOT_DIR
+
 echo "Setting specific values in env"
 if [ -n "${FILE_ENV}" ]; then
   set -o errexit;
@@ -13,34 +19,54 @@ if [ -n "${FILE_ENV}" ]; then
   source "${FILE_ENV}"
 fi
 
-BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
-export BASE_ROOT_DIR
-
 echo "Fallback to default values in env (if not yet set)"
 # The number of parallel jobs to pass down to make and test_runner.py
 export MAKEJOBS=${MAKEJOBS:--j4}
 # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
-export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/}
-export HOST=${HOST:-x86_64-unknown-linux-gnu}
+# This folder only exists on the ci host.
+export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch}
+# What host to compile for. See also ./depends/README.md
+# Tests that need cross-compilation export the appropriate HOST.
+# Tests that run natively guess the host
+export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")}
+# Whether to prefer BusyBox over GNU utilities
+export USE_BUSY_BOX=${USE_BUSY_BOX:-false}
+
 export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
 export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true}
+export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false}
+# By how much to scale the test_runner timeouts (option --timeout-factor).
+# This is needed because some ci machines have slow CPU or disk, so sanitizers
+# might be slow or a reindex might be waiting on disk IO.
+export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-40}
+export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-}
 export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false}
+export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000}
+
+export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed}
 export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04}
 # Randomize test order.
 # See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/random.html
 export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1}
+# See man 7 debconf
+export DEBIAN_FRONTEND=noninteractive
 export CCACHE_SIZE=${CCACHE_SIZE:-100M}
 export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
 export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
+# The cache dir.
+# This folder exists on the ci host and ci guest. Changes are propagated back and forth.
 export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache}
-# Folder where the build is done (depends and dist). Can not be changed and is equal to the root of the git repo
-export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR}
-# Folder where the build is done (bin and lib). Can not be changed.
-export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST}
+# The depends dir.
+# This folder exists on the ci host and ci guest. Changes are propagated back and forth.
+export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends}
+# Folder where the build result is put (bin and lib).
+export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST}
+# Folder where the build is done (dist and out-of-tree build).
+export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build}
+export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST}
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
-export WINEDEBUG=${WINEDEBUG:-fixme-all}
-export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3}
+export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps}
 export GOAL=${GOAL:-install}
-export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets}
+export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets}
 export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH
-export CI_RETRY_EXE=${CI_RETRY_EXE:retry}
+export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"}

ci/test/00_setup_env_amd64_asan.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-export HOST=x86_64-unknown-linux-gnu
-export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev"
-export NO_DEPENDS=1
-export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"

ci/test/00_setup_env_amd64_nowallet.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-export HOST=x86_64-unknown-linux-gnu
-export PACKAGES="python3-zmq"
-export DEP_OPTS="NO_WALLET=1"
-export GOAL="install"
-export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"

ci/test/00_setup_env_amd64_qt5.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-export HOST=x86_64-unknown-linux-gnu
-export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev"
-export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1"
-export TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_dbcrash"  # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash
-export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\""

ci/test/00_setup_env_amd64_trusty.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-export HOST=x86_64-unknown-linux-gnu
-export DOCKER_NAME_TAG=ubuntu:14.04
-export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libicu-dev libpng-dev libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libzmq3-dev libqrencode-dev"
-export NO_DEPENDS=1
-export RUN_FUNCTIONAL_TESTS=false
-export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no"

ci/test/00_setup_env_amd64_tsan.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-export HOST=x86_64-unknown-linux-gnu
-export DOCKER_NAME_TAG=ubuntu:16.04
-export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev"
-export NO_DEPENDS=1
-export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++"

ci/test/00_setup_env_arm.sh

@@ -1,16 +1,28 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019 The Bitcoin Core developers
+# Copyright (c) 2019-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
 export HOST=arm-linux-gnueabihf
-export PACKAGES="python3 g++-arm-linux-gnueabihf"
-export RUN_UNIT_TESTS=false
+# The host arch is unknown, so we run the tests through qemu.
+# If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string.
+if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-arm -L /usr/arm-linux-gnueabihf/"}"; fi
+export DPKG_ADD_ARCH="armhf"
+export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf"
+if [ -n "$QEMU_USER_CMD" ]; then
+  # Likely cross-compiling, so install the needed gcc and qemu-user
+  export PACKAGES="$PACKAGES qemu-user"
+fi
+export CONTAINER_NAME=ci_arm_linux
+# Use debian to avoid 404 apt errors when cross compiling
+export DOCKER_NAME_TAG="debian:buster"
+export USE_BUSY_BOX=true
+export RUN_UNIT_TESTS=true
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="install"
 # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1"
 # This could be removed once the ABI change warning does not show up by default
-export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi"
+export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi --with-boost-process"

ci/test/00_setup_env_i686.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2019 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-export HOST=i686-pc-linux-gnu
-export DEP_OPTS="PROTOBUF=1"
-export PACKAGES="g++-multilib python3-zmq"
-export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-bip70 --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++"
-export CONFIG_SHELL="/bin/dash"

ci/test/00_setup_env_i686_centos.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=i686-pc-linux-gnu
+export CONTAINER_NAME=ci_i686_centos_7
+export DOCKER_NAME_TAG=centos:7
+export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python36-zmq which patch lbzip2 dash"
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports --with-boost-process"
+export CONFIG_SHELL="/bin/dash"

ci/test/00_setup_env_mac.sh

@@ -1,15 +1,18 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019 The Bitcoin Core developers
+# Copyright (c) 2019-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-export HOST=x86_64-apple-darwin14
+export CONTAINER_NAME=ci_macos_cross
+export DOCKER_NAME_TAG=ubuntu:18.04  # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well)
+export HOST=x86_64-apple-darwin16
 export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools"
-export OSX_SDK=10.11
+export XCODE_VERSION=11.3.1
+export XCODE_BUILD_ID=11C505
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="deploy"
-export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror"
+export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --with-boost-process"

ci/test/00_setup_env_mac_host.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=x86_64-apple-darwin16
+export PIP_PACKAGES="zmq"
+export GOAL="install"
+export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --with-boost-process"
+export CI_OS_NAME="macos"
+export NO_DEPENDS=1
+export OSX_SDK=""
+export CCACHE_SIZE=300M
+
+export RUN_SECURITY_TESTS="true"

ci/test/00_setup_env_native_asan.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_native_asan
+export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev libsqlite3-dev"
+export DOCKER_NAME_TAG=ubuntu:20.04
+export NO_DEPENDS=1
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++ --with-boost-process"

ci/test/00_setup_env_native_fuzz.sh

@@ -1,16 +1,18 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019 The Bitcoin Core developers
+# Copyright (c) 2019-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-export HOST=x86_64-unknown-linux-gnu
-export PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev"
+export DOCKER_NAME_TAG="ubuntu:20.04"
+export CONTAINER_NAME=ci_native_fuzz
+export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev"
 export NO_DEPENDS=1
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export RUN_FUZZ_TESTS=true
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++"
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++ --with-boost-process"
+export CCACHE_SIZE=200M

ci/test/00_setup_env_native_fuzz_with_valgrind.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export DOCKER_NAME_TAG="ubuntu:20.04"
+export CONTAINER_NAME=ci_native_fuzz_valgrind
+export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev valgrind"
+export NO_DEPENDS=1
+export RUN_UNIT_TESTS=false
+export RUN_FUNCTIONAL_TESTS=false
+export RUN_FUZZ_TESTS=true
+export FUZZ_TESTS_CONFIG="--valgrind"
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++"
+export CCACHE_SIZE=200M

ci/test/00_setup_env_native_msan.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export DOCKER_NAME_TAG="ubuntu:20.04"
+LIBCXX_DIR="${BASE_ROOT_DIR}/ci/scratch/msan/build/"
+export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
+LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument"
+export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
+export BDB_PREFIX="${BASE_ROOT_DIR}/db4"
+
+export CONTAINER_NAME="ci_native_msan"
+export PACKAGES="clang-9 llvm-9 cmake"
+export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++11 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++11 ${MSAN_AND_LIBCXX_FLAGS}'"
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${BASE_ROOT_DIR}/depends/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'"
+export USE_MEMORY_SANITIZER="true"
+export RUN_FUNCTIONAL_TESTS="false"
+export CCACHE_SIZE=250M

ci/test/00_setup_env_native_multiprocess.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_native_multiprocess
+export DOCKER_NAME_TAG=ubuntu:20.04
+export PACKAGES="cmake python3"
+export DEP_OPTS="MULTIPROCESS=1"
+export GOAL="install"
+export BITCOIN_CONFIG="--with-boost-process"
+export TEST_RUNNER_ENV="BITCOIND=bitcoin-node"

ci/test/00_setup_env_native_nowallet.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_native_nowallet
+export DOCKER_NAME_TAG=ubuntu:16.04  # Use xenial to have one config run the tests in python3.5, see doc/dependencies.md
+export PACKAGES="python3-zmq clang-3.8 llvm-3.8"  # Use clang-3.8 to test C++11 compatibility, see doc/dependencies.md
+export DEP_OPTS="NO_WALLET=1"
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CC=clang-3.8 CXX=clang++-3.8 --with-boost-process"

ci/test/00_setup_env_native_qt5.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_native_qt5
+export DOCKER_NAME_TAG=ubuntu:18.04  # Check that bionic can compile our c++17 and run our functional tests in python3
+export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev"
+export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1"
+export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash"  # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash
+export RUN_SECURITY_TESTS="true"
+export RUN_UNIT_TESTS_SEQUENTIAL="true"
+export RUN_UNIT_TESTS="false"
+export GOAL="install"
+export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1"
+export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-c++17 --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process"

ci/test/00_setup_env_native_tsan.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_native_tsan
+export DOCKER_NAME_TAG=ubuntu:20.04
+export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq"
+export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'"
+export TEST_RUNNER_EXTRA="--exclude feature_block"  # Low memory on Travis machines, exclude feature_block.
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-gui=no CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++' --with-boost-process"

ci/test/00_setup_env_native_valgrind.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_native_valgrind
+export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libsqlite3-dev"
+export USE_VALGRIND=1
+export NO_DEPENDS=1
+export TEST_RUNNER_EXTRA="--exclude rpc_bind"  # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++"  # TODO enable GUI

ci/test/00_setup_env_s390x.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=s390x-linux-gnu
+# The host arch is unknown, so we run the tests through qemu.
+# If the host is s390x and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string.
+if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-s390x"}"; fi
+export PACKAGES="python3-zmq"
+if [ -n "$QEMU_USER_CMD" ]; then
+  # Likely cross-compiling, so install the needed gcc and qemu-user
+  export DPKG_ADD_ARCH="s390x"
+  export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x libfontconfig1:s390x libxcb1:s390x"
+fi
+# Use debian to avoid 404 apt errors
+export CONTAINER_NAME=ci_s390x
+export DOCKER_NAME_TAG="debian:buster"
+export RUN_UNIT_TESTS=true
+export TEST_RUNNER_ENV="LC_ALL=C"
+export RUN_FUNCTIONAL_TESTS=true
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-reduce-exports --with-incompatible-bdb --with-boost-process"

ci/test/00_setup_env_win64.sh

@@ -1,13 +1,20 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019 The Bitcoin Core developers
+# Copyright (c) 2019-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
+export CONTAINER_NAME=ci_win64
+export DOCKER_NAME_TAG=ubuntu:18.04  # Check that bionic can cross-compile to win64 (bionic is used in the gitian build as well)
 export HOST=x86_64-w64-mingw32
-export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64"
+export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 file"
 export RUN_FUNCTIONAL_TESTS=false
+export RUN_SECURITY_TESTS="true"
 export GOAL="deploy"
-export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests"
+export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests --without-boost-process"
+
+# Compiler for MinGW-w64 causes false -Wreturn-type warning.
+# See https://sourceforge.net/p/mingw-w64/bugs/306/
+export NO_WERROR=1

ci/test/03_before_install.sh

@@ -1,15 +1,11 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018 The Bitcoin Core developers
+# Copyright (c) 2018-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-# Add llvm-symbolizer directory to PATH. Needed to get symbolized stack traces from the sanitizers.
-PATH=$PATH:/usr/lib/llvm-6.0/bin/
-export PATH
-
 BEGIN_FOLD () {
   echo ""
   CURRENT_FOLD_NAME=$1

ci/test/04_install.sh

@@ -1,50 +1,113 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018 The Bitcoin Core developers
+# Copyright (c) 2018-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-mkdir -p "${BASE_SCRATCH_DIR}"
-ccache echo "Creating ccache dir if it didn't already exist"
-
-if [ ! -d ${DIR_QA_ASSETS} ]; then
-  git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
+if [[ $DOCKER_NAME_TAG == centos* ]]; then
+  export LC_ALL=en_US.utf8
+fi
+if [[ $QEMU_USER_CMD == qemu-s390* ]]; then
+  export LC_ALL=C
 fi
-export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
 
-mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/"
-export ASAN_OPTIONS=""
-export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan"
-export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan"
-export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1"
-env | grep -E '^(BITCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
-if [[ $HOST = *-mingw32 ]]; then
-  DOCKER_ADMIN="--cap-add SYS_ADMIN"
-elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
+if [ "$CI_OS_NAME" == "macos" ]; then
+  IN_GETOPT_BIN="/usr/local/opt/gnu-getopt/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
+fi
+
+# Create folders that are mounted into the docker
+mkdir -p "${CCACHE_DIR}"
+mkdir -p "${PREVIOUS_RELEASES_DIR}"
+
+export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1"
+export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan"
+export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan"
+export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
+env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env
+if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
   DOCKER_ADMIN="--cap-add SYS_PTRACE"
 fi
 
-if [ -z "$RUN_CI_ON_HOST" ]; then
+export P_CI_DIR="$PWD"
+
+if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
   echo "Creating $DOCKER_NAME_TAG container to run in"
   ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG"
 
-  DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
-
-  DOCKER_EXEC () {
-    docker exec $DOCKER_ID bash -c "cd $PWD && $*"
-  }
+  DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \
+                  --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \
+                  --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \
+                  --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \
+                  --mount type=bind,src=$PREVIOUS_RELEASES_DIR,dst=$PREVIOUS_RELEASES_DIR \
+                  -w $BASE_ROOT_DIR \
+                  --env-file /tmp/env \
+                  --name $CONTAINER_NAME \
+                  $DOCKER_NAME_TAG)
+  export DOCKER_CI_CMD_PREFIX="docker exec $DOCKER_ID"
 else
   echo "Running on host system without docker wrapper"
-  DOCKER_EXEC () {
-    bash -c "cd $PWD && $*"
-  }
 fi
 
-DOCKER_EXEC free -m -h
-DOCKER_EXEC echo "Number of CPUs \(nproc\): $(nproc)"
+DOCKER_EXEC () {
+  $DOCKER_CI_CMD_PREFIX bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*"
+}
+export -f DOCKER_EXEC
 
-${CI_RETRY_EXE} DOCKER_EXEC apt-get update
-${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES
+if [ -n "$DPKG_ADD_ARCH" ]; then
+  DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH"
+fi
 
+if [[ $DOCKER_NAME_TAG == centos* ]]; then
+  ${CI_RETRY_EXE} DOCKER_EXEC yum -y install epel-release
+  ${CI_RETRY_EXE} DOCKER_EXEC yum -y install $DOCKER_PACKAGES $PACKAGES
+elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
+  ${CI_RETRY_EXE} DOCKER_EXEC apt-get update
+  ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES
+fi
+
+if [ "$CI_OS_NAME" == "macos" ]; then
+  top -l 1 -s 0 | awk ' /PhysMem/ {print}'
+  echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
+else
+  DOCKER_EXEC free -m -h
+  DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\)
+  DOCKER_EXEC echo $(lscpu | grep Endian)
+fi
+DOCKER_EXEC echo "Free disk space:"
+DOCKER_EXEC df -h
+
+if [ ! -d ${DIR_QA_ASSETS} ]; then
+  DOCKER_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
+fi
+export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
+export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/
+
+DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/"
+
+if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
+  DOCKER_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-9) 100"
+  DOCKER_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-9) 100"
+  DOCKER_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/"
+  DOCKER_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-10.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project"
+  DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/"
+  DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx"
+fi
+
+if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
+  echo "Create $BASE_ROOT_DIR"
+  DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR
+fi
+
+if [ "$USE_BUSY_BOX" = "true" ]; then
+  echo "Setup to use BusyBox utils"
+  DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/
+  # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version)
+  # find excluded for now because it does not recognize the -delete option in ./depends (fixed in later BusyBox version)
+  # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed)
+  # shellcheck disable=SC1010
+  DOCKER_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \| grep -v "^find$"\)\; do ln -s \$\(command -v busybox\) $BASE_SCRATCH_DIR/bins/\$util\; done
+  # Print BusyBox version
+  DOCKER_EXEC patch --help
+fi

ci/test/05_before_script.sh

@@ -1,24 +1,53 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018 The Bitcoin Core developers
+# Copyright (c) 2018-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-DOCKER_EXEC echo \> \$HOME/.bitcoin  # Make sure default datadir does not exist and is never read by creating a dummy file
-
-mkdir -p depends/SDKs depends/sdk-sources
-
-if [ -n "$OSX_SDK" ] && [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
-  curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+# Make sure default datadir does not exist and is never read by creating a dummy file
+if [ "$CI_OS_NAME" == "macos" ]; then
+  echo > $HOME/Library/Application\ Support/Bitcoin
+else
+  DOCKER_EXEC echo \> \$HOME/.bitcoin
 fi
-if [ -n "$OSX_SDK" ] && [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
-  tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+
+DOCKER_EXEC mkdir -p ${DEPENDS_DIR}/SDKs ${DEPENDS_DIR}/sdk-sources
+
+OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
+OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}"
+
+if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then
+  curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
+fi
+
+if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
+  # Use BDB compiled using install_db4.sh script to work around linking issue when using BDB
+  # from depends. See https://github.com/bitcoin/bitcoin/pull/18288#discussion_r433189350 for
+  # details.
+  DOCKER_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+fi
+
+if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then
+  DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH"
 fi
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\)
 fi
 if [ -z "$NO_DEPENDS" ]; then
-  DOCKER_EXEC CONFIG_SHELL= make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
+  if [[ $DOCKER_NAME_TAG == centos* ]]; then
+    # CentOS has problems building the depends if the config shell is not explicitly set
+    # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to
+    #  an error as the first command is executed)
+    SHELL_OPTS="CONFIG_SHELL=/bin/bash"
+  else
+    SHELL_OPTS="CONFIG_SHELL="
+  fi
+  DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
+fi
+if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then
+  BEGIN_FOLD previous-versions
+  DOCKER_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}"
+  END_FOLD
 fi

ci/test/06_script_a.sh

@@ -1,15 +1,16 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018 The Bitcoin Core developers
+# Copyright (c) 2018-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$BASE_BUILD_DIR/depends/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib"
-if [ -z "$NO_DEPENDS" ]; then
-  DOCKER_EXEC ccache --max-size=$CCACHE_SIZE
+BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib"
+if [ -z "$NO_WERROR" ]; then
+  BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror"
 fi
+DOCKER_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE"
 
 BEGIN_FOLD autogen
 if [ -n "$CONFIG_SHELL" ]; then
@@ -19,28 +20,40 @@ else
 fi
 END_FOLD
 
-mkdir -p build
-cd build || (echo "could not enter build directory"; exit 1)
+DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}"
+export P_CI_DIR="${BASE_BUILD_DIR}"
 
 BEGIN_FOLD configure
-DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
+DOCKER_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false)
 END_FOLD
 
 BEGIN_FOLD distdir
 DOCKER_EXEC make distdir VERSION=$HOST
 END_FOLD
 
-cd "bitcoin-$HOST" || (echo "could not enter distdir bitcoin-$HOST"; exit 1)
+export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST"
 
 BEGIN_FOLD configure
-DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
+DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false)
 END_FOLD
 
 set -o errtrace
-trap 'DOCKER_EXEC "cat ${BASE_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR
+trap 'DOCKER_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR
+
+if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
+  # MemorySanitizer (MSAN) does not support tracking memory initialization done by
+  # using the Linux getrandom syscall. Avoid using getrandom by undefining
+  # HAVE_SYS_GETRANDOM. See https://github.com/google/sanitizers/issues/852 for
+  # details.
+  DOCKER_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h'
+fi
 
 BEGIN_FOLD build
 DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false )
 END_FOLD
 
-cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1)
+BEGIN_FOLD cache_stats
+DOCKER_EXEC "ccache --version | head -n 1 && ccache --show-stats"
+DOCKER_EXEC du -sh "${DEPENDS_DIR}"/*/
+DOCKER_EXEC du -sh "${PREVIOUS_RELEASES_DIR}"
+END_FOLD

ci/test/06_script_b.sh

@@ -1,29 +1,61 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018 The Bitcoin Core developers
+# Copyright (c) 2018-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
 
-cd "build/bitcoin-$HOST" || (echo "could not enter distdir build/bitcoin-$HOST"; exit 1)
+if [[ $HOST = *-mingw32 ]]; then
+  BEGIN_FOLD wrap-wine
+  # Generate all binaries, so that they can be wrapped
+  DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1
+  DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1
+  DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh"
+  END_FOLD
+fi
+
+if [ -n "$QEMU_USER_CMD" ]; then
+  BEGIN_FOLD wrap-qemu
+  # Generate all binaries, so that they can be wrapped
+  DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1
+  DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1
+  DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh"
+  END_FOLD
+fi
+
+if [ -n "$USE_VALGRIND" ]; then
+  BEGIN_FOLD wrap-valgrind
+  DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh"
+  END_FOLD
+fi
 
 if [ "$RUN_UNIT_TESTS" = "true" ]; then
   BEGIN_FOLD unit-tests
-  DOCKER_EXEC LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1
+  DOCKER_EXEC ${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib make $MAKEJOBS check VERBOSE=1
+  END_FOLD
+fi
+
+if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then
+  BEGIN_FOLD unit-tests-seq
+  DOCKER_EXEC ${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib "${BASE_BUILD_DIR}/bitcoin-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite
   END_FOLD
 fi
 
 if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
   BEGIN_FOLD functional-tests
-  DOCKER_EXEC test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast
+  DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --quiet --failfast
+  END_FOLD
+fi
+
+if [ "$RUN_SECURITY_TESTS" = "true" ]; then
+  BEGIN_FOLD security-tests
+  DOCKER_EXEC make test-security-check
   END_FOLD
 fi
 
 if [ "$RUN_FUZZ_TESTS" = "true" ]; then
   BEGIN_FOLD fuzz-tests
-  DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN}
+  DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN}
   END_FOLD
 fi
-
-cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1)

ci/test/wrap-qemu.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}; do
+    # shellcheck disable=SC2044
+    for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do
+      echo "Wrap $b ..."
+      mv "$b" "${b}_orig"
+      echo '#!/usr/bin/env bash' > "$b"
+      echo "$QEMU_USER_CMD \"${b}_orig\" \"\$@\"" >> "$b"
+      chmod +x "$b"
+    done
+done

ci/test/wrap-valgrind.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+for b_name in "${BASE_OUTDIR}/bin"/*; do
+    # shellcheck disable=SC2044
+    for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do
+      echo "Wrap $b ..."
+      mv "$b" "${b}_orig"
+      echo '#!/usr/bin/env bash' > "$b"
+      echo "valgrind --gen-suppressions=all --quiet --error-exitcode=1 --suppressions=${BASE_ROOT_DIR}/contrib/valgrind.supp \"${b}_orig\" \"\$@\"" >> "$b"
+      chmod +x "$b"
+    done
+done

ci/test/wrap-wine.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}.exe; do
+    # shellcheck disable=SC2044
+    for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename $b_name)"); do
+      if (file "$b" | grep "Windows"); then
+        echo "Wrap $b ..."
+        mv "$b" "${b}_orig"
+        echo '#!/usr/bin/env bash' > "$b"
+        echo "wine64 \"${b}_orig\" \"\$@\"" >> "$b"
+        chmod +x "$b"
+      fi
+    done
+done

contrib/bitcoin-cli.bash-completion

@@ -1,5 +1,5 @@
 # bash programmable completion for bitcoin-cli(1)
-# Copyright (c) 2012-2016 The Bitcoin Core developers
+# Copyright (c) 2012-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -17,13 +17,6 @@ _bitcoin_rpc() {
     $bitcoin_cli "${rpcargs[@]}" "$@"
 }
 
-# Add wallet accounts to COMPREPLY
-_bitcoin_accounts() {
-    local accounts
-    accounts=$(_bitcoin_rpc listaccounts | awk -F '"' '{ print $2 }')
-    COMPREPLY=( "${COMPREPLY[@]}" $( compgen -W "$accounts" -- "$cur" ) )
-}
-
 _bitcoin_cli() {
     local cur prev words=() cword
     local bitcoin_cli
@@ -60,10 +53,9 @@ _bitcoin_cli() {
     if ((cword > 3)); then
         case ${words[cword-3]} in
             addmultisigaddress)
-                _bitcoin_accounts
                 return 0
                 ;;
-            getbalance|gettxout|importaddress|importpubkey|importprivkey|listreceivedbyaccount|listreceivedbyaddress|listsinceblock)
+            getbalance|gettxout|importaddress|importpubkey|importprivkey|listreceivedbyaddress|listsinceblock)
                 COMPREPLY=( $( compgen -W "true false" -- "$cur" ) )
                 return 0
                 ;;
@@ -80,14 +72,10 @@ _bitcoin_cli() {
                 COMPREPLY=( $( compgen -W "add remove" -- "$cur" ) )
                 return 0
                 ;;
-            fundrawtransaction|getblock|getblockheader|getmempoolancestors|getmempooldescendants|getrawtransaction|gettransaction|listaccounts|listreceivedbyaccount|listreceivedbyaddress|sendrawtransaction)
+            fundrawtransaction|getblock|getblockheader|getmempoolancestors|getmempooldescendants|getrawtransaction|gettransaction|listreceivedbyaddress|sendrawtransaction)
                 COMPREPLY=( $( compgen -W "true false" -- "$cur" ) )
                 return 0
                 ;;
-            move|setaccount)
-                _bitcoin_accounts
-                return 0
-                ;;
         esac
     fi
 
@@ -96,12 +84,11 @@ _bitcoin_cli() {
             _filedir
             return 0
             ;;
-        getaddednodeinfo|getrawmempool|lockunspent|setgenerate)
+        getaddednodeinfo|getrawmempool|lockunspent)
             COMPREPLY=( $( compgen -W "true false" -- "$cur" ) )
             return 0
             ;;
-        getaccountaddress|getaddressesbyaccount|getbalance|getnewaddress|getreceivedbyaccount|listtransactions|move|sendfrom|sendmany)
-            _bitcoin_accounts
+        getbalance|getnewaddress|listtransactions|sendmany)
             return 0
             ;;
     esac

contrib/bitcoind.bash-completion

@@ -1,5 +1,5 @@
 # bash programmable completion for bitcoind(1) and bitcoin-qt(1)
-# Copyright (c) 2012-2016 The Bitcoin Core developers
+# Copyright (c) 2012-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -15,7 +15,7 @@ _bitcoind() {
     _get_comp_words_by_ref -n = cur prev words cword
 
     case "$cur" in
-        -conf=*|-pid=*|-loadblock=*|-rootcertificates=*|-rpccookiefile=*|-wallet=*)
+        -conf=*|-pid=*|-loadblock=*|-rpccookiefile=*|-wallet=*)
             cur="${cur#*=}"
             _filedir
             return 0

contrib/debian/copyright

@@ -5,7 +5,7 @@ Upstream-Contact: Satoshi Nakamoto <satoshin@gmx.com>
 Source: https://github.com/bitcoin/bitcoin
 
 Files: *
-Copyright: 2009-2019, Bitcoin Core Developers
+Copyright: 2009-2020, Bitcoin Core Developers
 License: Expat
 Comment: The Bitcoin Core Developers encompasses the current developers listed on bitcoin.org,
          as well as the numerous contributors to the project.

contrib/devtools/README.md

@@ -89,31 +89,29 @@ example:
 BUILDDIR=$PWD/build contrib/devtools/gen-manpages.sh
 ```
 
-optimize-pngs.py
-================
-
-A script to optimize png files in the bitcoin
-repository (requires pngcrush).
-
 security-check.py and test-security-check.py
 ============================================
 
-Perform basic ELF security checks on a series of executables.
+Perform basic security checks on a series of executables.
 
 symbol-check.py
 ===============
 
-A script to check that the (Linux) executables produced by gitian only contain
-allowed gcc, glibc and libstdc++ version symbols. This makes sure they are
-still compatible with the minimum supported Linux distribution versions.
+A script to check that the executables produced by gitian only contain
+certain symbols and are only linked against allowed libraries.
+
+For Linux this means checking for allowed gcc, glibc and libstdc++ version symbols.
+This makes sure they are still compatible with the minimum supported distribution versions.
+
+For macOS and Windows we check that the executables are only linked against libraries we allow.
 
 Example usage after a gitian build:
 
     find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py
 
-If only supported symbols are used the return value will be 0 and the output will be empty.
+If no errors occur the return value will be 0 and the output will be empty.
 
-If there are 'unsupported' symbols, the return value will be 1 a list like this will be printed:
+If there are any errors the return value will be 1 and output like this will be printed:
 
     .../64/test_bitcoin: symbol memcpy from unsupported version GLIBC_2.14
     .../64/test_bitcoin: symbol __fdelt_chk from unsupported version GLIBC_2.15

contrib/devtools/circular-dependencies.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+# Copyright (c) 2018-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 import sys
 import re

contrib/devtools/copyright_header.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2016-2018 The Bitcoin Core developers
+# Copyright (c) 2016-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -19,7 +19,10 @@ EXCLUDE = [
     'src/qt/bitcoinstrings.cpp',
     'src/chainparamsseeds.h',
     # other external copyrights:
+    'src/reverse_iterator.h',
+    'src/test/fuzz/FuzzedDataProvider.h',
     'src/tinyformat.h',
+    'src/bench/nanobench.h',
     'test/functional/test_framework/bignum.py',
     # python init:
     '*__init__.py',
@@ -32,9 +35,10 @@ EXCLUDE_DIRS = [
     "src/leveldb/",
     "src/secp256k1/",
     "src/univalue/",
+    "src/crc32c/",
 ]
 
-INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.mm', '*.py']
+INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.mm', '*.py', '*.sh', '*.bash-completion']
 INCLUDE_COMPILED = re.compile('|'.join([fnmatch.translate(m) for m in INCLUDE]))
 
 def applies_to_file(filename):
@@ -455,14 +459,14 @@ CPP_HEADER = '''
 def get_cpp_header_lines_to_insert(start_year, end_year):
     return reversed(get_header_lines(CPP_HEADER, start_year, end_year))
 
-PYTHON_HEADER = '''
+SCRIPT_HEADER = '''
 # Copyright (c) %s The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 '''
 
-def get_python_header_lines_to_insert(start_year, end_year):
-    return reversed(get_header_lines(PYTHON_HEADER, start_year, end_year))
+def get_script_header_lines_to_insert(start_year, end_year):
+    return reversed(get_header_lines(SCRIPT_HEADER, start_year, end_year))
 
 ################################################################################
 # query git for year of last change
@@ -491,17 +495,18 @@ def file_has_hashbang(file_lines):
         return False
     return file_lines[0][:2] == '#!'
 
-def insert_python_header(filename, file_lines, start_year, end_year):
+def insert_script_header(filename, file_lines, start_year, end_year):
     if file_has_hashbang(file_lines):
         insert_idx = 1
     else:
         insert_idx = 0
-    header_lines = get_python_header_lines_to_insert(start_year, end_year)
+    header_lines = get_script_header_lines_to_insert(start_year, end_year)
     for line in header_lines:
         file_lines.insert(insert_idx, line)
     write_file_lines(filename, file_lines)
 
 def insert_cpp_header(filename, file_lines, start_year, end_year):
+    file_lines.insert(0, '\n')
     header_lines = get_cpp_header_lines_to_insert(start_year, end_year)
     for line in header_lines:
         file_lines.insert(0, line)
@@ -513,8 +518,8 @@ def exec_insert_header(filename, style):
         sys.exit('*** %s already has a copyright by The Bitcoin Core developers'
                  % (filename))
     start_year, end_year = get_git_change_year_range(filename)
-    if style == 'python':
-        insert_python_header(filename, file_lines, start_year, end_year)
+    if style in ['python', 'shell']:
+        insert_script_header(filename, file_lines, start_year, end_year)
     else:
         insert_cpp_header(filename, file_lines, start_year, end_year)
 
@@ -555,11 +560,13 @@ def insert_cmd(argv):
     if not os.path.isfile(filename):
         sys.exit("*** bad filename: %s" % filename)
     _, extension = os.path.splitext(filename)
-    if extension not in ['.h', '.cpp', '.cc', '.c', '.py']:
+    if extension not in ['.h', '.cpp', '.cc', '.c', '.py', '.sh']:
         sys.exit("*** cannot insert for file extension %s" % extension)
 
     if extension == '.py':
         style = 'python'
+    elif extension == '.sh':
+        style = 'shell'
     else:
         style = 'cpp'
     exec_insert_header(filename, style)

contrib/devtools/gen-manpages.sh

@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
+# Copyright (c) 2016-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C
 TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)}

contrib/devtools/optimize-pngs.py

@@ -1,76 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2014-2018 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-'''
-Run this script every time you change one of the png files. Using pngcrush, it will optimize the png files, remove various color profiles, remove ancillary chunks (alla) and text chunks (text).
-#pngcrush -brute -ow -rem gAMA -rem cHRM -rem iCCP -rem sRGB -rem alla -rem text
-'''
-import os
-import sys
-import subprocess
-import hashlib
-from PIL import Image  # pip3 install Pillow
-
-def file_hash(filename):
-    '''Return hash of raw file contents'''
-    with open(filename, 'rb') as f:
-        return hashlib.sha256(f.read()).hexdigest()
-
-def content_hash(filename):
-    '''Return hash of RGBA contents of image'''
-    i = Image.open(filename)
-    i = i.convert('RGBA')
-    data = i.tobytes()
-    return hashlib.sha256(data).hexdigest()
-
-pngcrush = 'pngcrush'
-git = 'git'
-folders = ["src/qt/res/movies", "src/qt/res/icons", "share/pixmaps"]
-basePath = subprocess.check_output([git, 'rev-parse', '--show-toplevel'], universal_newlines=True, encoding='utf8').rstrip('\n')
-totalSaveBytes = 0
-noHashChange = True
-
-outputArray = []
-for folder in folders:
-    absFolder=os.path.join(basePath, folder)
-    for file in os.listdir(absFolder):
-        extension = os.path.splitext(file)[1]
-        if extension.lower() == '.png':
-            print("optimizing {}...".format(file), end =' ')
-            file_path = os.path.join(absFolder, file)
-            fileMetaMap = {'file' : file, 'osize': os.path.getsize(file_path), 'sha256Old' : file_hash(file_path)}
-            fileMetaMap['contentHashPre'] = content_hash(file_path)
-
-            try:
-                subprocess.call([pngcrush, "-brute", "-ow", "-rem", "gAMA", "-rem", "cHRM", "-rem", "iCCP", "-rem", "sRGB", "-rem", "alla", "-rem", "text", file_path],
-                                stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-            except:
-                print("pngcrush is not installed, aborting...")
-                sys.exit(0)
-
-            #verify
-            if "Not a PNG file" in subprocess.check_output([pngcrush, "-n", "-v", file_path], stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf8'):
-                print("PNG file "+file+" is corrupted after crushing, check out pngcursh version")
-                sys.exit(1)
-
-            fileMetaMap['sha256New'] = file_hash(file_path)
-            fileMetaMap['contentHashPost'] = content_hash(file_path)
-
-            if fileMetaMap['contentHashPre'] != fileMetaMap['contentHashPost']:
-                print("Image contents of PNG file {} before and after crushing don't match".format(file))
-                sys.exit(1)
-
-            fileMetaMap['psize'] = os.path.getsize(file_path)
-            outputArray.append(fileMetaMap)
-            print("done")
-
-print("summary:\n+++++++++++++++++")
-for fileDict in outputArray:
-    oldHash = fileDict['sha256Old']
-    newHash = fileDict['sha256New']
-    totalSaveBytes += fileDict['osize'] - fileDict['psize']
-    noHashChange = noHashChange and (oldHash == newHash)
-    print(fileDict['file']+"\n  size diff from: "+str(fileDict['osize'])+" to: "+str(fileDict['psize'])+"\n  old sha256: "+oldHash+"\n  new sha256: "+newHash+"\n")
-
-print("completed. Checksum stable: "+str(noHashChange)+". Total reduction: "+str(totalSaveBytes)+" bytes")

contrib/devtools/security-check.py

@@ -1,89 +1,114 @@
 #!/usr/bin/env python3
-# Copyright (c) 2015-2018 The Bitcoin Core developers
+# Copyright (c) 2015-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 '''
-Perform basic ELF security checks on a series of executables.
+Perform basic security checks on a series of executables.
 Exit status will be 0 if successful, and the program will be silent.
 Otherwise the exit status will be 1 and it will log which executables failed which checks.
-Needs `readelf` (for ELF) and `objdump` (for PE).
+Needs `readelf` (for ELF), `objdump` (for PE) and `otool` (for MACHO).
 '''
 import subprocess
 import sys
 import os
 
+from typing import List, Optional
+
 READELF_CMD = os.getenv('READELF', '/usr/bin/readelf')
 OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump')
-NONFATAL = {} # checks which are non-fatal for now but only generate a warning
+OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool')
 
-def check_ELF_PIE(executable):
+def run_command(command) -> str:
+    p = subprocess.run(command, stdout=subprocess.PIPE, check=True, universal_newlines=True)
+    return p.stdout
+
+def check_ELF_PIE(executable) -> bool:
     '''
     Check for position independent executable (PIE), allowing for address space randomization.
     '''
-    p = subprocess.Popen([READELF_CMD, '-h', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
-    (stdout, stderr) = p.communicate()
-    if p.returncode:
-        raise IOError('Error opening file')
+    stdout = run_command([READELF_CMD, '-h', '-W', executable])
 
     ok = False
     for line in stdout.splitlines():
-        line = line.split()
-        if len(line)>=2 and line[0] == 'Type:' and line[1] == 'DYN':
+        tokens = line.split()
+        if len(line)>=2 and tokens[0] == 'Type:' and tokens[1] == 'DYN':
             ok = True
     return ok
 
 def get_ELF_program_headers(executable):
     '''Return type and flags for ELF program headers'''
-    p = subprocess.Popen([READELF_CMD, '-l', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
-    (stdout, stderr) = p.communicate()
-    if p.returncode:
-        raise IOError('Error opening file')
+    stdout = run_command([READELF_CMD, '-l', '-W', executable])
+
     in_headers = False
-    count = 0
     headers = []
     for line in stdout.splitlines():
         if line.startswith('Program Headers:'):
             in_headers = True
+            count = 0
         if line == '':
             in_headers = False
         if in_headers:
             if count == 1: # header line
-                ofs_typ = line.find('Type')
-                ofs_offset = line.find('Offset')
-                ofs_flags = line.find('Flg')
-                ofs_align = line.find('Align')
-                if ofs_typ == -1 or ofs_offset == -1 or ofs_flags == -1 or ofs_align  == -1:
+                header = [x.strip() for x in line.split()]
+                ofs_typ = header.index('Type')
+                ofs_flags = header.index('Flg')
+                # assert readelf output is what we expect
+                if ofs_typ == -1 or ofs_flags == -1:
                     raise ValueError('Cannot parse elfread -lW output')
             elif count > 1:
-                typ = line[ofs_typ:ofs_offset].rstrip()
-                flags = line[ofs_flags:ofs_align].rstrip()
-                headers.append((typ, flags))
+                splitline = [x.strip() for x in line.split()]
+                typ = splitline[ofs_typ]
+                if not typ.startswith('[R'): # skip [Requesting ...]
+                    splitline = [x.strip() for x in line.split()]
+                    flags = splitline[ofs_flags]
+                    # check for 'R', ' E'
+                    if splitline[ofs_flags + 1] == 'E':
+                        flags += ' E'
+                    headers.append((typ, flags, []))
+            count += 1
+
+        if line.startswith(' Section to Segment mapping:'):
+            in_mapping = True
+            count = 0
+        if line == '':
+            in_mapping = False
+        if in_mapping:
+            if count == 1: # header line
+                ofs_segment = line.find('Segment')
+                ofs_sections = line.find('Sections...')
+                if ofs_segment == -1 or ofs_sections == -1:
+                    raise ValueError('Cannot parse elfread -lW output')
+            elif count > 1:
+                segment = int(line[ofs_segment:ofs_sections].strip())
+                sections = line[ofs_sections:].strip().split()
+                headers[segment][2].extend(sections)
             count += 1
     return headers
 
-def check_ELF_NX(executable):
+def check_ELF_NX(executable) -> bool:
     '''
     Check that no sections are writable and executable (including the stack)
     '''
     have_wx = False
     have_gnu_stack = False
-    for (typ, flags) in get_ELF_program_headers(executable):
+    for (typ, flags, _) in get_ELF_program_headers(executable):
         if typ == 'GNU_STACK':
             have_gnu_stack = True
         if 'W' in flags and 'E' in flags: # section is both writable and executable
             have_wx = True
     return have_gnu_stack and not have_wx
 
-def check_ELF_RELRO(executable):
+def check_ELF_RELRO(executable) -> bool:
     '''
     Check for read-only relocations.
     GNU_RELRO program header must exist
     Dynamic section must have BIND_NOW flag
     '''
     have_gnu_relro = False
-    for (typ, flags) in get_ELF_program_headers(executable):
+    for (typ, flags, _) in get_ELF_program_headers(executable):
         # Note: not checking flags == 'R': here as linkers set the permission differently
-        # This does not affect security: the permission flags of the GNU_RELRO program header are ignored, the PT_LOAD header determines the effective permissions.
+        # This does not affect security: the permission flags of the GNU_RELRO program
+        # header are ignored, the PT_LOAD header determines the effective permissions.
         # However, the dynamic linker need to write to this area so these are RW.
         # Glibc itself takes care of mprotecting this area R after relocations are finished.
         # See also https://marc.info/?l=binutils&m=1498883354122353
@@ -91,98 +116,221 @@ def check_ELF_RELRO(executable):
             have_gnu_relro = True
 
     have_bindnow = False
-    p = subprocess.Popen([READELF_CMD, '-d', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
-    (stdout, stderr) = p.communicate()
-    if p.returncode:
-        raise IOError('Error opening file')
+    stdout = run_command([READELF_CMD, '-d', '-W', executable])
+
     for line in stdout.splitlines():
         tokens = line.split()
         if len(tokens)>1 and tokens[1] == '(BIND_NOW)' or (len(tokens)>2 and tokens[1] == '(FLAGS)' and 'BIND_NOW' in tokens[2:]):
             have_bindnow = True
     return have_gnu_relro and have_bindnow
 
-def check_ELF_Canary(executable):
+def check_ELF_Canary(executable) -> bool:
     '''
     Check for use of stack canary
     '''
-    p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
-    (stdout, stderr) = p.communicate()
-    if p.returncode:
-        raise IOError('Error opening file')
+    stdout = run_command([READELF_CMD, '--dyn-syms', '-W', executable])
+
     ok = False
     for line in stdout.splitlines():
         if '__stack_chk_fail' in line:
             ok = True
     return ok
 
-def get_PE_dll_characteristics(executable):
+def check_ELF_separate_code(executable):
     '''
-    Get PE DllCharacteristics bits.
-    Returns a tuple (arch,bits) where arch is 'i386:x86-64' or 'i386'
-    and bits is the DllCharacteristics value.
+    Check that sections are appropriately separated in virtual memory,
+    based on their permissions. This checks for missing -Wl,-z,separate-code
+    and potentially other problems.
     '''
-    p = subprocess.Popen([OBJDUMP_CMD, '-x',  executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
-    (stdout, stderr) = p.communicate()
-    if p.returncode:
-        raise IOError('Error opening file')
-    arch = ''
+    EXPECTED_FLAGS = {
+        # Read + execute
+        '.init': 'R E',
+        '.plt': 'R E',
+        '.plt.got': 'R E',
+        '.plt.sec': 'R E',
+        '.text': 'R E',
+        '.fini': 'R E',
+        # Read-only data
+        '.interp': 'R',
+        '.note.gnu.property': 'R',
+        '.note.gnu.build-id': 'R',
+        '.note.ABI-tag': 'R',
+        '.gnu.hash': 'R',
+        '.dynsym': 'R',
+        '.dynstr': 'R',
+        '.gnu.version': 'R',
+        '.gnu.version_r': 'R',
+        '.rela.dyn': 'R',
+        '.rela.plt': 'R',
+        '.rodata': 'R',
+        '.eh_frame_hdr': 'R',
+        '.eh_frame': 'R',
+        '.qtmetadata': 'R',
+        '.gcc_except_table': 'R',
+        '.stapsdt.base': 'R',
+        # Writable data
+        '.init_array': 'RW',
+        '.fini_array': 'RW',
+        '.dynamic': 'RW',
+        '.got': 'RW',
+        '.data': 'RW',
+        '.bss': 'RW',
+    }
+    # For all LOAD program headers get mapping to the list of sections,
+    # and for each section, remember the flags of the associated program header.
+    flags_per_section = {}
+    for (typ, flags, sections) in get_ELF_program_headers(executable):
+        if typ == 'LOAD':
+            for section in sections:
+                assert(section not in flags_per_section)
+                flags_per_section[section] = flags
+    # Spot-check ELF LOAD program header flags per section
+    # If these sections exist, check them against the expected R/W/E flags
+    for (section, flags) in flags_per_section.items():
+        if section in EXPECTED_FLAGS:
+            if EXPECTED_FLAGS[section] != flags:
+                return False
+    return True
+
+def get_PE_dll_characteristics(executable) -> int:
+    '''Get PE DllCharacteristics bits'''
+    stdout = run_command([OBJDUMP_CMD, '-x',  executable])
+
     bits = 0
     for line in stdout.splitlines():
         tokens = line.split()
-        if len(tokens)>=2 and tokens[0] == 'architecture:':
-            arch = tokens[1].rstrip(',')
         if len(tokens)>=2 and tokens[0] == 'DllCharacteristics':
             bits = int(tokens[1],16)
-    return (arch,bits)
+    return bits
 
 IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020
 IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE    = 0x0040
 IMAGE_DLL_CHARACTERISTICS_NX_COMPAT       = 0x0100
 
-def check_PE_DYNAMIC_BASE(executable):
+def check_PE_DYNAMIC_BASE(executable) -> bool:
     '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)'''
-    (arch,bits) = get_PE_dll_characteristics(executable)
-    reqbits = IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE
-    return (bits & reqbits) == reqbits
+    bits = get_PE_dll_characteristics(executable)
+    return (bits & IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE) == IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE
 
-# On 64 bit, must support high-entropy 64-bit address space layout randomization in addition to DYNAMIC_BASE
-# to have secure ASLR.
-def check_PE_HIGH_ENTROPY_VA(executable):
+# Must support high-entropy 64-bit address space layout randomization
+# in addition to DYNAMIC_BASE to have secure ASLR.
+def check_PE_HIGH_ENTROPY_VA(executable) -> bool:
     '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR'''
-    (arch,bits) = get_PE_dll_characteristics(executable)
-    if arch == 'i386:x86-64':
-        reqbits = IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA
-    else: # Unnecessary on 32-bit
-        assert(arch == 'i386')
-        reqbits = 0
-    return (bits & reqbits) == reqbits
+    bits = get_PE_dll_characteristics(executable)
+    return (bits & IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA) == IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA
 
-def check_PE_NX(executable):
+def check_PE_RELOC_SECTION(executable) -> bool:
+    '''Check for a reloc section. This is required for functional ASLR.'''
+    stdout = run_command([OBJDUMP_CMD, '-h',  executable])
+
+    for line in stdout.splitlines():
+        if '.reloc' in line:
+            return True
+    return False
+
+def check_PE_NX(executable) -> bool:
     '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)'''
-    (arch,bits) = get_PE_dll_characteristics(executable)
+    bits = get_PE_dll_characteristics(executable)
     return (bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT
 
+def get_MACHO_executable_flags(executable) -> List[str]:
+    stdout = run_command([OTOOL_CMD, '-vh', executable])
+
+    flags = []
+    for line in stdout.splitlines():
+        tokens = line.split()
+        # filter first two header lines
+        if 'magic' in tokens or 'Mach' in tokens:
+            continue
+        # filter ncmds and sizeofcmds values
+        flags += [t for t in tokens if not t.isdigit()]
+    return flags
+
+def check_MACHO_PIE(executable) -> bool:
+    '''
+    Check for position independent executable (PIE), allowing for address space randomization.
+    '''
+    flags = get_MACHO_executable_flags(executable)
+    if 'PIE' in flags:
+        return True
+    return False
+
+def check_MACHO_NOUNDEFS(executable) -> bool:
+    '''
+    Check for no undefined references.
+    '''
+    flags = get_MACHO_executable_flags(executable)
+    if 'NOUNDEFS' in flags:
+        return True
+    return False
+
+def check_MACHO_NX(executable) -> bool:
+    '''
+    Check for no stack execution
+    '''
+    flags = get_MACHO_executable_flags(executable)
+    if 'ALLOW_STACK_EXECUTION' in flags:
+        return False
+    return True
+
+def check_MACHO_LAZY_BINDINGS(executable) -> bool:
+    '''
+    Check for no lazy bindings.
+    We don't use or check for MH_BINDATLOAD. See #18295.
+    '''
+    stdout = run_command([OTOOL_CMD, '-l', executable])
+
+    for line in stdout.splitlines():
+        tokens = line.split()
+        if 'lazy_bind_off' in tokens or 'lazy_bind_size' in tokens:
+            if tokens[1] != '0':
+                return False
+    return True
+
+def check_MACHO_Canary(executable) -> bool:
+    '''
+    Check for use of stack canary
+    '''
+    stdout = run_command([OTOOL_CMD, '-Iv', executable])
+
+    ok = False
+    for line in stdout.splitlines():
+        if '___stack_chk_fail' in line:
+            ok = True
+    return ok
+
 CHECKS = {
 'ELF': [
     ('PIE', check_ELF_PIE),
     ('NX', check_ELF_NX),
     ('RELRO', check_ELF_RELRO),
-    ('Canary', check_ELF_Canary)
+    ('Canary', check_ELF_Canary),
+    ('separate_code', check_ELF_separate_code),
 ],
 'PE': [
     ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE),
     ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA),
-    ('NX', check_PE_NX)
+    ('NX', check_PE_NX),
+    ('RELOC_SECTION', check_PE_RELOC_SECTION)
+],
+'MACHO': [
+    ('PIE', check_MACHO_PIE),
+    ('NOUNDEFS', check_MACHO_NOUNDEFS),
+    ('NX', check_MACHO_NX),
+    ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS),
+    ('Canary', check_MACHO_Canary)
 ]
 }
 
-def identify_executable(executable):
+def identify_executable(executable) -> Optional[str]:
     with open(filename, 'rb') as f:
         magic = f.read(4)
     if magic.startswith(b'MZ'):
         return 'PE'
     elif magic.startswith(b'\x7fELF'):
         return 'ELF'
+    elif magic.startswith(b'\xcf\xfa'):
+        return 'MACHO'
     return None
 
 if __name__ == '__main__':
@@ -196,18 +344,12 @@ if __name__ == '__main__':
                 continue
 
             failed = []
-            warning = []
             for (name, func) in CHECKS[etype]:
                 if not func(filename):
-                    if name in NONFATAL:
-                        warning.append(name)
-                    else:
-                        failed.append(name)
+                    failed.append(name)
             if failed:
                 print('%s: failed %s' % (filename, ' '.join(failed)))
                 retval = 1
-            if warning:
-                print('%s: warning %s' % (filename, ' '.join(warning)))
         except IOError:
             print('%s: cannot open' % filename)
             retval = 1

contrib/devtools/symbol-check.py

@@ -3,9 +3,8 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 '''
-A script to check that the (Linux) executables produced by gitian only contain
-allowed gcc, glibc and libstdc++ version symbols.  This makes sure they are
-still compatible with the minimum supported Linux distribution versions.
+A script to check that the executables produced by gitian only contain
+certain symbols and are only linked against allowed libraries.
 
 Example usage:
 
@@ -15,31 +14,32 @@ import subprocess
 import re
 import sys
 import os
+from typing import List, Optional, Tuple
 
-# Debian 6.0.9 (Squeeze) has:
+# Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases
 #
-# - g++ version 4.4.5 (https://packages.debian.org/search?suite=default&section=all&arch=any&searchon=names&keywords=g%2B%2B)
-# - libc version 2.11.3 (https://packages.debian.org/search?suite=default&section=all&arch=any&searchon=names&keywords=libc6)
-# - libstdc++ version 4.4.5 (https://packages.debian.org/search?suite=default&section=all&arch=any&searchon=names&keywords=libstdc%2B%2B6)
+# - g++ version 4.9.2 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=g%2B%2B)
+# - libc version 2.19 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=libc6)
 #
-# Ubuntu 10.04.4 (Lucid Lynx) has:
+# Ubuntu 16.04 (Xenial) EOL: 2024. https://wiki.ubuntu.com/Releases
 #
-# - g++ version 4.4.3 (http://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=lucid&section=all)
-# - libc version 2.11.1 (http://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=lucid&section=all)
-# - libstdc++ version 4.4.3 (http://packages.ubuntu.com/search?suite=lucid&section=all&arch=any&keywords=libstdc%2B%2B&searchon=names)
+# - g++ version 5.3.1 (https://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=xenial&section=all)
+# - libc version 2.23.0 (https://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=xenial&section=all)
+#
+# CentOS 7 EOL: 2024. https://wiki.centos.org/FAQ/General
+#
+# - g++ version 4.8.5 (http://mirror.centos.org/centos/7/os/x86_64/Packages/)
+# - libc version 2.17 (http://mirror.centos.org/centos/7/os/x86_64/Packages/)
 #
 # Taking the minimum of these as our target.
 #
-# According to GNU ABI document (http://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to:
-#   GCC 4.4.0: GCC_4.4.0
-#   GCC 4.4.2: GLIBCXX_3.4.13, CXXABI_1.3.3
-#   (glibc)    GLIBC_2_11
+# According to GNU ABI document (https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to:
+#   GCC 4.8.5: GCC_4.8.0
+#   (glibc)    GLIBC_2_17
 #
 MAX_VERSIONS = {
-'GCC':       (4,4,0),
-'CXXABI':    (1,3,3),
-'GLIBCXX':   (3,4,13),
-'GLIBC':     (2,11),
+'GCC':       (4,8,0),
+'GLIBC':     (2,17),
 'LIBATOMIC': (1,0)
 }
 # See here for a description of _IO_stdin_used:
@@ -47,17 +47,20 @@ MAX_VERSIONS = {
 
 # Ignore symbols that are exported as part of every executable
 IGNORE_EXPORTS = {
-'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr'
+'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr',
+'environ', '_environ', '__environ',
 }
 READELF_CMD = os.getenv('READELF', '/usr/bin/readelf')
 CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt')
+OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump')
+OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool')
+
 # Allowed NEEDED libraries
-ALLOWED_LIBRARIES = {
+ELF_ALLOWED_LIBRARIES = {
 # bitcoind and bitcoin-qt
 'libgcc_s.so.1', # GCC base support
 'libc.so.6', # C library
 'libpthread.so.0', # threading
-'libanl.so.1', # DNS resolve
 'libm.so.6', # math library
 'librt.so.1', # real-time (clock)
 'libatomic.so.1',
@@ -79,6 +82,45 @@ ARCH_MIN_GLIBC_VER = {
 'AArch64':(2,17),
 'RISC-V': (2,27)
 }
+
+MACHO_ALLOWED_LIBRARIES = {
+# bitcoind and bitcoin-qt
+'libc++.1.dylib', # C++ Standard Library
+'libSystem.B.dylib', # libc, libm, libpthread, libinfo
+# bitcoin-qt only
+'AppKit', # user interface
+'ApplicationServices', # common application tasks.
+'Carbon', # deprecated c back-compat API
+'CoreFoundation', # low level func, data types
+'CoreGraphics', # 2D rendering
+'CoreServices', # operating system services
+'CoreText', # interface for laying out text and handling fonts.
+'Foundation', # base layer functionality for apps/frameworks
+'ImageIO', # read and write image file formats.
+'IOKit', # user-space access to hardware devices and drivers.
+'libobjc.A.dylib', # Objective-C runtime library
+}
+
+PE_ALLOWED_LIBRARIES = {
+'ADVAPI32.dll', # security & registry
+'IPHLPAPI.DLL', # IP helper API
+'KERNEL32.dll', # win32 base APIs
+'msvcrt.dll', # C standard library for MSVC
+'SHELL32.dll', # shell API
+'USER32.dll', # user interface
+'WS2_32.dll', # sockets
+# bitcoin-qt only
+'dwmapi.dll', # desktop window manager
+'GDI32.dll', # graphics device interface
+'IMM32.dll', # input method editor
+'ole32.dll', # component object model
+'OLEAUT32.dll', # OLE Automation API
+'SHLWAPI.dll', # light weight shell API
+'UxTheme.dll',
+'VERSION.dll', # version checking
+'WINMM.dll', # WinMM audio API
+}
+
 class CPPFilt(object):
     '''
     Demangle C++ symbol names.
@@ -98,15 +140,15 @@ class CPPFilt(object):
         self.proc.stdout.close()
         self.proc.wait()
 
-def read_symbols(executable, imports=True):
+def read_symbols(executable, imports=True) -> List[Tuple[str, str, str]]:
     '''
-    Parse an ELF executable and return a list of (symbol,version) tuples
+    Parse an ELF executable and return a list of (symbol,version, arch) tuples
     for dynamic, imported symbols.
     '''
     p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', '-h', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
     (stdout, stderr) = p.communicate()
     if p.returncode:
-        raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip()))
+        raise IOError('Could not read symbols for {}: {}'.format(executable, stderr.strip()))
     syms = []
     for line in stdout.splitlines():
         line = line.split()
@@ -121,7 +163,7 @@ def read_symbols(executable, imports=True):
                 syms.append((sym, version, arch))
     return syms
 
-def check_version(max_versions, version, arch):
+def check_version(max_versions, version, arch) -> bool:
     if '_' in version:
         (lib, _, ver) = version.rpartition('_')
     else:
@@ -132,7 +174,7 @@ def check_version(max_versions, version, arch):
         return False
     return ver <= max_versions[lib] or lib == 'GLIBC' and ver <= ARCH_MIN_GLIBC_VER[arch]
 
-def read_libraries(filename):
+def elf_read_libraries(filename) -> List[str]:
     p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
     (stdout, stderr) = p.communicate()
     if p.returncode:
@@ -148,26 +190,117 @@ def read_libraries(filename):
                 raise ValueError('Unparseable (NEEDED) specification')
     return libraries
 
-if __name__ == '__main__':
+def check_imported_symbols(filename) -> bool:
     cppfilt = CPPFilt()
+    ok = True
+    for sym, version, arch in read_symbols(filename, True):
+        if version and not check_version(MAX_VERSIONS, version, arch):
+            print('{}: symbol {} from unsupported version {}'.format(filename, cppfilt(sym), version))
+            ok = False
+    return ok
+
+def check_exported_symbols(filename) -> bool:
+    cppfilt = CPPFilt()
+    ok = True
+    for sym,version,arch in read_symbols(filename, False):
+        if arch == 'RISC-V' or sym in IGNORE_EXPORTS:
+            continue
+        print('{}: export of symbol {} not allowed'.format(filename, cppfilt(sym)))
+        ok = False
+    return ok
+
+def check_ELF_libraries(filename) -> bool:
+    ok = True
+    for library_name in elf_read_libraries(filename):
+        if library_name not in ELF_ALLOWED_LIBRARIES:
+            print('{}: NEEDED library {} is not allowed'.format(filename, library_name))
+            ok = False
+    return ok
+
+def macho_read_libraries(filename) -> List[str]:
+    p = subprocess.Popen([OTOOL_CMD, '-L', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
+    (stdout, stderr) = p.communicate()
+    if p.returncode:
+        raise IOError('Error opening file')
+    libraries = []
+    for line in stdout.splitlines():
+        tokens = line.split()
+        if len(tokens) == 1: # skip executable name
+            continue
+        libraries.append(tokens[0].split('/')[-1])
+    return libraries
+
+def check_MACHO_libraries(filename) -> bool:
+    ok = True
+    for dylib in macho_read_libraries(filename):
+        if dylib not in MACHO_ALLOWED_LIBRARIES:
+            print('{} is not in ALLOWED_LIBRARIES!'.format(dylib))
+            ok = False
+    return ok
+
+def pe_read_libraries(filename) -> List[str]:
+    p = subprocess.Popen([OBJDUMP_CMD, '-x', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
+    (stdout, stderr) = p.communicate()
+    if p.returncode:
+        raise IOError('Error opening file')
+    libraries = []
+    for line in stdout.splitlines():
+        if 'DLL Name:' in line:
+            tokens = line.split(': ')
+            libraries.append(tokens[1])
+    return libraries
+
+def check_PE_libraries(filename) -> bool:
+    ok = True
+    for dylib in pe_read_libraries(filename):
+        if dylib not in PE_ALLOWED_LIBRARIES:
+            print('{} is not in ALLOWED_LIBRARIES!'.format(dylib))
+            ok = False
+    return ok
+
+CHECKS = {
+'ELF': [
+    ('IMPORTED_SYMBOLS', check_imported_symbols),
+    ('EXPORTED_SYMBOLS', check_exported_symbols),
+    ('LIBRARY_DEPENDENCIES', check_ELF_libraries)
+],
+'MACHO': [
+    ('DYNAMIC_LIBRARIES', check_MACHO_libraries)
+],
+'PE' : [
+    ('DYNAMIC_LIBRARIES', check_PE_libraries)
+]
+}
+
+def identify_executable(executable) -> Optional[str]:
+    with open(filename, 'rb') as f:
+        magic = f.read(4)
+    if magic.startswith(b'MZ'):
+        return 'PE'
+    elif magic.startswith(b'\x7fELF'):
+        return 'ELF'
+    elif magic.startswith(b'\xcf\xfa'):
+        return 'MACHO'
+    return None
+
+if __name__ == '__main__':
     retval = 0
     for filename in sys.argv[1:]:
-        # Check imported symbols
-        for sym,version,arch in read_symbols(filename, True):
-            if version and not check_version(MAX_VERSIONS, version, arch):
-                print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym), version))
-                retval = 1
-        # Check exported symbols
-        if arch != 'RISC-V':
-            for sym,version,arch in read_symbols(filename, False):
-                if sym in IGNORE_EXPORTS:
-                    continue
-                print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym)))
-                retval = 1
-        # Check dependency libraries
-        for library_name in read_libraries(filename):
-            if library_name not in ALLOWED_LIBRARIES:
-                print('%s: NEEDED library %s is not allowed' % (filename, library_name))
+        try:
+            etype = identify_executable(filename)
+            if etype is None:
+                print('{}: unknown format'.format(filename))
                 retval = 1
+                continue
 
+            failed = []
+            for (name, func) in CHECKS[etype]:
+                if not func(filename):
+                    failed.append(name)
+            if failed:
+                print('{}: failed {}'.format(filename, ' '.join(failed)))
+                retval = 1
+        except IOError:
+            print('{}: cannot open'.format(filename))
+            retval = 1
     sys.exit(retval)

contrib/devtools/test-security-check.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2015-2018 The Bitcoin Core developers
+# Copyright (c) 2015-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 '''
@@ -20,10 +20,9 @@ def write_testcode(filename):
     ''')
 
 def call_security_check(cc, source, executable, options):
-    subprocess.check_call([cc,source,'-o',executable] + options)
-    p = subprocess.Popen(['./security-check.py',executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
-    (stdout, stderr) = p.communicate()
-    return (p.returncode, stdout.rstrip())
+    subprocess.run([cc,source,'-o',executable] + options, check=True)
+    p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True)
+    return (p.returncode, p.stdout.rstrip())
 
 class TestSecurityChecks(unittest.TestCase):
     def test_ELF(self):
@@ -32,27 +31,54 @@ class TestSecurityChecks(unittest.TestCase):
         cc = 'gcc'
         write_testcode(source)
 
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE']),
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']),
                 (1, executable+': failed PIE NX RELRO Canary'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE']),
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']),
                 (1, executable+': failed PIE RELRO Canary'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE']),
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']),
                 (1, executable+': failed PIE RELRO'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE']),
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']),
                 (1, executable+': failed RELRO'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE']),
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']),
+                (1, executable+': failed separate_code'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']),
                 (0, ''))
 
-    def test_64bit_PE(self):
+    def test_PE(self):
         source = 'test1.c'
         executable = 'test1.exe'
         cc = 'x86_64-w64-mingw32-gcc'
         write_testcode(source)
 
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va']), (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA NX'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va']), (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--no-high-entropy-va']), (1, executable+': failed HIGH_ENTROPY_VA'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va']), (0, ''))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']),
+            (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']),
+            (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']),
+            (1, executable+': failed HIGH_ENTROPY_VA RELOC_SECTION'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va','-no-pie','-fno-PIE']),
+            (1, executable+': failed RELOC_SECTION'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']),
+            (0, ''))
+
+    def test_MACHO(self):
+        source = 'test1.c'
+        executable = 'test1'
+        cc = 'clang'
+        write_testcode(source)
+
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']),
+            (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS Canary'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']),
+            (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']),
+            (1, executable+': failed PIE NOUNDEFS LAZY_BINDINGS'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']),
+            (1, executable+': failed PIE LAZY_BINDINGS'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']),
+            (1, executable+': failed PIE'))
+        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all']),
+            (0, ''))
 
 if __name__ == '__main__':
     unittest.main()

contrib/devtools/test_deterministic_coverage.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019 The Bitcoin Core developers
+# Copyright (c) 2019-2020 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 #
@@ -16,13 +16,12 @@ GCOV_EXECUTABLE="gcov"
 NON_DETERMINISTIC_TESTS=(
     "blockfilter_index_tests/blockfilter_index_initial_sync"  # src/checkqueue.h: In CCheckQueue::Loop(): while (queue.empty()) { ... }
     "coinselector_tests/knapsack_solver_test"                 # coinselector_tests.cpp: if (equal_sets(setCoinsRet, setCoinsRet2))
-    "denialofservice_tests/DoS_mapOrphans"                    # denialofservice_tests.cpp: it = mapOrphanTransactions.lower_bound(InsecureRand256());
     "fs_tests/fsbridge_fstream"                               # deterministic test failure?
     "miner_tests/CreateNewBlock_validity"                     # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
     "scheduler_tests/manythreads"                             # scheduler.cpp: CScheduler::serviceQueue()
     "scheduler_tests/singlethreadedscheduler_ordered"         # scheduler.cpp: CScheduler::serviceQueue()
-    "tx_validationcache_tests/checkinputs_test"               # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
-    "tx_validationcache_tests/tx_mempool_block_doublespend"   # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
+    "txvalidationcache_tests/checkinputs_test"                # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
+    "txvalidationcache_tests/tx_mempool_block_doublespend"    # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
     "txindex_tests/txindex_initial_sync"                      # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
     "txvalidation_tests/tx_mempool_reject_coinbase"           # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
     "validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)

contrib/devtools/utxo_snapshot.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+export LC_ALL=C
+
+set -ueo pipefail
+
+if (( $# < 3 )); then
+  echo 'Usage: utxo_snapshot.sh <generate-at-height> <snapshot-out-path> <bitcoin-cli-call ...>'
+  echo
+  echo "  if <snapshot-out-path> is '-', don't produce a snapshot file but instead print the "
+  echo "  expected assumeutxo hash"
+  echo
+  echo 'Examples:'
+  echo
+  echo "  ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/bitcoin-cli -datadir=\$(pwd)/testdata"
+  echo '  ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/bitcoin-cli'
+  exit 1
+fi
+
+GENERATE_AT_HEIGHT="${1}"; shift;
+OUTPUT_PATH="${1}"; shift;
+# Most of the calls we make take a while to run, so pad with a lengthy timeout.
+BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999"
+
+# Block we'll invalidate/reconsider to rewind/fast-forward the chain.
+PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) )
+
+(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while")
+${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}"
+
+if [[ "${OUTPUT_PATH}" = "-" ]]; then
+  (>&2 echo "Generating txoutset info...")
+  ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g'
+else
+  (>&2 echo "Generating UTXO snapshot...")
+  ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}"
+fi
+
+(>&2 echo "Restoring chain to original height; this may take a while")
+${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}"

contrib/filter-lcov.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+# Copyright (c) 2017-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 import argparse
 

contrib/gitian-build.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+# Copyright (c) 2018-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 import argparse
 import os
@@ -51,10 +54,8 @@ def build():
     os.chdir('gitian-builder')
     os.makedirs('inputs', exist_ok=True)
 
-    subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz'])
-    subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch'])
-    subprocess.check_call(["echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c"], shell=True)
-    subprocess.check_call(["echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c"], shell=True)
+    subprocess.check_call(['wget', '-O', 'inputs/osslsigncode-2.0.tar.gz', 'https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz'])
+    subprocess.check_call(["echo '5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz' | sha256sum -c"], shell=True)
     subprocess.check_call(['make', '-C', '../bitcoin/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common'])
 
     if args.linux:
@@ -208,7 +209,7 @@ def main():
     args.macos = 'm' in args.os
 
     # Disable for MacOS if no SDK found
-    if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'):
+    if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz'):
         print('Cannot build for MacOS, SDK does not exist. Will build for other OSes')
         args.macos = False
 

contrib/gitian-descriptors/assign_DISTNAME

@@ -0,0 +1,12 @@
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# A helper script to be sourced into the gitian descriptors
+
+if RECENT_TAG="$(git describe --exact-match HEAD)"; then
+    VERSION="${RECENT_TAG#v}"
+else
+    VERSION="$(git rev-parse --short=12 HEAD)"
+fi
+DISTNAME="bitcoin-${VERSION}"