From bd897ce79f72a44a2e609f95433e251a3fd9eb9c Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sat, 29 Aug 2020 11:26:25 +0300 Subject: [PATCH 1/2] scripted-diff: Move previous_release.py to test/get_previous_releases.py -BEGIN VERIFY SCRIPT- OLD=contrib/devtools/previous_release.py NEW=test/get_previous_releases.py sed -i "s|$OLD|$NEW|g" $(git grep -l $OLD) git mv $OLD $NEW -END VERIFY SCRIPT- --- ci/test/05_before_script.sh | 2 +- test/functional/feature_backwards_compatibility.py | 2 +- test/functional/mempool_compatibility.py | 2 +- test/functional/wallet_upgradewallet.py | 2 +- .../previous_release.py => test/get_previous_releases.py | 0 5 files changed, 4 insertions(+), 4 deletions(-) rename contrib/devtools/previous_release.py => test/get_previous_releases.py (100%) diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 131ea21677d..8ce839fc046 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -48,6 +48,6 @@ if [ -z "$NO_DEPENDS" ]; then fi if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then BEGIN_FOLD previous-versions - DOCKER_EXEC contrib/devtools/previous_release.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" + DOCKER_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" END_FOLD fi diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index 07dd0f8f82c..126c9fe9295 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -6,7 +6,7 @@ Test various backwards compatibility scenarios. Download the previous node binaries: -contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 +test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py. Due to a hardfork in regtest, it can't be used to sync nodes. diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index 31fb7519048..75ca8d32369 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -8,7 +8,7 @@ NOTE: The test is designed to prevent cases when compatibility is broken acciden In case we need to break mempool compatibility we can continue to use the test by just bumping the version number. Download node binaries: -contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 +test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 Only v0.15.2 is required by this test. The rest is used in other backwards compatibility tests. """ diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index 1a76f65215b..f3d6e748294 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -6,7 +6,7 @@ Test upgradewallet RPC. Download node binaries: -contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 +test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2 Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py """ diff --git a/contrib/devtools/previous_release.py b/test/get_previous_releases.py similarity index 100% rename from contrib/devtools/previous_release.py rename to test/get_previous_releases.py From 0374e821bd9e9498ce9c03aa8e5435870019978b Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Sat, 29 Aug 2020 11:28:53 +0300 Subject: [PATCH 2/2] util: Hard code previous release tarball checksums --- ci/test/00_setup_env_native_qt5.sh | 2 +- test/get_previous_releases.py | 61 ++++++++++++++++++------------ 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index f9d869b4fdd..6f2e39429c1 100644 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -15,5 +15,5 @@ export RUN_SECURITY_TESTS="true" export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.1 v0.18.1 v0.19.1" +export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1" export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-c++17 --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process" diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index 5599051cf30..1348b8246b4 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -20,6 +20,40 @@ import sys import hashlib +SHA256_SUMS = { +"d40f18b4e43c6e6370ef7db9131f584fbb137276ec2e3dba67a4b267f81cb644": "bitcoin-0.15.2-aarch64-linux-gnu.tar.gz", +"54fb877a148a6ad189a1e1ab1ff8b11181e58ff2aaf430da55b3fd46ae549a6b": "bitcoin-0.15.2-arm-linux-gnueabihf.tar.gz", +"2b843506c3f1af0eeca5854a920264f9a829f02d0d50328005950ddcbe88874d": "bitcoin-0.15.2-i686-pc-linux-gnu.tar.gz", +"87e9340ff3d382d543b2b69112376077f0c8b4f7450d372e83b68f5a1e22b2df": "bitcoin-0.15.2-osx64.tar.gz", +"566be44190fd76daa01f13d428939dadfb8e3daacefc8fa17f433cad28f73bd5": "bitcoin-0.15.2-x86_64-linux-gnu.tar.gz", + +"0768c6c15caffbaca6524824c9563b42c24f70633c681c2744649158aa3fd484": "bitcoin-0.16.3-aarch64-linux-gnu.tar.gz", +"fb2818069854a6ad20ea03b28b55dbd35d8b1f7d453e90b83eace5d0098a2a87": "bitcoin-0.16.3-arm-linux-gnueabihf.tar.gz", +"75a537844313b0a84bdb61ffcdc5c4ce19a738f7ddf71007cd2edf664efd7c37": "bitcoin-0.16.3-i686-pc-linux-gnu.tar.gz", +"78c3bff3b619a19aed575961ea43cc9e142959218835cf51aede7f0b764fc25d": "bitcoin-0.16.3-osx64.tar.gz", +"5d422a9d544742bc0df12427383f9c2517433ce7b58cf672b9a9b17c2ef51e4f": "bitcoin-0.16.3-x86_64-linux-gnu.tar.gz", + +"5a6b35d1a348a402f2d2d6ab5aed653a1a1f13bc63aaaf51605e3501b0733b7a": "bitcoin-0.17.2-aarch64-linux-gnu.tar.gz", +"d1913a5d19c8e8da4a67d1bd5205d03c8614dfd2e02bba2fe3087476643a729e": "bitcoin-0.17.2-arm-linux-gnueabihf.tar.gz", +"d295fc93f39bbf0fd937b730a93184899a2eb6c3a6d53f3d857cbe77ef89b98c": "bitcoin-0.17.2-i686-pc-linux-gnu.tar.gz", +"a783ba20706dbfd5b47fbedf42165fce70fbbc7d78003305d964f6b3da14887f": "bitcoin-0.17.2-osx64.tar.gz", +"943f9362b9f11130177839116f48f809d83478b4c28591d486ee9a7e35179da6": "bitcoin-0.17.2-x86_64-linux-gnu.tar.gz", + +"88f343af72803b851c7da13874cc5525026b0b55e63e1b5e1298390c4688adc6": "bitcoin-0.18.1-aarch64-linux-gnu.tar.gz", +"cc7d483e4b20c5dabd4dcaf304965214cf4934bcc029ca99cbc9af00d3771a1f": "bitcoin-0.18.1-arm-linux-gnueabihf.tar.gz", +"989e847b3e95fc9fedc0b109cae1b4fa43348f2f712e187a118461876af9bd16": "bitcoin-0.18.1-i686-pc-linux-gnu.tar.gz", +"b7bbcee7a7540f711b171d6981f939ca8482005fde22689bc016596d80548bb1": "bitcoin-0.18.1-osx64.tar.gz", +"425ee5ec631ae8da71ebc1c3f5c0269c627cf459379b9b030f047107a28e3ef8": "bitcoin-0.18.1-riscv64-linux-gnu.tar.gz", +"600d1db5e751fa85903e935a01a74f5cc57e1e7473c15fd3e17ed21e202cfe5a": "bitcoin-0.18.1-x86_64-linux-gnu.tar.gz", + +"3a80431717842672df682bdb619e66523b59541483297772a7969413be3502ff": "bitcoin-0.19.1-aarch64-linux-gnu.tar.gz", +"657f28213823d240dd3324d14829702f9ad6f0710f8bdd1c379cb3c447197f48": "bitcoin-0.19.1-arm-linux-gnueabihf.tar.gz", +"10d1e53208aa7603022f4acc084a046299ab4ccf25fe01e81b3fb6f856772589": "bitcoin-0.19.1-i686-pc-linux-gnu.tar.gz", +"1ae1b87de26487075cd2fd22e0d4ead87d969bd55c44f2f1d873ecdc6147ebb3": "bitcoin-0.19.1-osx64.tar.gz", +"aa7a9563b48aa79252c8e7b6a41c07a5441bd9f14c5e4562cc72720ea6cb0ee5": "bitcoin-0.19.1-riscv64-linux-gnu.tar.gz", +"5fcac9416e486d4960e1a946145566350ca670f9aaba99de6542080851122e4c": "bitcoin-0.19.1-x86_64-linux-gnu.tar.gz" +} + @contextlib.contextmanager def pushd(new_dir) -> None: previous_dir = os.getcwd() @@ -44,14 +78,10 @@ def download_binary(tag, args) -> int: match.group(1), match.group(2)) tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format( tag=tag[1:], platform=args.platform) - sha256Sums = "SHA256SUMS-{tag}.asc".format(tag=tag[1:]) tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format( bin_path=bin_path, tarball=tarball) - sha256SumsUrl = 'https://bitcoincore.org/{bin_path}/SHA256SUMS.asc'.format( - bin_path=bin_path) print('Fetching: {tarballUrl}'.format(tarballUrl=tarballUrl)) - print('Fetching: {sha256SumsUrl}'.format(sha256SumsUrl=sha256SumsUrl)) header, status = subprocess.Popen( ['curl', '--head', tarballUrl], stdout=subprocess.PIPE).communicate() @@ -60,8 +90,7 @@ def download_binary(tag, args) -> int: return 1 curlCmds = [ - ['curl', '--remote-name', tarballUrl], - ['curl', '--output', sha256Sums, sha256SumsUrl], + ['curl', '--remote-name', tarballUrl] ] for cmd in curlCmds: @@ -73,29 +102,12 @@ def download_binary(tag, args) -> int: with open(tarball, "rb") as afile: hasher.update(afile.read()) tarballHash = hasher.hexdigest() - tarballHash = '{} {}\n'.format(tarballHash, tarball) - with open(sha256Sums, 'r', encoding="utf-8") as afile: - shasums = afile.readlines() - if tarballHash not in shasums: + if tarballHash not in SHA256_SUMS or SHA256_SUMS[tarballHash] != tarball: print("Checksum did not match") - Path(tarball).unlink() return 1 print("Checksum matched") - # Bitcoin Core Release Signing Keys v0.11.0+ - signingKey = "01EA5486DE18A882D4C2684590C8019E36C2E964" - - isKeyPresent = subprocess.run( - ["gpg", "--list-keys", signingKey]).returncode - if isKeyPresent: - return isKeyPresent - - isVerified = subprocess.run( - ["gpg", "--verify", sha256Sums]).returncode - if isVerified: - return isVerified - # Extract tarball ret = subprocess.run(['tar', '-zxf', tarball, '-C', tag, '--strip-components=1', @@ -104,7 +116,6 @@ def download_binary(tag, args) -> int: return ret Path(tarball).unlink() - Path(sha256Sums).unlink() return 0