From e1e5960e10a9329d9f55a3967d546ffbdd896030 Mon Sep 17 00:00:00 2001
From: Brian Liotti <bliotti@protonmail.com>
Date: Sat, 4 Jul 2020 01:18:41 -0400
Subject: [PATCH 1/2] script: Add previous_release.py

closes #18132
added GPG verify for binaries

co-authored-by: bboot <bboot@cisco.com>
---
 contrib/devtools/previous_release.py | 222 +++++++++++++++++++++++++++
 1 file changed, 222 insertions(+)
 create mode 100755 contrib/devtools/previous_release.py

diff --git a/contrib/devtools/previous_release.py b/contrib/devtools/previous_release.py
new file mode 100755
index 00000000000..bea4f560105
--- /dev/null
+++ b/contrib/devtools/previous_release.py
@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2018-2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Build previous releases.
+
+import argparse
+import contextlib
+from fnmatch import fnmatch
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+import hashlib
+
+
+@contextlib.contextmanager
+def pushd(new_dir) -> None:
+    previous_dir = os.getcwd()
+    os.chdir(new_dir)
+    try:
+        yield
+    finally:
+        os.chdir(previous_dir)
+
+
+def download_binary(tag, args) -> int:
+    if Path(tag).is_dir():
+        if not args.remove_dir:
+            print('Using cached {}'.format(tag))
+            return 0
+        shutil.rmtree(tag)
+    Path(tag).mkdir()
+    bin_path = 'bin/bitcoin-core-{}'.format(tag[1:])
+    match = re.compile('v(.*)(rc[0-9]+)$').search(tag)
+    if match:
+        bin_path = 'bin/bitcoin-core-{}/test.{}'.format(
+            match.group(1), match.group(2))
+    tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format(
+        tag=tag[1:], platform=args.platform)
+    sha256Sums = "SHA256SUMS-{tag}.asc".format(tag=tag[1:])
+    tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format(
+        bin_path=bin_path, tarball=tarball)
+    sha256SumsUrl = 'https://bitcoincore.org/{bin_path}/SHA256SUMS.asc'.format(
+        bin_path=bin_path)
+
+    print('Fetching: {tarballUrl}'.format(tarballUrl=tarballUrl))
+    print('Fetching: {sha256SumsUrl}'.format(sha256SumsUrl=sha256SumsUrl))
+
+    header, status = subprocess.Popen(
+        ['curl', '-I', tarballUrl], stdout=subprocess.PIPE).communicate()
+    if re.search("404 Not Found", header.decode("utf-8")):
+        print("Binary tag was not found")
+        return 1
+
+    curlCmds = [
+        ['curl', '-O', tarballUrl],
+        ['curl', "-o", sha256Sums, sha256SumsUrl],
+    ]
+
+    for cmd in curlCmds:
+        ret = subprocess.run(cmd).returncode
+        if ret:
+            return ret
+
+    hasher = hashlib.sha256()
+    with open(tarball, "rb") as afile:
+        buf = afile.read()
+        hasher.update(buf)
+    afile.close()
+    tarballHash = hasher.hexdigest()
+    file = open(sha256Sums, 'r', encoding="utf-8")
+    lst = list(file.readlines())
+    file.close()
+    lastline = lst[len(lst)-1]
+
+    for line in lst:
+        if re.search(tarballHash, line):
+            print("Checksum matched")
+            break
+        elif lastline == line:
+            print("Checksum did not match")
+            Path(tarball).unlink()
+            return 1
+
+    # Bitcoin Core Release Signing Keys v0.11.0+
+    signingKey = "01EA5486DE18A882D4C2684590C8019E36C2E964"
+
+    isKeyPresent = subprocess.run(
+        ["gpg", "--list-keys", signingKey]).returncode
+    if isKeyPresent:
+        return isKeyPresent
+
+    isVerified = subprocess.run(
+        ["gpg", "--verify", sha256Sums]).returncode
+    if isVerified:
+        return isVerified
+
+    # Extract tarball
+    ret = subprocess.run(['tar', '-zxf', tarball, '-C', tag,
+                          '--strip-components=1',
+                          'bitcoin-{tag}'.format(tag=tag[1:])]).returncode
+    if ret:
+        return ret
+
+    Path(tarball).unlink()
+    Path(sha256Sums).unlink()
+    return 0
+
+
+def build_release(tag, args) -> int:
+    githubUrl = "https://github.com/bitcoin/bitcoin"
+    if args.remove_dir:
+        if Path(tag).is_dir():
+            shutil.rmtree(tag)
+    if not Path(tag).is_dir():
+        # fetch new tags
+        subprocess.run(
+            ["git", "fetch", githubUrl, "--tags"])
+        output = subprocess.check_output(['git', 'tag', '-l', tag])
+        if not output:
+            print('Tag {} not found'.format(tag))
+            return 1
+    ret = subprocess.run([
+        'git', 'clone', githubUrl, tag
+    ]).returncode
+    if ret:
+        return ret
+    with pushd(tag):
+        ret = subprocess.run(['git', 'checkout', tag]).returncode
+        if ret:
+            return ret
+        host = args.host
+        if args.depends:
+            with pushd('depends'):
+                ret = subprocess.run(['make', 'NO_QT=1']).returncode
+                if ret:
+                    return ret
+                host = os.environ.get(
+                    'HOST', subprocess.check_output(['./config.guess']))
+        config_flags = '--prefix={pwd}/depends/{host} '.format(
+            pwd=os.getcwd(),
+            host=host) + args.config_flags
+        cmds = [
+            './autogen.sh',
+            './configure {}'.format(config_flags),
+            'make',
+        ]
+        for cmd in cmds:
+            ret = subprocess.run(cmd.split()).returncode
+            if ret:
+                return ret
+        # Move binaries, so they're in the same place as in the
+        # release download
+        Path('bin').mkdir(exist_ok=True)
+        files = ['bitcoind', 'bitcoin-cli', 'bitcoin-tx']
+        for f in files:
+            Path('src/'+f).rename('bin/'+f)
+    return 0
+
+
+def check_host(args) -> int:
+    args.host = os.environ.get('HOST', subprocess.check_output(
+        './depends/config.guess').decode())
+    if args.download_binary:
+        platforms = {
+            'x86_64-*-linux*': 'x86_64-linux-gnu',
+            'x86_64-apple-darwin*': 'osx64',
+        }
+        args.platform = ''
+        for pattern, target in platforms.items():
+            if fnmatch(args.host, pattern):
+                args.platform = target
+        if not args.platform:
+            print('Not sure which binary to download for {}'.format(args.host))
+            return 1
+    return 0
+
+
+def main(args) -> int:
+    if not Path(args.target_dir).is_dir():
+        Path(args.target_dir).mkdir(exist_ok=True, parents=True)
+    print("Releases directory: {}".format(args.target_dir))
+    ret = check_host(args)
+    if ret:
+        return ret
+    if args.download_binary:
+        with pushd(args.target_dir):
+            for tag in args.tags:
+                ret = download_binary(tag, args)
+                if ret:
+                    return ret
+        return 0
+    args.config_flags = os.environ.get('CONFIG_FLAGS', '')
+    args.config_flags += ' --without-gui --disable-tests --disable-bench'
+    with pushd(args.target_dir):
+        for tag in args.tags:
+            ret = build_release(tag, args)
+            if ret:
+                return ret
+    return 0
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('-r', '--remove-dir', action='store_true',
+                        help='remove existing directory.')
+    parser.add_argument('-d', '--depends', action='store_true',
+                        help='use depends.')
+    parser.add_argument('-b', '--download-binary', action='store_true',
+                        help='download release binary.')
+    parser.add_argument('-t', '--target-dir', action='store',
+                        help='target directory.', default='releases')
+    parser.add_argument('tags', nargs='+',
+                        help="release tags. e.g.: v0.18.1 v0.20.0rc2")
+    args = parser.parse_args()
+    sys.exit(main(args))

From 9c34aff39309b8adc99d347e07b6ddb5366498e9 Mon Sep 17 00:00:00 2001
From: Brian Liotti <bliotti@protonmail.com>
Date: Sat, 4 Jul 2020 01:22:58 -0400
Subject: [PATCH 2/2] Remove previous_release.sh

---
 ci/test/05_before_script.sh                   |   2 +-
 contrib/devtools/previous_release.sh          | 152 ------------------
 .../feature_backwards_compatibility.py        |   2 +-
 test/functional/mempool_compatibility.py      |   2 +-
 test/functional/wallet_upgradewallet.py       |   2 +-
 5 files changed, 4 insertions(+), 156 deletions(-)
 delete mode 100755 contrib/devtools/previous_release.sh

diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh
index 057395eba68..131ea21677d 100755
--- a/ci/test/05_before_script.sh
+++ b/ci/test/05_before_script.sh
@@ -48,6 +48,6 @@ if [ -z "$NO_DEPENDS" ]; then
 fi
 if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then
   BEGIN_FOLD previous-versions
-  DOCKER_EXEC contrib/devtools/previous_release.sh -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}"
+  DOCKER_EXEC contrib/devtools/previous_release.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}"
   END_FOLD
 fi
diff --git a/contrib/devtools/previous_release.sh b/contrib/devtools/previous_release.sh
deleted file mode 100755
index d375291f47d..00000000000
--- a/contrib/devtools/previous_release.sh
+++ /dev/null
@@ -1,152 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018-2020 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-#
-# Build previous releases.
-
-export LC_ALL=C
-
-CONFIG_FLAGS=""
-FUNCTIONAL_TESTS=0
-DELETE_EXISTING=0
-USE_DEPENDS=0
-DOWNLOAD_BINARY=0
-CONFIG_FLAGS=""
-TARGET="releases"
-
-while getopts ":hfrdbt:" opt; do
-  case $opt in
-    h)
-      echo "Usage: .previous_release.sh [options] tag1 tag2"
-      echo "  options:"
-      echo "  -h   Print this message"
-      echo "  -f   Configure for functional tests"
-      echo "  -r   Remove existing directory"
-      echo "  -d   Use depends"
-      echo "  -b   Download release binary"
-      echo "  -t   Target directory (default: releases)"
-      exit 0
-      ;;
-    f)
-      FUNCTIONAL_TESTS=1
-      CONFIG_FLAGS="$CONFIG_FLAGS --without-gui --disable-tests --disable-bench"
-      ;;
-    r)
-      DELETE_EXISTING=1
-      ;;
-    d)
-      USE_DEPENDS=1
-      ;;
-    b)
-      DOWNLOAD_BINARY=1
-      ;;
-    t)
-      TARGET=$OPTARG
-      ;;
-    \?)
-      echo "Invalid option: -$OPTARG" >&2
-      exit 1
-      ;;
-  esac
-done
-
-shift $((OPTIND-1))
-
-if [ -z "$1" ]; then
-  echo "Specify release tag(s), e.g.: .previous_release v0.15.1"
-  exit 1
-fi
-
-if [ ! -d "$TARGET" ]; then
-  mkdir -p $TARGET
-fi
-
-if [ "$DOWNLOAD_BINARY" -eq "1" ]; then
-  HOST="${HOST:-$(./depends/config.guess)}"
-  case "$HOST" in
-    x86_64-*-linux*)
-      PLATFORM=x86_64-linux-gnu
-      ;;
-    x86_64-apple-darwin*)
-      PLATFORM=osx64
-      ;;
-    *)
-      echo "Not sure which binary to download for $HOST."
-      exit 1
-      ;;
-  esac
-fi
-
-echo "Releases directory: $TARGET"
-pushd "$TARGET" || exit 1
-{
-  for tag in "$@"
-  do
-    if [ "$DELETE_EXISTING" -eq "1" ]; then
-      if [ -d "$tag" ]; then
-        rm -r "$tag"
-      fi
-    fi
-
-    if [ "$DOWNLOAD_BINARY" -eq "0" ]; then
-
-      if [ ! -d "$tag" ]; then
-        if [ -z $(git tag -l "$tag") ]; then
-          echo "Tag $tag not found"
-          exit 1
-        fi
-
-        git clone https://github.com/bitcoin/bitcoin "$tag"
-        pushd "$tag" || exit 1
-        {
-          git checkout "$tag"
-          if [ "$USE_DEPENDS" -eq "1" ]; then
-            pushd depends || exit 1
-            {
-              if [ "$FUNCTIONAL_TESTS" -eq "1" ]; then
-                make NO_QT=1
-              else
-                make
-              fi
-              HOST="${HOST:-$(./config.guess)}"
-            }
-            popd || exit 1
-            CONFIG_FLAGS="--prefix=$PWD/depends/$HOST $CONFIG_FLAGS"
-          fi
-          ./autogen.sh
-          ./configure $CONFIG_FLAGS
-          make
-          # Move binaries, so they're in the same place as in the release download:
-          mkdir bin
-          mv src/bitcoind src/bitcoin-cli src/bitcoin-tx bin
-          if [ "$FUNCTIONAL_TESTS" -eq "0" ]; then
-            mv src/qt/bitcoin-qt bin
-          fi
-        }
-        popd || exit 1
-      fi
-    else
-      if [ -d "$tag" ]; then
-        echo "Using cached $tag"
-      else
-        mkdir "$tag"
-        if [[ "$tag" =~ v(.*)(rc[0-9]+)$ ]]; then
-            BIN_PATH="bin/bitcoin-core-${BASH_REMATCH[1]}/test.${BASH_REMATCH[2]}"
-        else
-            BIN_PATH="bin/bitcoin-core-${tag:1}"
-        fi
-        URL="https://bitcoincore.org/$BIN_PATH/bitcoin-${tag:1}-$PLATFORM.tar.gz"
-        echo "Fetching: $URL"
-        if ! curl -O -f $URL; then
-            echo "Download failed."
-            exit 1
-        fi
-        tar -zxf "bitcoin-${tag:1}-$PLATFORM.tar.gz" -C "$tag" --strip-components=1 "bitcoin-${tag:1}"
-        rm "bitcoin-${tag:1}-$PLATFORM.tar.gz"
-      fi
-    fi
-  done
-}
-popd || exit 1
diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py
index 5cddd6527e7..07dd0f8f82c 100755
--- a/test/functional/feature_backwards_compatibility.py
+++ b/test/functional/feature_backwards_compatibility.py
@@ -6,7 +6,7 @@
 
 Test various backwards compatibility scenarios. Download the previous node binaries:
 
-contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
+contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
 
 v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py.
 Due to a hardfork in regtest, it can't be used to sync nodes.
diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py
index 999399dec0a..31fb7519048 100755
--- a/test/functional/mempool_compatibility.py
+++ b/test/functional/mempool_compatibility.py
@@ -8,7 +8,7 @@ NOTE: The test is designed to prevent cases when compatibility is broken acciden
 In case we need to break mempool compatibility we can continue to use the test by just bumping the version number.
 
 Download node binaries:
-contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
+contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
 
 Only v0.15.2 is required by this test. The rest is used in other backwards compatibility tests.
 """
diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py
index cc2139a027e..1a76f65215b 100755
--- a/test/functional/wallet_upgradewallet.py
+++ b/test/functional/wallet_upgradewallet.py
@@ -6,7 +6,7 @@
 
 Test upgradewallet RPC. Download node binaries:
 
-contrib/devtools/previous_release.sh -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
+contrib/devtools/previous_release.py -b v0.19.1 v0.18.1 v0.17.1 v0.16.3 v0.15.2
 
 Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py
 """