From eaf7da9acbae0dfb19f0d0fd6619623e7270019e Mon Sep 17 00:00:00 2001
From: Stephan Oeste <emzy@emzy.de>
Date: Fri, 29 Jul 2022 20:13:48 +0200
Subject: [PATCH 1/2] Add Unfurl to the prod installer

---
 production/install                    | 173 +++++++++++++++++++-------
 production/mempool-config.unfurl.json |  13 ++
 production/unfurl-build               |  62 +++++++++
 production/unfurl-kill                |   2 +
 production/unfurl-start               |   6 +
 5 files changed, 211 insertions(+), 45 deletions(-)
 create mode 100644 production/mempool-config.unfurl.json
 create mode 100755 production/unfurl-build
 create mode 100755 production/unfurl-kill
 create mode 100755 production/unfurl-start

diff --git a/production/install b/production/install
index 729ff33e0..053ffef70 100755
--- a/production/install
+++ b/production/install
@@ -39,6 +39,9 @@ BITCOIN_INSTALL=ON
 BISQ_INSTALL=ON
 ELEMENTS_INSTALL=ON
 
+# install UNFURL
+UNFURL_INSTALL=ON
+
 # configure 4 network instances
 BITCOIN_MAINNET_ENABLE=ON
 BITCOIN_MAINNET_MINFEE_ENABLE=ON
@@ -48,6 +51,9 @@ BISQ_MAINNET_ENABLE=ON
 ELEMENTS_LIQUID_ENABLE=ON
 ELEMENTS_LIQUIDTESTNET_ENABLE=ON
 
+# install Electrs
+ELECTRS_INSTALL=ON
+
 # enable lightmode and disable compaction to fit on 1TB SSD drive
 BITCOIN_ELECTRS_LIGHT_MODE=ON
 BITCOIN_ELECTRS_COMPACTION=OFF
@@ -278,6 +284,12 @@ BISQ_GROUP=bisq
 # bisq home folder, needs about 1GB
 BISQ_HOME=/bisq
 
+# Unfurl user/group
+UNFURL_USER=unfurl
+UNFURL_GROUP=unfurl
+# Unfurl home folder
+UNFURL_HOME=/unfurl
+
 # liquid user/group
 ELEMENTS_USER=elements
 ELEMENTS_GROUP=elements
@@ -315,6 +327,13 @@ BISQ_REPO_BRANCH=master
 BISQ_LATEST_RELEASE=master
 echo -n '.'
 
+UNFURL_REPO_URL=https://github.com/mempool/mempool
+UNFURL_REPO_NAME=unfurl
+UNFURL_REPO_BRANCH=master
+#UNFURL_LATEST_RELEASE=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
+UNFURL_LATEST_RELEASE=master
+echo -n '.'
+
 ELEMENTS_REPO_URL=https://github.com/ElementsProject/elements
 ELEMENTS_REPO_NAME=elements
 ELEMENTS_REPO_BRANCH=master
@@ -351,6 +370,10 @@ DEBIAN_PKG+=(libboost-system-dev libboost-filesystem-dev libboost-chrono-dev lib
 DEBIAN_PKG+=(nodejs npm mariadb-server nginx-core python3-certbot-nginx rsync ufw)
 DEBIAN_PKG+=(geoipupdate)
 
+DEBIAN_UNFURL_PKG=()
+DEBIAN_UNFURL_PKG+=(cups chromium-bsu libatk1.0 libatk-bridge2.0 libxkbcommon-dev libxcomposite-dev)
+DEBIAN_UNFURL_PKG+=(libxdamage-dev libxrandr-dev libgbm-dev libpango1.0-dev libasound-dev)
+
 # packages needed for mempool ecosystem
 FREEBSD_PKG=()
 FREEBSD_PKG+=(zsh sudo git screen curl wget calc neovim)
@@ -712,6 +735,7 @@ Signet:Enable Bitcoin Signet:ON
 Liquid:Enable Elements Liquid:ON
 Liquidtestnet:Enable Elements Liquidtestnet:ON
 Bisq:Enable Bisq:ON
+Unfurl:Enable Unfurl:ON
 EOF
 
 cat $input | sed -e 's/^/"/' -e 's/:/" "/g' -e 's/$/"/' >$output
@@ -785,6 +809,12 @@ else
     ELEMENTS_INSTALL=OFF
 fi
 
+if [ "${BITCOIN_INSTALL}" = ON -o "${ELEMENTS_INSTALL}" = ON ];then
+    ELECTRS_INSTALL=ON
+else
+    ELECTRS_INSTALL=OFF
+fi
+
 if grep Bisq $tempfile >/dev/null 2>&1;then
     BISQ_INSTALL=ON
     BISQ_MAINNET_ENABLE=ON
@@ -793,6 +823,12 @@ else
     BISQ_MAINNET_ENABLE=OFF
 fi
 
+if grep Unfurl $tempfile >/dev/null 2>&1;then
+    UNFURL_INSTALL=ON
+else
+    UNFURL_INSTALL=OFF
+fi
+
 ##################
 ## dialog part 2 #
 ##################
@@ -1074,52 +1110,55 @@ fi
 # Bitcoin -> Electrs installation #
 ###################################
 
-echo "[*] Creating Bitcoin Electrs data folder"
-osSudo "${ROOT_USER}" mkdir -p "${BITCOIN_ELECTRS_HOME}"
-osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_ELECTRS_HOME}"
-if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then 
-    osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_MAINNET_DATA}"
+if [ "${ELECTRS_INSTALL}" = ON ];then
+
+    echo "[*] Creating Bitcoin Electrs data folder"
+    osSudo "${ROOT_USER}" mkdir -p "${BITCOIN_ELECTRS_HOME}"
+    osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_ELECTRS_HOME}"
+    if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then
+        osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_MAINNET_DATA}"
+    fi
+    if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then
+        osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_TESTNET_DATA}"
+    fi
+    if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then
+        osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_SIGNET_DATA}"
+    fi
+
+    echo "[*] Cloning Bitcoin Electrs repo from ${BITCOIN_ELECTRS_REPO_URL}"
+    osSudo "${BITCOIN_USER}" git config --global advice.detachedHead false
+    osSudo "${BITCOIN_USER}" git clone --branch "${BITCOIN_ELECTRS_REPO_BRANCH}" "${BITCOIN_ELECTRS_REPO_URL}" "${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME}"
+
+    echo "[*] Checking out Electrs ${BITCOIN_ELECTRS_LATEST_RELEASE}"
+    osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME} && git checkout ${BITCOIN_ELECTRS_LATEST_RELEASE}"
+
+    case $OS in
+        FreeBSD)
+           echo "[*] Installing Rust from pkg install"
+        ;;
+        Debian)
+           echo "[*] Installing Rust from rustup.rs"
+           osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y"
+        ;;
+    esac
+
+    echo "[*] Building Bitcoin Electrs release binary"
+    osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version" || true
+
+    case $OS in
+        FreeBSD)
+            echo "[*] Patching Bitcoin Electrs code for FreeBSD"
+            osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_HOME}/.cargo/registry/src/github.com-1ecc6299db9ec823/sysconf-0.3.4\" && patch -p1 < \"${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/freebsd/sysconf.patch\""
+            osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/new_index/\" && sed -i.bak -e s/Snappy/None/ db.rs && rm db.rs.bak"
+            osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/bin/\" && sed -i.bak -e 's/from_secs(5)/from_secs(1)/' electrs.rs && rm electrs.rs.bak"
+        ;;
+        Debian)
+        ;;
+    esac
+
+    echo "[*] Building Bitcoin Electrs release binary"
+    osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version"
 fi
-if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then 
-    osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_TESTNET_DATA}"
-fi
-if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then 
-    osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_SIGNET_DATA}"
-fi
-
-echo "[*] Cloning Bitcoin Electrs repo from ${BITCOIN_ELECTRS_REPO_URL}"
-osSudo "${BITCOIN_USER}" git config --global advice.detachedHead false
-osSudo "${BITCOIN_USER}" git clone --branch "${BITCOIN_ELECTRS_REPO_BRANCH}" "${BITCOIN_ELECTRS_REPO_URL}" "${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME}"
-
-echo "[*] Checking out Electrs ${BITCOIN_ELECTRS_LATEST_RELEASE}"
-osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME} && git checkout ${BITCOIN_ELECTRS_LATEST_RELEASE}"
-
-case $OS in
-    FreeBSD)
-       echo "[*] Installing Rust from pkg install"
-    ;;
-    Debian)
-       echo "[*] Installing Rust from rustup.rs"
-       osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y"
-    ;;
-esac
-
-echo "[*] Building Bitcoin Electrs release binary"
-osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version" || true
-
-case $OS in
-    FreeBSD)
-        echo "[*] Patching Bitcoin Electrs code for FreeBSD"
-        osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_HOME}/.cargo/registry/src/github.com-1ecc6299db9ec823/sysconf-0.3.4\" && patch -p1 < \"${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/freebsd/sysconf.patch\""
-        osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/new_index/\" && sed -i.bak -e s/Snappy/None/ db.rs && rm db.rs.bak"
-        osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_ELECTRS_HOME}/src/bin/\" && sed -i.bak -e 's/from_secs(5)/from_secs(1)/' electrs.rs && rm electrs.rs.bak"
-    ;;
-    Debian)
-    ;;
-esac
-
-echo "[*] Building Bitcoin Electrs release binary"
-osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version"
 
 ##################################
 # Liquid -> Electrs installation #
@@ -1246,6 +1285,50 @@ if [ "${BISQ_INSTALL}" = ON ];then
     esac
 fi
 
+#######################
+# Unfurl installation #
+#######################
+
+if [ "${UNFURL_INSTALL}" = ON ];then
+
+    echo "[*] Creating Unfurl user"
+    osGroupCreate "${UNFURL_GROUP}"
+    osUserCreate "${UNFURL_USER}" "${UNFURL_HOME}" "${UNFURL_GROUP}"
+    osSudo "${ROOT_USER}" chsh -s `which zsh` "${UNFURL_USER}"
+
+    echo "[*] Creating Unfurl folder"
+    osSudo "${ROOT_USER}" mkdir -p "${UNFURL_HOME}"
+    osSudo "${ROOT_USER}" chown -R "${UNFURL_USER}:${UNFURL_GROUP}" "${UNFURL_HOME}"
+    osSudo "${UNFURL_USER}" touch "${UNFURL_HOME}/.zshrc"
+
+    echo "[*] Insalling Unfurl source"
+    case $OS in
+
+        FreeBSD)
+            echo "[*] FIXME: Unfurl must be installed manually on FreeBSD"
+        ;;
+
+        Debian)
+            echo "[*] Installing packages for Unfurl"
+            osPackageInstall ${DEBIAN_UNFURL_PKG[@]}
+            echo "[*] Cloning Mempool (Unfurl) repo from ${UNFURL_REPO_URL}"
+            osSudo "${UNFURL_USER}" git config --global pull.rebase true
+            osSudo "${UNFURL_USER}" git config --global advice.detachedHead false
+            osSudo "${UNFURL_USER}" git clone --branch "${UNFURL_REPO_BRANCH}" "${UNFURL_REPO_URL}" "${UNFURL_HOME}/${UNFURL_REPO_NAME}"
+            osSudo "${UNFURL_USER}" ln -s unfurl/production/unfurl-build upgrade
+            osSudo "${UNFURL_USER}" ln -s unfurl/production/unfurl-kill stop
+            osSudo "${UNFURL_USER}" ln -s unfurl/production/unfurl-start start
+            echo "[*] Installing nvm.sh from GitHub"
+            osSudo "${UNFURL_USER}" sh -c 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | zsh'
+
+            echo "[*] Building NodeJS via nvm.sh"
+            osSudo "${UNFURL_USER}" zsh -c 'source ~/.zshrc ; nvm install v16.16.0 --shared-zlib'
+
+        ;;
+    esac
+
+fi
+
 ################################
 # Bitcoin instance for Mainnet #
 ################################
diff --git a/production/mempool-config.unfurl.json b/production/mempool-config.unfurl.json
new file mode 100644
index 000000000..5cf67d5ac
--- /dev/null
+++ b/production/mempool-config.unfurl.json
@@ -0,0 +1,13 @@
+{
+  "SERVER": {
+    "HOST": "https://mempool.space",
+    "HTTP_PORT": 8001
+  },
+  "MEMPOOL": {
+    "HTTP_HOST": "https://mempool.space",
+    "HTTP_PORT": 443
+  },
+  "PUPPETEER": {
+    "CLUSTER_SIZE": 8
+  }
+}
diff --git a/production/unfurl-build b/production/unfurl-build
new file mode 100755
index 000000000..5b838e0ae
--- /dev/null
+++ b/production/unfurl-build
@@ -0,0 +1,62 @@
+#!/usr/bin/env zsh
+PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:$HOME/bin
+HOSTNAME=$(hostname)
+LOCATION=$(hostname|cut -d . -f2)
+LOCKFILE="${HOME}/lock"
+REF=$(echo "${1:=origin/master}"|sed -e 's!:!/!')
+
+if [ -f "${LOCKFILE}" ];then
+    echo "upgrade already running? check lockfile ${LOCKFILE}"
+    exit 1
+fi
+
+# on exit, remove lockfile but preserve exit code
+trap "rv=\$?; rm -f "${LOCKFILE}"; exit \$rv" INT TERM EXIT
+
+# create lockfile
+touch "${LOCKFILE}"
+
+# notify logged in users
+echo "Upgrading unfurler to ${REF}" | wall
+
+update_repo()
+{
+    echo "[*] Upgrading unfurler to ${REF}"
+    cd "$HOME/unfurl/unfurler" || exit 1
+
+    git fetch origin || exit 1
+    for remote in origin;do
+        git remote add "${remote}" "https://github.com/${remote}/mempool" >/dev/null 2>&1
+        git fetch "${remote}" || exit 1
+    done
+
+    if [ $(git tag -l "${REF}") ];then
+        git reset --hard "tags/${REF}" || exit 1
+    elif [ $(git branch -r -l "origin/${REF}") ];then
+        git reset --hard "origin/${REF}" || exit 1
+    else
+        git reset --hard "${REF}" || exit 1
+    fi
+    export HASH=$(git rev-parse HEAD)
+}
+
+build_backend()
+{
+    echo "[*] Building backend for unfurler"
+    [ -z "${HASH}" ] && exit 1
+    cd "$HOME/unfurl/unfurler" || exit 1
+    if [ ! -e "config.json" ];then
+        cp "${HOME}/unfurl/production/mempool-config.unfurl.json" "config.json"
+    fi
+    npm install || exit 1
+    npm run build || exit 1
+}
+
+update_repo
+build_backend
+
+# notify everyone
+echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general mempool.dev
+echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general "mempool.ops.${LOCATION}"
+
+exit 0
diff --git a/production/unfurl-kill b/production/unfurl-kill
new file mode 100755
index 000000000..ae48552c2
--- /dev/null
+++ b/production/unfurl-kill
@@ -0,0 +1,2 @@
+#!/usr/bin/env zsh
+killall sh node
diff --git a/production/unfurl-start b/production/unfurl-start
new file mode 100755
index 000000000..29b5ddf3e
--- /dev/null
+++ b/production/unfurl-start
@@ -0,0 +1,6 @@
+#!/usr/bin/env zsh
+export NVM_DIR="$HOME/.nvm"
+source "$NVM_DIR/nvm.sh"
+
+cd "${HOME}/unfurl/unfurler/" && \
+screen -dmS "unfurl" sh -c 'while true;do npm run start-production;sleep 1;done'

From a413c6ebb86b2da7e9c34fe721d8a6575c7ec41c Mon Sep 17 00:00:00 2001
From: wiz <j@wiz.biz>
Date: Sat, 30 Jul 2022 15:25:02 +0200
Subject: [PATCH 2/2] Separate electrs into bitcoin electrs and elements
 electrs

---
 production/install | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/production/install b/production/install
index 053ffef70..6022a9a5e 100755
--- a/production/install
+++ b/production/install
@@ -51,12 +51,11 @@ BISQ_MAINNET_ENABLE=ON
 ELEMENTS_LIQUID_ENABLE=ON
 ELEMENTS_LIQUIDTESTNET_ENABLE=ON
 
-# install Electrs
-ELECTRS_INSTALL=ON
-
 # enable lightmode and disable compaction to fit on 1TB SSD drive
+BITCOIN_ELECTRS_INSTALL=ON
 BITCOIN_ELECTRS_LIGHT_MODE=ON
 BITCOIN_ELECTRS_COMPACTION=OFF
+ELEMENTS_ELECTRS_INSTALL=ON
 ELEMENTS_ELECTRS_LIGHT_MODE=ON
 ELEMENTS_ELECTRS_COMPACTION=OFF
 
@@ -809,10 +808,16 @@ else
     ELEMENTS_INSTALL=OFF
 fi
 
-if [ "${BITCOIN_INSTALL}" = ON -o "${ELEMENTS_INSTALL}" = ON ];then
-    ELECTRS_INSTALL=ON
+if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then
+    BITCOIN_ELECTRS_INSTALL=ON
 else
-    ELECTRS_INSTALL=OFF
+    BITCOIN_ELECTRS_INSTALL=OFF
+fi
+
+if [ "${ELEMENTS_LIQUID_ENABLE}" = ON -o "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
+    ELEMENTS_ELECTRS_INSTALL=ON
+else
+    ELEMENTS_ELECTRS_INSTALL=OFF
 fi
 
 if grep Bisq $tempfile >/dev/null 2>&1;then
@@ -1110,7 +1115,7 @@ fi
 # Bitcoin -> Electrs installation #
 ###################################
 
-if [ "${ELECTRS_INSTALL}" = ON ];then
+if [ "${BITCOIN_ELECTRS_INSTALL}" = ON ];then
 
     echo "[*] Creating Bitcoin Electrs data folder"
     osSudo "${ROOT_USER}" mkdir -p "${BITCOIN_ELECTRS_HOME}"
@@ -1164,14 +1169,18 @@ fi
 # Liquid -> Electrs installation #
 ##################################
 
-if [ "${ELEMENTS_INSTALL}" = ON ;then
+if [ "${ELEMENTS_ELECTRS_INSTALL}" = ON ];then
 
     echo "[*] Creating Liquid Electrs data folder"
     osSudo "${ROOT_USER}" mkdir -p "${ELEMENTS_ELECTRS_HOME}"
     osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}"
     osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_ELECTRS_HOME}"
-    osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUID_DATA}"
-    osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUIDTESTNET_DATA}"
+    if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then
+        osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUID_DATA}"
+    fi
+    if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
+        osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUIDTESTNET_DATA}"
+    fi
     
     echo "[*] Cloning Liquid Electrs repo from ${ELEMENTS_ELECTRS_REPO_URL}"
     osSudo "${ELEMENTS_USER}" git config --global advice.detachedHead false