From f7ff6336f2fd739edb28aee0ab482ab4e07498f2 Mon Sep 17 00:00:00 2001 From: Bastien Date: Mon, 1 Feb 2021 15:54:27 +0100 Subject: [PATCH 1/7] Add CACHE_DIR var --- backend/mempool-config.sample.json | 3 ++- backend/src/api/disk-cache.ts | 5 +++-- backend/src/config.ts | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index c4cea1cbd..fe9122572 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -5,7 +5,8 @@ "HTTP_PORT": 8999, "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", - "POLL_RATE_MS": 2000 + "POLL_RATE_MS": 2000, + "CACHE_DIR": "./" }, "CORE_RPC": { "HOST": "127.0.0.1", diff --git a/backend/src/api/disk-cache.ts b/backend/src/api/disk-cache.ts index 201391d84..98faf2a9b 100644 --- a/backend/src/api/disk-cache.ts +++ b/backend/src/api/disk-cache.ts @@ -5,10 +5,11 @@ import * as cluster from 'cluster'; import memPool from './mempool'; import blocks from './blocks'; import logger from '../logger'; +import config from '../config'; class DiskCache { - private static FILE_NAME = './cache.json'; - private static FILE_NAMES = './cache{number}.json'; + private static FILE_NAME = config.MEMPOOL.CACHE_DIR + 'cache.json'; + private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + 'cache{number}.json'; private static CHUNK_SIZE = 10000; constructor() { if (!cluster.isMaster) { diff --git a/backend/src/config.ts b/backend/src/config.ts index d6546284a..314fcd7c3 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -8,6 +8,7 @@ interface IConfig { SPAWN_CLUSTER_PROCS: number; API_URL_PREFIX: string; POLL_RATE_MS: number; + CACHE_DIR: string; }; ESPLORA: { REST_API_URL: string; @@ -66,7 +67,8 @@ const defaults: IConfig = { 'HTTP_PORT': 8999, 'SPAWN_CLUSTER_PROCS': 0, 'API_URL_PREFIX': '/api/v1/', - 'POLL_RATE_MS': 2000 + 'POLL_RATE_MS': 2000, + 'CACHE_DIR': './' }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', From feb5e9632330114548531718bda25843b6755b76 Mon Sep 17 00:00:00 2001 From: Bastien Date: Mon, 1 Feb 2021 17:25:08 +0100 Subject: [PATCH 2/7] Add docker dir to build mempool containers --- docker/README.md | 5 + docker/backend/Dockerfile | 28 +++++ docker/backend/mempool-config.json | 38 ++++++ docker/backend/start.sh | 21 ++++ docker/backend/wait-for-it.sh | 182 +++++++++++++++++++++++++++++ docker/build.sh | 14 +++ docker/frontend/Dockerfile | 35 ++++++ docker/frontend/entrypoint.sh | 6 + docker/frontend/wait-for | 84 +++++++++++++ docker/init.sh | 22 ++++ docker/mysql/Dockerfile | 7 ++ docker/mysql/mysql-mempool.cnf | 2 + 12 files changed, 444 insertions(+) create mode 100644 docker/README.md create mode 100644 docker/backend/Dockerfile create mode 100644 docker/backend/mempool-config.json create mode 100644 docker/backend/start.sh create mode 100644 docker/backend/wait-for-it.sh create mode 100755 docker/build.sh create mode 100644 docker/frontend/Dockerfile create mode 100644 docker/frontend/entrypoint.sh create mode 100644 docker/frontend/wait-for create mode 100755 docker/init.sh create mode 100644 docker/mysql/Dockerfile create mode 100644 docker/mysql/mysql-mempool.cnf diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..559cb0240 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,5 @@ +## Init +./docker/init.sh + +## Build +./docker/build.sh YOUR_DOCKER_HUB_NAME VERSION diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile new file mode 100644 index 000000000..f8740a1eb --- /dev/null +++ b/docker/backend/Dockerfile @@ -0,0 +1,28 @@ +FROM node:12-buster-slim AS builder + +WORKDIR /build +COPY . . +RUN apt-get update +RUN apt-get install -y build-essential python3 pkg-config +RUN npm ci --production +RUN npm i typescript +RUN npm run build + +FROM node:12-buster-slim + +WORKDIR /backend + +RUN mkdir cache + +COPY --from=builder /build/ . + +RUN chmod +x /backend/start.sh +RUN chmod +x /backend/wait-for-it.sh + +RUN chown -R 1000:1000 /backend && chmod -R 755 /backend + +USER 1000 + +EXPOSE 8999 + +CMD ["/backend/start.sh"] diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json new file mode 100644 index 000000000..f8820560e --- /dev/null +++ b/docker/backend/mempool-config.json @@ -0,0 +1,38 @@ +{ + "MEMPOOL": { + "NETWORK": "mainnet", + "BACKEND": "electrum", + "HTTP_PORT": 8999, + "SPAWN_CLUSTER_PROCS": 0, + "API_URL_PREFIX": "/api/v1/", + "POLL_RATE_MS": 2000, + "CACHE_DIR": "" + }, + "CORE_RPC": { + "HOST": "", + "PORT": , + "USERNAME": "", + "PASSWORD": "" + }, + "ELECTRUM": { + "HOST": "", + "PORT": , + "TLS_ENABLED": false, + "TX_LOOKUPS": true + }, + "ESPLORA": { + "REST_API_URL": "http://127.0.0.1:3000" + }, + "DATABASE": { + "ENABLED": true, + "HOST": "", + "PORT": 3306, + "DATABASE": "mempool", + "USERNAME": "mempool", + "PASSWORD": "mempool" + }, + "STATISTICS": { + "ENABLED": true, + "TX_PER_SECOND_SAMPLE_PERIOD": 150 + } +} diff --git a/docker/backend/start.sh b/docker/backend/start.sh new file mode 100644 index 000000000..480ef598e --- /dev/null +++ b/docker/backend/start.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +RPC_HOST=$RPC_HOST +RPC_PORT=$RPC_PORT +RPC_USER=$RPC_USER +RPC_PASS=$RPC_PASS +ELECTRS_HOST=$ELECTRS_HOST +ELECTRS_PORT=$ELECTRS_PORT +MYSQL_HOST=$MYSQL_HOST +CACHE_DIR=$CACHE_DIR + +sed -i "s//${RPC_HOST}/g" mempool-config.json +sed -i "s//${RPC_PORT}/g" mempool-config.json +sed -i "s//${RPC_USER}/g" mempool-config.json +sed -i "s//${RPC_PASS}/g" mempool-config.json +sed -i "s//${ELECTRS_HOST}/g" mempool-config.json +sed -i "s//${ELECTRS_PORT}/g" mempool-config.json +sed -i "s//${MYSQL_HOST}/g" mempool-config.json +sed -i "s//${CACHE_DIR}/g" mempool-config.json + +node /backend/dist/index.js diff --git a/docker/backend/wait-for-it.sh b/docker/backend/wait-for-it.sh new file mode 100644 index 000000000..d990e0d36 --- /dev/null +++ b/docker/backend/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 000000000..8817d7c82 --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +VERSION=$(cat mempool-version) + +cd mempool + +cd ./backend +docker buildx build --platform linux/amd64,linux/arm64 -t $1/mempool-backend:$2 --push . + +cd ../frontend +docker buildx build --platform linux/amd64,linux/arm64 -t $1/mempool-frontend:$2 --push . + +cd ../mysql +docker buildx build --platform linux/amd64,linux/arm64 -t $1/mempool-db:$2 --push . diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile new file mode 100644 index 000000000..e06b9563b --- /dev/null +++ b/docker/frontend/Dockerfile @@ -0,0 +1,35 @@ +FROM node:12-buster-slim AS builder + +WORKDIR /build +COPY . . +RUN apt-get update +RUN apt-get install -y build-essential rsync +RUN npm i +RUN npm run build + +FROM nginx:1.17.8-alpine + +WORKDIR /patch + +COPY --from=builder /build/entrypoint.sh . +COPY --from=builder /build/wait-for . +COPY --from=builder /build/dist/mempool /var/www/mempool +COPY --from=builder /build/nginx.conf /etc/nginx/ +COPY --from=builder /build/nginx-mempool.conf /etc/nginx/conf.d/ + +RUN chmod +x /patch/entrypoint.sh +RUN chmod +x /patch/wait-for + +RUN chown -R 1000:1000 /patch && chmod -R 755 /patch && \ + chown -R 1000:1000 /var/cache/nginx && \ + chown -R 1000:1000 /var/log/nginx && \ + chown -R 1000:1000 /etc/nginx/conf.d +RUN touch /var/run/nginx.pid && \ + chown -R 1000:1000 /var/run/nginx.pid + +USER 1000 + +EXPOSE 8080 + +ENTRYPOINT ["/patch/entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/docker/frontend/entrypoint.sh b/docker/frontend/entrypoint.sh new file mode 100644 index 000000000..d95b84eb8 --- /dev/null +++ b/docker/frontend/entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/sh +API_IP=$API_IP + +sed -i "s//${API_IP}/g" /etc/nginx/conf.d/nginx-mempool.conf + +exec "$@" diff --git a/docker/frontend/wait-for b/docker/frontend/wait-for new file mode 100644 index 000000000..e28e54cb0 --- /dev/null +++ b/docker/frontend/wait-for @@ -0,0 +1,84 @@ +#!/bin/sh + +TIMEOUT=15 +QUIET=0 + +echoerr() { + if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi +} + +usage() { + exitcode="$1" + cat << USAGE >&2 +Usage: + $cmdname host:port [-t timeout] [-- command args] + -q | --quiet Do not output any status messages + -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit "$exitcode" +} + +wait_for() { + if ! command -v nc >/dev/null; then + echoerr 'nc command is missing!' + exit 1 + fi + + for i in `seq $TIMEOUT` ; do + nc -z "$HOST" "$PORT" > /dev/null 2>&1 + + result=$? + if [ $result -eq 0 ] ; then + if [ $# -gt 0 ] ; then + exec "$@" + fi + exit 0 + fi + sleep 1 + done + echo "Operation timed out" >&2 + exit 1 +} + +while [ $# -gt 0 ] +do + case "$1" in + *:* ) + HOST=$(printf "%s\n" "$1"| cut -d : -f 1) + PORT=$(printf "%s\n" "$1"| cut -d : -f 2) + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -t) + TIMEOUT="$2" + if [ "$TIMEOUT" = "" ]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + break + ;; + --help) + usage 0 + ;; + *) + echoerr "Unknown argument: $1" + usage 1 + ;; + esac +done + +if [ "$HOST" = "" -o "$PORT" = "" ]; then + echoerr "Error: you need to provide a host and port to test." + usage 2 +fi + +wait_for "$@" diff --git a/docker/init.sh b/docker/init.sh new file mode 100755 index 000000000..e78f9912c --- /dev/null +++ b/docker/init.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +#backend +gitMaster="\.\.\/\.git\/refs\/heads\/master" +cp .git/refs/heads/master ./backend +cp ./docker/backend/* ./backend/ +sed -i "s/${gitMaster}/master/g" ./backend/src/api/backend-info.ts + +#frontend +localIP="127.0.0.1" +cp ./docker/frontend/* ./frontend +cp ./nginx.conf ./frontend/ +cp ./nginx-mempool.conf ./frontend/ +sed -i "s/${localIP}:/0.0.0.0:80/g" ./frontend/nginx.conf +sed -i "s/${localIP}/0.0.0.0/g" ./frontend/nginx.conf +sed -i "s/user nobody;//g" ./frontend/nginx.conf +sed -i "s/\/etc\/nginx\/nginx-mempool.conf/\/etc\/nginx\/conf.d\/nginx-mempool.conf/g" ./frontend/nginx.conf +sed -i "s/${localIP}:/:/g" ./frontend/nginx-mempool.conf + +#db +cp -fr docker/mysql . +cp ./mariadb-structure.sql ./mysql/setup.sql diff --git a/docker/mysql/Dockerfile b/docker/mysql/Dockerfile new file mode 100644 index 000000000..30a2780ee --- /dev/null +++ b/docker/mysql/Dockerfile @@ -0,0 +1,7 @@ +FROM mariadb:10.5.8 + +WORKDIR /db +COPY . . + +COPY setup.sql /docker-entrypoint-initdb.d/ +COPY mysql-mempool.cnf /etc/mysql/conf.d/ diff --git a/docker/mysql/mysql-mempool.cnf b/docker/mysql/mysql-mempool.cnf new file mode 100644 index 000000000..f4a7fb3bb --- /dev/null +++ b/docker/mysql/mysql-mempool.cnf @@ -0,0 +1,2 @@ +[mysqld] +sql_mode="NO_AUTO_VALUE_ON_ZERO" From c26737ffd642363a44a651542f4c689123ddc5cb Mon Sep 17 00:00:00 2001 From: Bastien Date: Mon, 1 Feb 2021 18:17:50 +0100 Subject: [PATCH 3/7] Update build.sh --- docker/build.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker/build.sh b/docker/build.sh index 8817d7c82..0bfbae4d6 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -1,9 +1,5 @@ #!/bin/sh -VERSION=$(cat mempool-version) - -cd mempool - cd ./backend docker buildx build --platform linux/amd64,linux/arm64 -t $1/mempool-backend:$2 --push . From 6efe055003ee0a7ff0b81e15a6223024cdd99e1f Mon Sep 17 00:00:00 2001 From: Bastien Guillaumat <36412124+bguillaumat@users.noreply.github.com> Date: Tue, 2 Feb 2021 16:34:21 +0100 Subject: [PATCH 4/7] Apply Wiz suggestions Co-authored-by: wiz --- docker/backend/mempool-config.json | 20 ++++++++-------- docker/backend/start.sh | 38 +++++++++++++++++------------- docker/frontend/entrypoint.sh | 4 ++-- docker/init.sh | 10 ++++---- 4 files changed, 39 insertions(+), 33 deletions(-) diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index f8820560e..3bd9d64a0 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -2,21 +2,21 @@ "MEMPOOL": { "NETWORK": "mainnet", "BACKEND": "electrum", - "HTTP_PORT": 8999, + "HTTP_PORT": __MEMPOOL_BACKEND_MAINNET_HTTP_PORT__, "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", "POLL_RATE_MS": 2000, - "CACHE_DIR": "" + "CACHE_DIR": "__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__" }, "CORE_RPC": { - "HOST": "", - "PORT": , - "USERNAME": "", - "PASSWORD": "" + "HOST": "__BITCOIN_MAINNET_RPC_HOST__", + "PORT": __BITCOIN_MAINNET_RPC_PORT__, + "USERNAME": "__BITCOIN_MAINNET_RPC_USER__", + "PASSWORD": "__BITCOIN_MAINNET_RPC_PASS__" }, "ELECTRUM": { - "HOST": "", - "PORT": , + "HOST": "__ELECTRS_MAINNET_HTTP_HOST__", + "PORT": __ELECTRS_MAINNET_HTTP_PORT__, "TLS_ENABLED": false, "TX_LOOKUPS": true }, @@ -25,8 +25,8 @@ }, "DATABASE": { "ENABLED": true, - "HOST": "", - "PORT": 3306, + "HOST": "__MYSQL_HOST__", + "PORT": __MYSQL_PORT__, "DATABASE": "mempool", "USERNAME": "mempool", "PASSWORD": "mempool" diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 480ef598e..ca8e66463 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -1,21 +1,27 @@ #!/bin/sh -RPC_HOST=$RPC_HOST -RPC_PORT=$RPC_PORT -RPC_USER=$RPC_USER -RPC_PASS=$RPC_PASS -ELECTRS_HOST=$ELECTRS_HOST -ELECTRS_PORT=$ELECTRS_PORT -MYSQL_HOST=$MYSQL_HOST -CACHE_DIR=$CACHE_DIR +__BITCOIN_MAINNET_RPC_HOST__=${RPC_HOST} +__BITCOIN_MAINNET_RPC_PORT__=${RPC_PORT} +__BITCOIN_MAINNET_RPC_USER__=${RPC_USER} +__BITCOIN_MAINNET_RPC_PASS__=${RPC_PASS} +__ELECTRS_MAINNET_HTTP_HOST__=${ELECTRS_HOST} +__ELECTRS_MAINNET_HTTP_PORT__=${ELECTRS_PORT} +__MYSQL_HOST__=${MYSQL_HOST} +__MYSQL_PORT__=${MYSQL_PORT} +__MEMPOOL_BACKEND_CACHE_DIR__=${CACHE_DIR} +__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__=${BACKEND_MAINNET_HTTP_HOST} +__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT} -sed -i "s//${RPC_HOST}/g" mempool-config.json -sed -i "s//${RPC_PORT}/g" mempool-config.json -sed -i "s//${RPC_USER}/g" mempool-config.json -sed -i "s//${RPC_PASS}/g" mempool-config.json -sed -i "s//${ELECTRS_HOST}/g" mempool-config.json -sed -i "s//${ELECTRS_PORT}/g" mempool-config.json -sed -i "s//${MYSQL_HOST}/g" mempool-config.json -sed -i "s//${CACHE_DIR}/g" mempool-config.json +sed -i "s/__BITCOIN_MAINNET_RPC_HOST__/${__BITCOIN_MAINNET_RPC_HOST__}/g" mempool-config.json +sed -i "s/__BITCOIN_MAINNET_RPC_PORT__/${__BITCOIN_MAINNET_RPC_PORT__}/g" mempool-config.json +sed -i "s/__BITCOIN_MAINNET_RPC_USER__/${__BITCOIN_MAINNET_RPC_USER__}/g" mempool-config.json +sed -i "s/__BITCOIN_MAINNET_RPC_PASS__/${__BITCOIN_MAINNET_RPC_PASS__}/g" mempool-config.json +sed -i "s/__ELECTRS_MAINNET_HTTP_HOST__/${__ELECTRS_MAINNET_HTTP_HOST__}/g" mempool-config.json +sed -i "s/__ELECTRS_MAINNET_HTTP_PORT__/${__ELECTRS_MAINNET_HTTP_PORT__}/g" mempool-config.json +sed -i "s/__MYSQL_HOST__/${__MYSQL_HOST__}/g" mempool-config.json +sed -i "s/__MYSQL_PORT__/${__MYSQL_PORT__}/g" mempool-config.json +sed -i "s/__MEMPOOL_BACKEND_CACHE_DIR__/${__MEMPOOL_BACKEND_CACHE_DIR__}/g" mempool-config.json +sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT_}/g" mempool-config.json +sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__/${__MEMPOOL_BACKEND_MAINNET_HTTP_HOST_}/g" mempool-config.json node /backend/dist/index.js diff --git a/docker/frontend/entrypoint.sh b/docker/frontend/entrypoint.sh index d95b84eb8..359c8b466 100644 --- a/docker/frontend/entrypoint.sh +++ b/docker/frontend/entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/sh -API_IP=$API_IP +__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__=${BACKEND_MAINNET_HTTP_HOST} -sed -i "s//${API_IP}/g" /etc/nginx/conf.d/nginx-mempool.conf +sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__/${__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__}/g" /etc/nginx/conf.d/nginx-mempool.conf exec "$@" diff --git a/docker/init.sh b/docker/init.sh index e78f9912c..437f48f4f 100755 --- a/docker/init.sh +++ b/docker/init.sh @@ -7,15 +7,15 @@ cp ./docker/backend/* ./backend/ sed -i "s/${gitMaster}/master/g" ./backend/src/api/backend-info.ts #frontend -localIP="127.0.0.1" +localhostIP="127.0.0.1" cp ./docker/frontend/* ./frontend cp ./nginx.conf ./frontend/ cp ./nginx-mempool.conf ./frontend/ -sed -i "s/${localIP}:/0.0.0.0:80/g" ./frontend/nginx.conf -sed -i "s/${localIP}/0.0.0.0/g" ./frontend/nginx.conf +sed -i "s/${localhostIP}:/0.0.0.0:80/g" ./frontend/nginx.conf +sed -i "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf sed -i "s/user nobody;//g" ./frontend/nginx.conf -sed -i "s/\/etc\/nginx\/nginx-mempool.conf/\/etc\/nginx\/conf.d\/nginx-mempool.conf/g" ./frontend/nginx.conf -sed -i "s/${localIP}:/:/g" ./frontend/nginx-mempool.conf +sed -i "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" ./frontend/nginx.conf +sed -i "s/${localhostIP}:/__MEMPOOL_MAINNET_BACKEND_HOST__:/g" ./frontend/nginx-mempool.conf #db cp -fr docker/mysql . From 91d4017aa680b60c1c20d4f10db8bda71f8c8586 Mon Sep 17 00:00:00 2001 From: Bastien Date: Tue, 2 Feb 2021 17:54:01 +0100 Subject: [PATCH 5/7] Update ENV_VARIABLES in scripts && Fix some errors --- docker/backend/Dockerfile | 3 --- docker/backend/start.sh | 28 ++++++++++++++++------------ docker/frontend/entrypoint.sh | 2 +- docker/init.sh | 2 +- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index f8740a1eb..2b42eab91 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -5,15 +5,12 @@ COPY . . RUN apt-get update RUN apt-get install -y build-essential python3 pkg-config RUN npm ci --production -RUN npm i typescript RUN npm run build FROM node:12-buster-slim WORKDIR /backend -RUN mkdir cache - COPY --from=builder /build/ . RUN chmod +x /backend/start.sh diff --git a/docker/backend/start.sh b/docker/backend/start.sh index ca8e66463..b96626f99 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -1,16 +1,21 @@ #!/bin/sh -__BITCOIN_MAINNET_RPC_HOST__=${RPC_HOST} -__BITCOIN_MAINNET_RPC_PORT__=${RPC_PORT} -__BITCOIN_MAINNET_RPC_USER__=${RPC_USER} -__BITCOIN_MAINNET_RPC_PASS__=${RPC_PASS} -__ELECTRS_MAINNET_HTTP_HOST__=${ELECTRS_HOST} -__ELECTRS_MAINNET_HTTP_PORT__=${ELECTRS_PORT} -__MYSQL_HOST__=${MYSQL_HOST} -__MYSQL_PORT__=${MYSQL_PORT} -__MEMPOOL_BACKEND_CACHE_DIR__=${CACHE_DIR} -__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__=${BACKEND_MAINNET_HTTP_HOST} -__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT} +#MEMPOOL +__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT:=8999} +__MEMPOOL_BACKEND_CACHE_DIR__=${CACHE_DIR:=/backend/cache} +# BITCOIN +__BITCOIN_MAINNET_RPC_HOST__=${RPC_HOST:=127.0.0.1} +__BITCOIN_MAINNET_RPC_PORT__=${RPC_PORT:=8332} +__BITCOIN_MAINNET_RPC_USER__=${RPC_USER:=mempool} +__BITCOIN_MAINNET_RPC_PASS__=${RPC_PASS:=mempool} +# ELECTRUM +__ELECTRS_MAINNET_HTTP_HOST__=${ELECTRS_HOST:=127.0.0.1} +__ELECTRS_MAINNET_HTTP_PORT__=${ELECTRS_PORT:=50002} +# MYSQL +__MYSQL_HOST__=${MYSQL_HOST:=127.0.0.1} +__MYSQL_PORT__=${MYSQL_PORT:=3306} + +mkdir __MEMPOOL_BACKEND_CACHE_DIR__ sed -i "s/__BITCOIN_MAINNET_RPC_HOST__/${__BITCOIN_MAINNET_RPC_HOST__}/g" mempool-config.json sed -i "s/__BITCOIN_MAINNET_RPC_PORT__/${__BITCOIN_MAINNET_RPC_PORT__}/g" mempool-config.json @@ -22,6 +27,5 @@ sed -i "s/__MYSQL_HOST__/${__MYSQL_HOST__}/g" mempool-config.json sed -i "s/__MYSQL_PORT__/${__MYSQL_PORT__}/g" mempool-config.json sed -i "s/__MEMPOOL_BACKEND_CACHE_DIR__/${__MEMPOOL_BACKEND_CACHE_DIR__}/g" mempool-config.json sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT_}/g" mempool-config.json -sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__/${__MEMPOOL_BACKEND_MAINNET_HTTP_HOST_}/g" mempool-config.json node /backend/dist/index.js diff --git a/docker/frontend/entrypoint.sh b/docker/frontend/entrypoint.sh index 359c8b466..7539f74b8 100644 --- a/docker/frontend/entrypoint.sh +++ b/docker/frontend/entrypoint.sh @@ -1,5 +1,5 @@ #!/bin/sh -__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__=${BACKEND_MAINNET_HTTP_HOST} +__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__=${BACKEND_MAINNET_HTTP_HOST:=127.0.0.1} sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__/${__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__}/g" /etc/nginx/conf.d/nginx-mempool.conf diff --git a/docker/init.sh b/docker/init.sh index 437f48f4f..f3f4363f2 100755 --- a/docker/init.sh +++ b/docker/init.sh @@ -15,7 +15,7 @@ sed -i "s/${localhostIP}:/0.0.0.0:80/g" ./frontend/nginx.conf sed -i "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf sed -i "s/user nobody;//g" ./frontend/nginx.conf sed -i "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" ./frontend/nginx.conf -sed -i "s/${localhostIP}:/__MEMPOOL_MAINNET_BACKEND_HOST__:/g" ./frontend/nginx-mempool.conf +sed -i "s/${localhostIP}:/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:/g" ./frontend/nginx-mempool.conf #db cp -fr docker/mysql . From 2d872bda4784130a8d36e859830e7fa212d6dd9d Mon Sep 17 00:00:00 2001 From: Bastien Date: Tue, 2 Feb 2021 18:01:23 +0100 Subject: [PATCH 6/7] Update package.json && Fix start.sh script --- backend/package.json | 1 + docker/backend/start.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/package.json b/backend/package.json index 1ce696cca..1c2907f44 100644 --- a/backend/package.json +++ b/backend/package.json @@ -36,6 +36,7 @@ "locutus": "^2.0.12", "mysql2": "^1.6.1", "node-worker-threads-pool": "^1.4.2", + "typescript": "~3.9.7", "ws": "^7.3.1" }, "devDependencies": { diff --git a/docker/backend/start.sh b/docker/backend/start.sh index b96626f99..cec6573ba 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -15,7 +15,7 @@ __ELECTRS_MAINNET_HTTP_PORT__=${ELECTRS_PORT:=50002} __MYSQL_HOST__=${MYSQL_HOST:=127.0.0.1} __MYSQL_PORT__=${MYSQL_PORT:=3306} -mkdir __MEMPOOL_BACKEND_CACHE_DIR__ +mkdir "${__MEMPOOL_BACKEND_CACHE_DIR__}" sed -i "s/__BITCOIN_MAINNET_RPC_HOST__/${__BITCOIN_MAINNET_RPC_HOST__}/g" mempool-config.json sed -i "s/__BITCOIN_MAINNET_RPC_PORT__/${__BITCOIN_MAINNET_RPC_PORT__}/g" mempool-config.json From a36b5c660db4cda0585ff613ef6f060dd51df0fd Mon Sep 17 00:00:00 2001 From: Bastien Date: Tue, 2 Feb 2021 19:20:05 +0100 Subject: [PATCH 7/7] Fix ENV_VAR issues --- docker/backend/Dockerfile | 1 + docker/backend/start.sh | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 2b42eab91..5db88de53 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -5,6 +5,7 @@ COPY . . RUN apt-get update RUN apt-get install -y build-essential python3 pkg-config RUN npm ci --production +RUN npm i typescript RUN npm run build FROM node:12-buster-slim diff --git a/docker/backend/start.sh b/docker/backend/start.sh index cec6573ba..d758290f7 100644 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -2,7 +2,7 @@ #MEMPOOL __MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT:=8999} -__MEMPOOL_BACKEND_CACHE_DIR__=${CACHE_DIR:=/backend/cache} +__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__=${CACHE_DIR:=./cache} # BITCOIN __BITCOIN_MAINNET_RPC_HOST__=${RPC_HOST:=127.0.0.1} __BITCOIN_MAINNET_RPC_PORT__=${RPC_PORT:=8332} @@ -15,7 +15,7 @@ __ELECTRS_MAINNET_HTTP_PORT__=${ELECTRS_PORT:=50002} __MYSQL_HOST__=${MYSQL_HOST:=127.0.0.1} __MYSQL_PORT__=${MYSQL_PORT:=3306} -mkdir "${__MEMPOOL_BACKEND_CACHE_DIR__}" +mkdir -p "${__MEMPOOL_BACKEND_CACHE_DIR__}" sed -i "s/__BITCOIN_MAINNET_RPC_HOST__/${__BITCOIN_MAINNET_RPC_HOST__}/g" mempool-config.json sed -i "s/__BITCOIN_MAINNET_RPC_PORT__/${__BITCOIN_MAINNET_RPC_PORT__}/g" mempool-config.json @@ -25,7 +25,7 @@ sed -i "s/__ELECTRS_MAINNET_HTTP_HOST__/${__ELECTRS_MAINNET_HTTP_HOST__}/g" memp sed -i "s/__ELECTRS_MAINNET_HTTP_PORT__/${__ELECTRS_MAINNET_HTTP_PORT__}/g" mempool-config.json sed -i "s/__MYSQL_HOST__/${__MYSQL_HOST__}/g" mempool-config.json sed -i "s/__MYSQL_PORT__/${__MYSQL_PORT__}/g" mempool-config.json -sed -i "s/__MEMPOOL_BACKEND_CACHE_DIR__/${__MEMPOOL_BACKEND_CACHE_DIR__}/g" mempool-config.json -sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT_}/g" mempool-config.json +sed -i "s!__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__!${__MEMPOOL_BACKEND_MAINNET_CACHE_DIR__}!g" mempool-config.json +sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__}/g" mempool-config.json node /backend/dist/index.js