diff --git a/production/mempool-build-all b/production/mempool-build-all index d44569fe2..ede944546 100755 --- a/production/mempool-build-all +++ b/production/mempool-build-all @@ -16,10 +16,13 @@ if [ -f "${LOCKFILE}" ];then exit 1 fi -trap "rv=\$?; rm -rf "${LOCKFILE}"; exit \$rv" INT TERM EXIT +# on exit, remove lockfile but preserve exit code +trap "rv=\$?; rm -f "${LOCKFILE}"; exit \$rv" INT TERM EXIT +# create lockfile touch "${LOCKFILE}" +# notify logged in users echo "Upgrading mempool to ${REF}" | wall update_repo() @@ -84,25 +87,48 @@ ship_frontend() rsync -av "./dist/mempool/browser/" "${HOME}/public_html/${site}/" || exit 1 } +# load nvm if necessary export NVM_DIR="${HOME}/.nvm" source "${NVM_DIR}/nvm.sh" -for target in mainnet testnet signet liquid liquidtestnet bisq;do - update_repo "${target}" +# what to look for +frontends=(mainnet liquid bisq) +backends=(mainnet testnet signet liquid liquidtestnet bisq) +frontend_repos=() +backend_repos=() + +# find which frontend repos we have +for repo in $frontends;do + [ -d "${repo}" ] && frontend_repos+="${repo}" done -for target in mainnet testnet signet liquid liquidtestnet bisq;do - build_backend "${target}" +# find which backend repos we have +for repo in $backends;do + [ -d "${repo}" ] && backend_repos+="${repo}" + [ -d "${repo}-lightning" ] && backend_repos+="${repo}-lightning" done -for target in mainnet liquid bisq;do - build_frontend "${target}" +# update all repos +for repo in $backend_repos;do + update_repo "${repo}" done +# build backends +for repo in $backend_repos;do + build_backend "${repo}" +done + +# build frontends +for repo in $frontend_repos;do + build_frontend "${repo}" +done + +# ship frontend dist folders to public_html for target in mainnet liquid bisq;do ship_frontend "${target}" done +# notify everyone echo "${HOSTNAME} updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general mempool.dev echo "${HOSTNAME} updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general "mempool.ops.${LOCATION}" diff --git a/production/mempool-start-all b/production/mempool-start-all index 5d5199166..ebb17f45b 100755 --- a/production/mempool-start-all +++ b/production/mempool-start-all @@ -1,8 +1,8 @@ #!/usr/local/bin/zsh export NVM_DIR="$HOME/.nvm" source "$NVM_DIR/nvm.sh" -for site in mainnet liquid testnet bisq signet liquidtestnet -do + +for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-lightning bisq liquid liquidtestnet;do cd "${HOME}/${site}/backend/" && \ screen -dmS "${site}" sh -c 'while true;do npm run start-production;sleep 1;done' done diff --git a/production/nginx/http-basic.conf b/production/nginx/http-basic.conf index a79c73382..5d707ca42 100644 --- a/production/nginx/http-basic.conf +++ b/production/nginx/http-basic.conf @@ -4,8 +4,6 @@ tcp_nopush on; tcp_nodelay on; server_tokens off; server_name_in_redirect off; -include /usr/local/etc/nginx/mime.types; -default_type application/octet-stream; # default logs access_log /var/log/nginx/access.log; diff --git a/production/nginx/location-api-v1-lightning.conf b/production/nginx/location-api-v1-lightning.conf new file mode 100644 index 000000000..288fe4184 --- /dev/null +++ b/production/nginx/location-api-v1-lightning.conf @@ -0,0 +1,20 @@ +# route lightning API endpoints to lightning backend +location /api/v1/lightning { + try_files /dev/null @mempool-api-v1-lightning; +} +location @mempool-api-v1-lightning { + proxy_pass $mempoolMainnetLightning; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 10s; + proxy_redirect off; + + expires 10s; +} diff --git a/production/nginx/location-api.conf b/production/nginx/location-api.conf index 886681f5c..2b5cb0152 100644 --- a/production/nginx/location-api.conf +++ b/production/nginx/location-api.conf @@ -1,26 +1,51 @@ -location /api/v1/statistics { - try_files /dev/null @mempool-api-v1-warmcache; -} -location /api/v1/mining { - try_files /dev/null @mempool-api-v1-warmcache; -} -location /api/v1/block/ { - try_files /dev/null @mempool-api-v1-forevercache; -} -location /api/v1 { - try_files /dev/null @mempool-api-v1-coldcache; -} -location /api/block/ { - rewrite ^/api/(.*) /$1 break; - try_files /dev/null @electrs-api-forevercache; -} -location /api/ { - rewrite ^/api/(.*) /$1 break; - try_files /dev/null @electrs-api-nocache; +########### +# mempool # +########### + +# websocket has special HTTP headers +location /api/v1/ws { + try_files /dev/null @mempool-api-v1-websocket; } -location @mempool-api-v1-forevercache { - proxy_pass $mempoolBackend; +# warm cache mining and mempool API responses +location /api/v1/statistics { + try_files /dev/null @mempool-api-v1-cache-warm; +} +location /api/v1/mining { + try_files /dev/null @mempool-api-v1-cache-warm; +} + +# it's ok to cache blockchain data "forever", so we do 30d +location /api/v1/block/ { + try_files /dev/null @mempool-api-v1-cache-forever; +} + +# everything else gets "normal" cache +location /api/v1 { + try_files /dev/null @mempool-api-v1-cache-normal; +} + +########### +# esplora # +########### + +# it's ok to cache blockchain data "forever", so we do 30d +location /api/block/ { + rewrite ^/api/(.*) /$1 break; + try_files /dev/null @esplora-api-cache-forever; +} +# other API responses cannot be cached +location /api/ { + rewrite ^/api/(.*) /$1 break; + try_files /dev/null @esplora-api-cache-disabled; +} + +########### +# routing # +########### + +location @mempool-api-v1-websocket { + proxy_pass $mempoolMainnet; proxy_http_version 1.1; proxy_set_header Host $http_host; @@ -29,8 +54,16 @@ location @mempool-api-v1-forevercache { proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; proxy_set_header X-Forwarded-Proto $scheme; +} + +location @mempool-api-v1-cache-forever { + proxy_pass $mempoolMainnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache_bypass $http_upgrade; proxy_cache_background_update on; proxy_cache_use_stale updating; proxy_cache api; @@ -40,18 +73,14 @@ location @mempool-api-v1-forevercache { expires 30d; } -location @mempool-api-v1-warmcache { - proxy_pass $mempoolBackend; - proxy_http_version 1.1; +location @mempool-api-v1-cache-warm { + proxy_pass $mempoolMainnet; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache_bypass $http_upgrade; proxy_cache_background_update on; proxy_cache_use_stale updating; proxy_cache api; @@ -59,18 +88,14 @@ location @mempool-api-v1-warmcache { proxy_redirect off; } -location @mempool-api-v1-coldcache { - proxy_pass $mempoolBackend; - proxy_http_version 1.1; +location @mempool-api-v1-cache-normal { + proxy_pass $mempoolMainnet; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache_bypass $http_upgrade; proxy_cache api; proxy_cache_valid 200 10s; proxy_redirect off; @@ -78,54 +103,42 @@ location @mempool-api-v1-coldcache { expires 10s; } -location @mempool-api-v1-nocache { - proxy_pass $mempoolBackend; - proxy_http_version 1.1; +location @mempool-api-v1-cache-disabled { + proxy_pass $mempoolMainnet; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache_bypass $http_upgrade; proxy_redirect off; proxy_buffering off; expires -1; } -location @electrs-api-nocache { - proxy_pass $electrsBackend; - proxy_http_version 1.1; +location @esplora-api-cache-disabled { + proxy_pass $esploraMainnet; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache_bypass $http_upgrade; proxy_redirect off; proxy_buffering off; expires -1; } -location @electrs-api-forevercache { - proxy_pass $electrsBackend; - proxy_http_version 1.1; +location @esplora-api-cache-forever { + proxy_pass $esploraMainnet; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache_bypass $http_upgrade; proxy_cache_background_update on; proxy_cache_use_stale updating; proxy_cache api; diff --git a/production/nginx/location-liquid-api.conf b/production/nginx/location-liquid-api.conf index 7c2f927a6..26ffffc70 100644 --- a/production/nginx/location-liquid-api.conf +++ b/production/nginx/location-liquid-api.conf @@ -1,12 +1,150 @@ +########### +# mempool # +########### + +# websocket has special HTTP headers location /liquid/api/v1/ws { - proxy_pass http://mempool-liquid-mainnet/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; + rewrite ^/liquid/(.*) /$1 break; + try_files /dev/null @mempool-liquid-api-v1-websocket; } + +# warm cache mempool API responses +location /liquid/api/v1/statistics { + rewrite ^/liquid/(.*) /$1 break; + try_files /dev/null @mempool-liquid-api-v1-cache-warm; +} + +# it's ok to cache blockchain data "forever", so we do 30d +location /liquid/api/v1/block/ { + rewrite ^/liquid/(.*) /$1 break; + try_files /dev/null @mempool-liquid-api-v1-cache-forever; +} + +# everything else gets "normal" cache location /liquid/api/v1 { - proxy_pass http://mempool-liquid-mainnet/api/v1; + rewrite ^/liquid/(.*) /$1 break; + try_files /dev/null @mempool-liquid-api-v1-cache-normal; } + +########### +# esplora # +########### + +# it's ok to cache blockchain data "forever", so we do 30d +location /liquid/api/block/ { + rewrite ^/liquid/api/(.*) /$1 break; + try_files /dev/null @esplora-liquid-api-cache-forever; +} +# other API responses cannot be cached location /liquid/api/ { - proxy_pass http://electrs-liquid-mainnet/; + rewrite ^/liquid/api/(.*) /$1 break; + try_files /dev/null @esplora-liquid-api-cache-disabled; +} + +########### +# routing # +########### + +location @mempool-liquid-api-v1-websocket { + proxy_pass $mempoolMainnet; + proxy_http_version 1.1; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location @mempool-liquid-api-v1-cache-forever { + proxy_pass $mempoolMainnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 30d; + proxy_redirect off; + + expires 30d; +} + +location @mempool-liquid-api-v1-cache-warm { + proxy_pass $mempoolMainnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 10s; + proxy_redirect off; +} + +location @mempool-liquid-api-v1-cache-normal { + proxy_pass $mempoolMainnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache api; + proxy_cache_valid 200 10s; + proxy_redirect off; + + expires 10s; +} + +location @mempool-liquid-api-v1-cache-disabled { + proxy_pass $mempoolMainnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_redirect off; + proxy_buffering off; + + expires -1; +} + +location @esplora-liquid-api-cache-disabled { + proxy_pass $esploraMainnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_redirect off; + proxy_buffering off; + + expires -1; +} + +location @esplora-liquid-api-cache-forever { + proxy_pass $esploraMainnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 30d; + proxy_redirect off; + + expires 30d; } diff --git a/production/nginx/location-liquidtestnet-api.conf b/production/nginx/location-liquidtestnet-api.conf index 8b03c6fc4..311a6c317 100644 --- a/production/nginx/location-liquidtestnet-api.conf +++ b/production/nginx/location-liquidtestnet-api.conf @@ -1,12 +1,154 @@ +########### +# mempool # +########### + +# websocket has special HTTP headers location /liquidtestnet/api/v1/ws { - proxy_pass http://mempool-liquid-testnet/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; + rewrite ^/liquidtestnet/(.*) /$1 break; + try_files /dev/null @mempool-liquidtestnet-api-v1-websocket; } + +# warm cache mining and mempool API responses +location /liquidtestnet/api/v1/statistics { + rewrite ^/liquidtestnet/(.*) /$1 break; + try_files /dev/null @mempool-liquidtestnet-api-v1-cache-warm; +} +location /liquidtestnet/api/v1/mining { + rewrite ^/liquidtestnet/(.*) /$1 break; + try_files /dev/null @mempool-liquidtestnet-api-v1-cache-warm; +} + +# it's ok to cache blockchain data "forever", so we do 30d +location /liquidtestnet/api/v1/block/ { + rewrite ^/liquidtestnet/(.*) /$1 break; + try_files /dev/null @mempool-liquidtestnet-api-v1-cache-forever; +} + +# everything else gets "normal" cache location /liquidtestnet/api/v1 { - proxy_pass http://mempool-liquid-testnet/api/v1; + rewrite ^/liquidtestnet/(.*) /$1 break; + try_files /dev/null @mempool-liquidtestnet-api-v1-cache-normal; } + +########### +# esplora # +########### + +# it's ok to cache blockchain data "forever", so we do 30d +location /liquidtestnet/api/block/ { + rewrite ^/liquidtestnet/api/(.*) /$1 break; + try_files /dev/null @esplora-liquidtestnet-api-cache-forever; +} +# other API responses cannot be cached location /liquidtestnet/api/ { - proxy_pass http://electrs-liquid-testnet/; + rewrite ^/liquidtestnet/api/(.*) /$1 break; + try_files /dev/null @esplora-liquidtestnet-api-cache-disabled; +} + +########### +# routing # +########### + +location @mempool-liquidtestnet-api-v1-websocket { + proxy_pass $mempoolTestnet; + proxy_http_version 1.1; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location @mempool-liquidtestnet-api-v1-cache-forever { + proxy_pass $mempoolTestnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 30d; + proxy_redirect off; + + expires 30d; +} + +location @mempool-liquidtestnet-api-v1-cache-warm { + proxy_pass $mempoolTestnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 10s; + proxy_redirect off; +} + +location @mempool-liquidtestnet-api-v1-cache-normal { + proxy_pass $mempoolTestnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache api; + proxy_cache_valid 200 10s; + proxy_redirect off; + + expires 10s; +} + +location @mempool-liquidtestnet-api-v1-cache-disabled { + proxy_pass $mempoolTestnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_redirect off; + proxy_buffering off; + + expires -1; +} + +location @esplora-liquidtestnet-api-cache-disabled { + proxy_pass $esploraTestnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_redirect off; + proxy_buffering off; + + expires -1; +} + +location @esplora-liquidtestnet-api-cache-forever { + proxy_pass $esploraTestnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 30d; + proxy_redirect off; + + expires 30d; } diff --git a/production/nginx/location-signet-api-v1-lightning.conf b/production/nginx/location-signet-api-v1-lightning.conf new file mode 100644 index 000000000..eb25c9820 --- /dev/null +++ b/production/nginx/location-signet-api-v1-lightning.conf @@ -0,0 +1,21 @@ +# route lightning API endpoints to lightning backend +location /signet/api/v1/lightning { + rewrite ^/signet/(.*) /$1 break; + try_files /dev/null @mempool-signet-api-v1-lightning; +} +location @mempool-signet-api-v1-lightning { + proxy_pass $mempoolSignetLightning; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 10s; + proxy_redirect off; + + expires 10s; +} diff --git a/production/nginx/location-signet-api.conf b/production/nginx/location-signet-api.conf index 245979229..262dea48e 100644 --- a/production/nginx/location-signet-api.conf +++ b/production/nginx/location-signet-api.conf @@ -1,12 +1,154 @@ +########### +# mempool # +########### + +# websocket has special HTTP headers location /signet/api/v1/ws { - proxy_pass http://mempool-bitcoin-signet/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; + rewrite ^/signet/(.*) /$1 break; + try_files /dev/null @mempool-signet-api-v1-websocket; } + +# warm cache mining and mempool API responses +location /signet/api/v1/statistics { + rewrite ^/signet/(.*) /$1 break; + try_files /dev/null @mempool-signet-api-v1-cache-warm; +} +location /signet/api/v1/mining { + rewrite ^/signet/(.*) /$1 break; + try_files /dev/null @mempool-signet-api-v1-cache-warm; +} + +# it's ok to cache blockchain data "forever", so we do 30d +location /signet/api/v1/block/ { + rewrite ^/signet/(.*) /$1 break; + try_files /dev/null @mempool-signet-api-v1-cache-forever; +} + +# everything else gets "normal" cache location /signet/api/v1 { - proxy_pass http://mempool-bitcoin-signet/api/v1; + rewrite ^/signet/(.*) /$1 break; + try_files /dev/null @mempool-signet-api-v1-cache-normal; } + +########### +# esplora # +########### + +# it's ok to cache blockchain data "forever", so we do 30d +location /signet/api/block/ { + rewrite ^/signet/api/(.*) /$1 break; + try_files /dev/null @esplora-signet-api-cache-forever; +} +# other API responses cannot be cached location /signet/api/ { - proxy_pass http://electrs-bitcoin-signet/; + rewrite ^/signet/api/(.*) /$1 break; + try_files /dev/null @esplora-signet-api-cache-disabled; +} + +########### +# routing # +########### + +location @mempool-signet-api-v1-websocket { + proxy_pass $mempoolSignet; + proxy_http_version 1.1; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location @mempool-signet-api-v1-cache-forever { + proxy_pass $mempoolSignet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 30d; + proxy_redirect off; + + expires 30d; +} + +location @mempool-signet-api-v1-cache-warm { + proxy_pass $mempoolSignet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 10s; + proxy_redirect off; +} + +location @mempool-signet-api-v1-cache-normal { + proxy_pass $mempoolSignet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache api; + proxy_cache_valid 200 10s; + proxy_redirect off; + + expires 10s; +} + +location @mempool-signet-api-v1-cache-disabled { + proxy_pass $mempoolSignet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_redirect off; + proxy_buffering off; + + expires -1; +} + +location @esplora-signet-api-cache-disabled { + proxy_pass $esploraSignet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_redirect off; + proxy_buffering off; + + expires -1; +} + +location @esplora-signet-api-cache-forever { + proxy_pass $esploraSignet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 30d; + proxy_redirect off; + + expires 30d; } diff --git a/production/nginx/location-testnet-api-v1-lightning.conf b/production/nginx/location-testnet-api-v1-lightning.conf new file mode 100644 index 000000000..dbabc9aef --- /dev/null +++ b/production/nginx/location-testnet-api-v1-lightning.conf @@ -0,0 +1,21 @@ +# route lightning API endpoints to lightning backend +location /testnet/api/v1/lightning { + rewrite ^/testnet/(.*) /$1 break; + try_files /dev/null @mempool-testnet-api-v1-lightning; +} +location @mempool-testnet-api-v1-lightning { + proxy_pass $mempoolSignetLightning; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 10s; + proxy_redirect off; + + expires 10s; +} diff --git a/production/nginx/location-testnet-api.conf b/production/nginx/location-testnet-api.conf index 857b1b957..1f74aa533 100644 --- a/production/nginx/location-testnet-api.conf +++ b/production/nginx/location-testnet-api.conf @@ -1,12 +1,154 @@ +########### +# mempool # +########### + +# websocket has special HTTP headers location /testnet/api/v1/ws { - proxy_pass http://mempool-bitcoin-testnet/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; + rewrite ^/testnet/(.*) /$1 break; + try_files /dev/null @mempool-testnet-api-v1-websocket; } + +# warm cache mining and mempool API responses +location /testnet/api/v1/statistics { + rewrite ^/testnet/(.*) /$1 break; + try_files /dev/null @mempool-testnet-api-v1-cache-warm; +} +location /testnet/api/v1/mining { + rewrite ^/testnet/(.*) /$1 break; + try_files /dev/null @mempool-testnet-api-v1-cache-warm; +} + +# it's ok to cache blockchain data "forever", so we do 30d +location /testnet/api/v1/block/ { + rewrite ^/testnet/(.*) /$1 break; + try_files /dev/null @mempool-testnet-api-v1-cache-forever; +} + +# everything else gets "normal" cache location /testnet/api/v1 { - proxy_pass http://mempool-bitcoin-testnet/api/v1; + rewrite ^/testnet/(.*) /$1 break; + try_files /dev/null @mempool-testnet-api-v1-cache-normal; } + +########### +# esplora # +########### + +# it's ok to cache blockchain data "forever", so we do 30d +location /testnet/api/block/ { + rewrite ^/testnet/api/(.*) /$1 break; + try_files /dev/null @esplora-testnet-api-cache-forever; +} +# other API responses cannot be cached location /testnet/api/ { - proxy_pass http://electrs-bitcoin-testnet/; + rewrite ^/testnet/api/(.*) /$1 break; + try_files /dev/null @esplora-testnet-api-cache-disabled; +} + +########### +# routing # +########### + +location @mempool-testnet-api-v1-websocket { + proxy_pass $mempoolTestnet; + proxy_http_version 1.1; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; +} + +location @mempool-testnet-api-v1-cache-forever { + proxy_pass $mempoolTestnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 30d; + proxy_redirect off; + + expires 30d; +} + +location @mempool-testnet-api-v1-cache-warm { + proxy_pass $mempoolTestnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 10s; + proxy_redirect off; +} + +location @mempool-testnet-api-v1-cache-normal { + proxy_pass $mempoolTestnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache api; + proxy_cache_valid 200 10s; + proxy_redirect off; + + expires 10s; +} + +location @mempool-testnet-api-v1-cache-disabled { + proxy_pass $mempoolTestnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_redirect off; + proxy_buffering off; + + expires -1; +} + +location @esplora-testnet-api-cache-disabled { + proxy_pass $esploraTestnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_redirect off; + proxy_buffering off; + + expires -1; +} + +location @esplora-testnet-api-cache-forever { + proxy_pass $esploraTestnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache api; + proxy_cache_valid 200 30d; + proxy_redirect off; + + expires 30d; } diff --git a/production/nginx/nginx.conf b/production/nginx/nginx.conf index ed0d35e52..24f389533 100644 --- a/production/nginx/nginx.conf +++ b/production/nginx/nginx.conf @@ -1,4 +1,3 @@ -# FreeBSD configuration user nobody; pid /var/run/nginx.pid; @@ -14,17 +13,21 @@ http { # DNS servers for on-demand recursive resolver resolver 8.8.8.8; + # include default mime types + include /usr/local/etc/nginx/mime.types; + default_type application/octet-stream; + # HTTP basic configuration include mempool/production/nginx/http-basic.conf; include mempool/production/nginx/http-proxy-cache.conf; include mempool/production/nginx/http-language.conf; - # mempool backend configuration + # mempool configuration include mempool/production/nginx/upstream-mempool.conf; - # electrs backend configuration - include mempool/production/nginx/upstream-electrs.conf; - include mempool/production/nginx/server-electrs.conf; + # esplora configuration + include mempool/production/nginx/upstream-esplora.conf; + include mempool/production/nginx/server-esplora.conf; # MEMPOOL.NINJA server { @@ -36,11 +39,18 @@ http { # for services from mempool.space like contributors on about page set $mempoolSpaceServices "https://mempool.space"; - # for mempool/backend daemon, see upstream-mempool.conf - set $mempoolBackend "http://mempool-bitcoin-mainnet"; + # for mempool daemons, see upstream-mempool.conf + set $mempoolMainnet "http://mempool-bitcoin-mainnet"; + set $mempoolMainnetLightning "http://mempool-bitcoin-mainnet-lightning"; + set $mempoolTestnet "http://mempool-bitcoin-testnet"; + set $mempoolTestnetLightning "http://mempool-bitcoin-testnet-lightning"; + set $mempoolSignet "http://mempool-bitcoin-signet"; + set $mempoolSignetLightning "http://mempool-bitcoin-signet-lightning"; - # for blockstream/electrs daemon, see upstream-electrs.conf - set $electrsBackend "http://electrs-bitcoin-mainnet"; + # for blockstream/esplora daemons, see upstream-esplora.conf + set $esploraMainnet "http://esplora-bitcoin-mainnet"; + set $esploraTestnet "http://esplora-bitcoin-testnet"; + set $esploraSignet "http://esplora-bitcoin-signet"; # tor v3 listen 127.0.0.1:81; @@ -70,11 +80,11 @@ http { # for services from mempool.space like contributors on about page set $mempoolSpaceServices "https://mempool.space"; - # for mempool/backend daemon, see upstream-mempool.conf - set $mempoolBackend "http://mempool-bisq-mainnet"; + # for mempool daemons, see upstream-mempool.conf + set $mempoolBisq "http://mempool-bitcoin-bisq"; - # for blockstream/electrs daemon, see upstream-electrs.conf - set $electrsBackend "http://electrs-bitcoin-mainnet"; + # for blockstream/esplora daemon, see upstream-esplora.conf + set $esploraMainnet "http://esplora-bitcoin-mainnet"; # tor v3 listen 127.0.0.1:82; @@ -104,11 +114,13 @@ http { # for services from mempool.space like contributors on about page set $mempoolSpaceServices "https://mempool.space"; - # for mempool/backend daemon, see upstream-mempool.conf - set $mempoolBackend "http://mempool-liquid-mainnet"; + # for mempool daemons, see upstream-mempool.conf + set $mempoolMainnet "http://mempool-liquid-mainnet"; + set $mempoolTestnet "http://mempool-liquid-testnet"; - # for blockstream/electrs daemon, see upstream-electrs.conf - set $electrsBackend "http://electrs-liquid-mainnet"; + # for blockstream/esplora daemon, see upstream-esplora.conf + set $esploraMainnet "http://esplora-liquid-mainnet"; + set $esploraTestnet "http://esplora-liquid-testnet"; # tor v3 listen 127.0.0.1:83; diff --git a/production/nginx/server-bisq.conf b/production/nginx/server-bisq.conf index 3933b6e4d..704ef5a6c 100644 --- a/production/nginx/server-bisq.conf +++ b/production/nginx/server-bisq.conf @@ -4,80 +4,98 @@ include mempool/production/nginx/location-api-v1-services.conf; proxy_cache markets; proxy_cache_valid 200 30s; -# route electrs APIs to electrs +# route esplora APIs to esplora location /api/tx/ { - proxy_pass http://electrs-bitcoin-mainnet/tx/; + rewrite ^/api/(.*) /$1 break; + try_files /dev/null @esplora-api-cache-disabled; } # rewrite APIs to match what backend expects location /api/currencies { rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break; - try_files $uri $uri/ @mempool-bisq; + try_files /dev/null @mempool-bisq; } location /api/depth { rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break; - try_files $uri $uri/ @mempool-bisq; + try_files /dev/null @mempool-bisq; } location /api/hloc { rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break; - try_files $uri $uri/ @mempool-bisq; + try_files /dev/null @mempool-bisq; } location /api/offers { rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break; - try_files $uri $uri/ @mempool-bisq; + try_files /dev/null @mempool-bisq; } location /api/ticker { rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break; - try_files $uri $uri/ @mempool-bisq; + try_files /dev/null @mempool-bisq; } location /api/trades { rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break; - try_files $uri $uri/ @mempool-bisq; + try_files /dev/null @mempool-bisq; } location /api/volumes { rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break; - try_files $uri $uri/ @mempool-bisq; + try_files /dev/null @mempool-bisq; } location /api/markets { rewrite ^/api/(.*) /api/v1/bisq/markets/$1 break; - try_files $uri $uri/ @mempool-bisq; + try_files /dev/null @mempool-bisq; } location /api/v1 { rewrite ^/api/v1/(.*) /api/v1/bisq/$1 break; - try_files $uri $uri/ @mempool-bisq; + try_files /dev/null @mempool-bisq; } location /api { rewrite ^/api/(.*) /api/v1/bisq/$1 break; - try_files $uri $uri/ @mempool-bisq; + try_files /dev/null @mempool-bisq; +} +location /api/v1/ws { + rewrite ^/api/(.*) /api/v1/bisq/$1 break; + try_files /dev/null @mempool-bisq-websocket; +} +location /bisq/api/v1/ws { + rewrite ^/bisq/api/v1/(.*) /api/v1/bisq/$1 break; + try_files /dev/null @mempool-bisq-websocket; } location /bisq/api/v1 { rewrite ^/bisq/api/v1/(.*) /api/v1/bisq/$1 break; - try_files $uri $uri/ @mempool-bisq; + try_files /dev/null @mempool-bisq; } location /bisq/api { - rewrite ^/bisq/api/(.*) /api/v1/bisq/$1 break; - try_files $uri $uri/ @mempool-bisq; + rewrite ^/bisq/api/(.*) /api/v1/bisq/$1 break; + try_files /dev/null @mempool-bisq; } -# special handling for websocket -location /api/v1/ws { - proxy_pass http://mempool-bitcoin-bisq/; +location @mempool-bisq-websocket { + proxy_pass $mempoolBisq; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; } location @mempool-bisq { - proxy_pass http://mempool-bitcoin-bisq; - proxy_http_version 1.1; + proxy_pass $mempoolBisq; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - - proxy_cache_bypass $http_upgrade; - proxy_redirect off; proxy_set_header X-Forwarded-Proto $scheme; + + proxy_redirect off; +} + +location @esplora-api-cache-disabled { + proxy_pass $esploraMainnet; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_redirect off; + proxy_buffering off; + + expires -1; } diff --git a/production/nginx/server-electrs.conf b/production/nginx/server-esplora.conf similarity index 62% rename from production/nginx/server-electrs.conf rename to production/nginx/server-esplora.conf index b817b8163..38cdb0bc2 100644 --- a/production/nginx/server-electrs.conf +++ b/production/nginx/server-esplora.conf @@ -2,34 +2,34 @@ server { listen 127.0.0.1:4000; access_log /dev/null; location / { - proxy_pass http://electrs-bitcoin-mainnet; + proxy_pass http://esplora-bitcoin-mainnet; } } server { listen 127.0.0.1:4001; access_log /dev/null; location / { - proxy_pass http://electrs-liquid-mainnet; + proxy_pass http://esplora-liquid-mainnet; } } server { listen 127.0.0.1:4002; access_log /dev/null; location / { - proxy_pass http://electrs-bitcoin-testnet; + proxy_pass http://esplora-bitcoin-testnet; } } server { listen 127.0.0.1:4003; access_log /dev/null; location / { - proxy_pass http://electrs-bitcoin-signet; + proxy_pass http://esplora-bitcoin-signet; } } server { listen 127.0.0.1:4004; access_log /dev/null; location / { - proxy_pass http://electrs-liquid-testnet; + proxy_pass http://esplora-liquid-testnet; } } diff --git a/production/nginx/server-mempool.conf b/production/nginx/server-mempool.conf index 5a84cce17..892fe50e1 100644 --- a/production/nginx/server-mempool.conf +++ b/production/nginx/server-mempool.conf @@ -1,6 +1,9 @@ include mempool/production/nginx/server-common.conf; include mempool/production/nginx/location-redirects.conf; include mempool/production/nginx/location-api-v1-services.conf; +include mempool/production/nginx/location-api-v1-lightning.conf; include mempool/production/nginx/location-api.conf; include mempool/production/nginx/location-testnet-api.conf; +include mempool/production/nginx/location-testnet-api-v1-lightning.conf; include mempool/production/nginx/location-signet-api.conf; +include mempool/production/nginx/location-signet-api-v1-lightning.conf; diff --git a/production/nginx/upstream-electrs.conf b/production/nginx/upstream-esplora.conf similarity index 65% rename from production/nginx/upstream-electrs.conf rename to production/nginx/upstream-esplora.conf index 476b73054..6cad0730b 100644 --- a/production/nginx/upstream-electrs.conf +++ b/production/nginx/upstream-esplora.conf @@ -1,15 +1,15 @@ -upstream electrs-bitcoin-mainnet { +upstream esplora-bitcoin-mainnet { server [::1]:3000 fail_timeout=10s max_fails=10 weight=99999; } -upstream electrs-liquid-mainnet { +upstream esplora-liquid-mainnet { server [::1]:3001 fail_timeout=10s max_fails=10 weight=99999; } -upstream electrs-bitcoin-testnet { +upstream esplora-bitcoin-testnet { server [::1]:3002 fail_timeout=10s max_fails=10 weight=99999; } -upstream electrs-bitcoin-signet { +upstream esplora-bitcoin-signet { server [::1]:3003 fail_timeout=10s max_fails=10 weight=99999; } -upstream electrs-liquid-testnet { +upstream esplora-liquid-testnet { server [::1]:3004 fail_timeout=10s max_fails=10 weight=99999; } diff --git a/production/nginx/upstream-mempool.conf b/production/nginx/upstream-mempool.conf index 8e0f12ab6..c9cf9c2bb 100644 --- a/production/nginx/upstream-mempool.conf +++ b/production/nginx/upstream-mempool.conf @@ -16,3 +16,12 @@ upstream mempool-bitcoin-signet { upstream mempool-liquid-testnet { server 127.0.0.1:8994 fail_timeout=10s max_fails=10 weight=99999; } +upstream mempool-bitcoin-mainnet-lightning { + server 127.0.0.1:8993 fail_timeout=10s max_fails=10 weight=99999; +} +upstream mempool-bitcoin-testnet-lightning { + server 127.0.0.1:8992 fail_timeout=10s max_fails=10 weight=99999; +} +upstream mempool-bitcoin-signet-lightning { + server 127.0.0.1:8991 fail_timeout=10s max_fails=10 weight=99999; +}