From 99f7201ec5a9f904f448cd7337d1bca69bd7bf73 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Mon, 22 Jan 2024 16:50:50 -0800 Subject: [PATCH 1/4] Cache build assets to prevent rate limiting --- .github/workflows/ci.yml | 233 ++++++++++++++++++++++++++++++++++ .github/workflows/cypress.yml | 64 ---------- frontend/sync-assets.js | 20 +-- 3 files changed, 243 insertions(+), 74 deletions(-) delete mode 100644 .github/workflows/cypress.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0602e916a..d7f8a7b6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,108 @@ jobs: run: npm run build working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend + + cache: + name: "Cache assets for builds" + runs-on: "ubuntu-latest" + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + path: assets + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node }} + registry-url: "https://registry.npmjs.org" + + - name: Install (Prod dependencies only) + run: npm ci --omit=dev --omit=optional + working-directory: assets/frontend + + - name: Restore cached mining pool assets + continue-on-error: true + id: cache-mining-pool-restore + uses: actions/cache/restore@v4 + with: + path: | + mining-pool-assets.zip + key: mining-pool-assets-cache + + - name: Restore promo video assets + continue-on-error: true + id: cache-promo-video-restore + uses: actions/cache/restore@v4 + with: + path: | + promo-video-assets.zip + key: promo-video-assets-cache + + # - name: Download mining pool artifact + # uses: actions/download-artifact@v4 + # continue-on-error: true + # with: + # name: mining-pool-assets + + - name: Unzip assets before building (src/resources) + continue-on-error: true + run: unzip -o mining-pool-assets.zip -d assets/frontend/src/resources/mining-pools + + - name: Download promo video artifact + uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: promo-video-assets + + - name: Unzip assets before building (src/resources) + continue-on-error: true + run: unzip -o promo-video-assets.zip -d assets/frontend/src/resources/promo-video + + # - name: Unzip assets before building (dist) + # continue-on-error: true + # run: unzip assets.zip -d assets/frontend/dist/mempool/browser/resources + + - name: Sync-assets + run: npm run sync-assets-dev + working-directory: assets/frontend + + - name: Zip mining-pool assets + run: zip -jrq mining-pool-assets.zip assets/frontend/src/resources/mining-pools/* + + - name: Zip promo-video assets + run: zip -jrq promo-video-assets.zip assets/frontend/src/resources/promo-video/* + + - name: Upload mining pool assets as artifact + uses: actions/upload-artifact@v4 + with: + name: mining-pool-assets + path: mining-pool-assets.zip + + - name: Upload promo video assets as artifact + uses: actions/upload-artifact@v4 + with: + name: promo-video-assets + path: promo-video-assets.zip + + - name: Save mining pool assets cache + id: cache-mining-pool-save + uses: actions/cache/save@v4 + with: + path: | + mining-pool-assets.zip + key: mining-pool-assets-cache + + - name: Save promo video assets cache + id: cache-promo-video-save + uses: actions/cache/save@v4 + with: + path: | + promo-video-assets.zip + key: promo-video-assets-cache + frontend: + needs: cache if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" strategy: matrix: @@ -103,9 +204,141 @@ jobs: # - name: Test # run: npm run test + - name: Restore cached mining pool assets + continue-on-error: true + id: cache-mining-pool-restore + uses: actions/cache/restore@v4 + with: + path: | + mining-pool-assets.zip + key: mining-pool-assets-cache + + - name: Restore promo video assets + continue-on-error: true + id: cache-promo-video-restore + uses: actions/cache/restore@v4 + with: + path: | + promo-video-assets.zip + key: promo-video-assets-cache + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: mining-pool-assets + + - name: Unzip assets before building (src/resources) + run: unzip -o mining-pool-assets.zip -d ${{ matrix.node }}/${{ matrix.flavor }}/frontend/src/resources/mining-pools + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: promo-video-assets + + - name: Unzip assets before building (src/resources) + run: unzip -o promo-video-assets.zip -d ${{ matrix.node }}/${{ matrix.flavor }}/frontend/src/resources/promo-video + + # - name: Unzip assets before building (dist) + # run: unzip assets.zip -d ${{ matrix.node }}/${{ matrix.flavor }}/frontend/dist/mempool/browser/resources + + - name: Display resulting source tree + run: ls -R + - name: Build run: npm run build working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + e2e: + if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" + runs-on: "ubuntu-latest" + needs: frontend + strategy: + fail-fast: false + matrix: + # module: ["mempool", "liquid", "bisq"] Disabling bisq support for now + module: ["mempool", "liquid"] + include: + - module: "mempool" + spec: | + cypress/e2e/mainnet/*.spec.ts + cypress/e2e/signet/*.spec.ts + cypress/e2e/testnet/*.spec.ts + - module: "liquid" + spec: | + cypress/e2e/liquid/liquid.spec.ts + cypress/e2e/liquidtestnet/liquidtestnet.spec.ts + # - module: "bisq" + # spec: | + # cypress/e2e/bisq/bisq.spec.ts + name: E2E tests for ${{ matrix.module }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + path: ${{ matrix.module }} + + - name: Setup node + uses: actions/setup-node@v3 + with: + node-version: 20 + cache: "npm" + cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json + + - name: Restore cached mining pool assets + continue-on-error: true + id: cache-mining-pool-restore + uses: actions/cache/restore@v4 + with: + path: | + mining-pool-assets.zip + key: mining-pool-assets-cache + + - name: Restore cached promo video assets + continue-on-error: true + id: cache-promo-video-restore + uses: actions/cache/restore@v4 + with: + path: | + promo-video-assets.zip + key: promo-video-assets-cache + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: mining-pool-assets + + - name: Unzip assets before building (src/resources) + run: unzip -o mining-pool-assets.zip -d ${{ matrix.module }}/frontend/src/resources/mining-pools + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: promo-video-assets + + - name: Unzip assets before building (src/resources) + run: unzip -o promo-video-assets.zip -d ${{ matrix.module }}/frontend/src/resources/promo-video + + - name: Chrome browser tests (${{ matrix.module }}) + uses: cypress-io/github-action@v5 + with: + tag: ${{ github.event_name }} + working-directory: ${{ matrix.module }}/frontend + build: npm run config:defaults:${{ matrix.module }} + start: npm run start:local-staging + wait-on: "http://localhost:4200" + wait-on-timeout: 120 + record: true + parallel: true + spec: ${{ matrix.spec }} + group: Tests on Chrome (${{ matrix.module }}) + browser: "chrome" + ci-build-id: "${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}" + env: + COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }} + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} + \ No newline at end of file diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml deleted file mode 100644 index f12aebe8b..000000000 --- a/.github/workflows/cypress.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Cypress Tests - -on: - push: - branches: [master] - pull_request: - types: [opened, synchronize] - -jobs: - cypress: - if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" - runs-on: "ubuntu-latest" - strategy: - fail-fast: false - matrix: - module: ["mempool", "liquid", "bisq"] - include: - - module: "mempool" - spec: | - cypress/e2e/mainnet/*.spec.ts - cypress/e2e/signet/*.spec.ts - cypress/e2e/testnet/*.spec.ts - - module: "liquid" - spec: | - cypress/e2e/liquid/liquid.spec.ts - cypress/e2e/liquidtestnet/liquidtestnet.spec.ts - - module: "bisq" - spec: | - cypress/e2e/bisq/bisq.spec.ts - - name: E2E tests for ${{ matrix.module }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - path: ${{ matrix.module }} - - - name: Setup node - uses: actions/setup-node@v3 - with: - node-version: 20 - cache: "npm" - cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json - - - name: Chrome browser tests (${{ matrix.module }}) - uses: cypress-io/github-action@v5 - with: - tag: ${{ github.event_name }} - working-directory: ${{ matrix.module }}/frontend - build: npm run config:defaults:${{ matrix.module }} - start: npm run start:local-staging - wait-on: "http://localhost:4200" - wait-on-timeout: 120 - record: true - parallel: true - spec: ${{ matrix.spec }} - group: Tests on Chrome (${{ matrix.module }}) - browser: "chrome" - ci-build-id: "${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}" - env: - COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }} - CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index a92c3b21d..b15b5e315 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -22,7 +22,7 @@ var PATH; if (process.argv[2]) { PATH = process.argv[2]; PATH += PATH.endsWith("/") ? "" : "/" - PATH = path.normalize(PATH); + PATH = path.resolve(path.normalize(PATH)); console.log(`[sync-assets] using PATH ${PATH}`); if (!fs.existsSync(PATH)){ console.log(`${LOG_TAG} ${PATH} does not exist, creating`); @@ -110,7 +110,7 @@ function downloadMiningPoolLogos$() { } let downloadedCount = 0; for (const poolLogo of poolLogos) { - const filePath = PATH + `mining-pools/${poolLogo.name}`; + const filePath = `${PATH}/mining-pools/${poolLogo.name}`; if (fs.existsSync(filePath)) { const localHash = getLocalHash(filePath); if (verbose) { @@ -124,7 +124,7 @@ function downloadMiningPoolLogos$() { } } else { console.log(`${LOG_TAG} ${poolLogo.name} is missing, downloading...`); - const miningPoolsDir = PATH + `mining-pools/`; + const miningPoolsDir = `${PATH}/mining-pools/`; if (!fs.existsSync(miningPoolsDir)){ fs.mkdirSync(miningPoolsDir, { recursive: true }); } @@ -179,7 +179,7 @@ function downloadPromoVideoSubtiles$() { } let downloadedCount = 0; for (const language of videoLanguages) { - const filePath = PATH + `promo-video/${language.name}`; + const filePath = `${PATH}/promo-video/${language.name}`; if (fs.existsSync(filePath)) { if (verbose) { console.log(`${LOG_TAG} ${language.name} remote promo video hash ${language.sha}`); @@ -193,7 +193,7 @@ function downloadPromoVideoSubtiles$() { } } else { console.log(`${LOG_TAG} ${language.name} is missing, downloading`); - const promoVideosDir = PATH + `promo-video/`; + const promoVideosDir = `${PATH}/promo-video/`; if (!fs.existsSync(promoVideosDir)){ fs.mkdirSync(promoVideosDir, { recursive: true }); } @@ -250,7 +250,7 @@ function downloadPromoVideo$() { if (item.name !== 'promo.mp4') { continue; } - const filePath = PATH + `promo-video/mempool-promo.mp4`; + const filePath = `${PATH}/promo-video/mempool-promo.mp4`; if (fs.existsSync(filePath)) { const localHash = getLocalHash(filePath); @@ -288,16 +288,16 @@ if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') { const testnetAssetsMinimalJsonUrl = 'https://raw.githubusercontent.com/Blockstream/asset_registry_testnet_db/master/index.minimal.json'; console.log(`${LOG_TAG} Downloading assets`); - download(PATH + 'assets.json', assetsJsonUrl); + download(`${PATH}/assets.json`, assetsJsonUrl); console.log(`${LOG_TAG} Downloading assets minimal`); - download(PATH + 'assets.minimal.json', assetsMinimalJsonUrl); + download(`${PATH}/assets.minimal.json`, assetsMinimalJsonUrl); console.log(`${LOG_TAG} Downloading testnet assets`); - download(PATH + 'assets-testnet.json', testnetAssetsJsonUrl); + download(`${PATH}/assets-testnet.json`, testnetAssetsJsonUrl); console.log(`${LOG_TAG} Downloading testnet assets minimal`); - download(PATH + 'assets-testnet.minimal.json', testnetAssetsMinimalJsonUrl); + download(`${PATH}/assets-testnet.minimal.json`, testnetAssetsMinimalJsonUrl); } else { if (verbose) { console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (${configContent.BASE_MODULE}), skipping downloading assets`); From 479007da5512637593e454b5098d823c0f3f8a47 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Mon, 22 Jan 2024 19:51:38 -0800 Subject: [PATCH 2/4] Remove the download artifacts step --- .github/workflows/ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7f8a7b6c..7b1cb1ad9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -222,11 +222,6 @@ jobs: promo-video-assets.zip key: promo-video-assets-cache - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: mining-pool-assets - - name: Unzip assets before building (src/resources) run: unzip -o mining-pool-assets.zip -d ${{ matrix.node }}/${{ matrix.flavor }}/frontend/src/resources/mining-pools From c789797d2d964e436a60fb21f336f3c92184c408 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Mon, 22 Jan 2024 20:05:20 -0800 Subject: [PATCH 3/4] Revert "Remove the download artifacts step" This reverts commit 479007da5512637593e454b5098d823c0f3f8a47. --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b1cb1ad9..d7f8a7b6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -222,6 +222,11 @@ jobs: promo-video-assets.zip key: promo-video-assets-cache + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: mining-pool-assets + - name: Unzip assets before building (src/resources) run: unzip -o mining-pool-assets.zip -d ${{ matrix.node }}/${{ matrix.flavor }}/frontend/src/resources/mining-pools From fe079a72bed459d1999edf75770fe1f0c396cff3 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Mon, 22 Jan 2024 20:06:27 -0800 Subject: [PATCH 4/4] Clean up action --- .github/workflows/ci.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7f8a7b6c..c435d6ea5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,22 +101,10 @@ jobs: promo-video-assets.zip key: promo-video-assets-cache - # - name: Download mining pool artifact - # uses: actions/download-artifact@v4 - # continue-on-error: true - # with: - # name: mining-pool-assets - - name: Unzip assets before building (src/resources) continue-on-error: true run: unzip -o mining-pool-assets.zip -d assets/frontend/src/resources/mining-pools - - name: Download promo video artifact - uses: actions/download-artifact@v4 - continue-on-error: true - with: - name: promo-video-assets - - name: Unzip assets before building (src/resources) continue-on-error: true run: unzip -o promo-video-assets.zip -d assets/frontend/src/resources/promo-video