From eeefaa63744331762095d136bd9d78f3fa384684 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 24 Aug 2023 17:49:27 +0900 Subject: [PATCH] Proof-of-concept "sipper" minimal pages for web crawlers --- unfurler/package-lock.json | 282 ++++++++++++++++++++++++++++++++----- unfurler/package.json | 2 + unfurler/src/config.ts | 10 ++ unfurler/src/index.ts | 35 +++++ unfurler/src/routes.ts | 53 +++++-- unfurler/views/block.ejs | 91 ++++++++++++ unfurler/views/footer.ejs | 130 +++++++++++++++++ unfurler/views/head.ejs | 23 +++ unfurler/views/header.ejs | 79 +++++++++++ 9 files changed, 661 insertions(+), 44 deletions(-) create mode 100644 unfurler/views/block.ejs create mode 100644 unfurler/views/footer.ejs create mode 100644 unfurler/views/head.ejs create mode 100644 unfurler/views/header.ejs diff --git a/unfurler/package-lock.json b/unfurler/package-lock.json index c2ab86486..1232e3716 100644 --- a/unfurler/package-lock.json +++ b/unfurler/package-lock.json @@ -9,7 +9,9 @@ "version": "3.0.0-dev", "dependencies": { "@types/node": "^16.11.41", + "ejs": "^3.1.9", "express": "^4.19.2", + "node-fetch-commonjs": "^3.3.1", "puppeteer": "^15.3.2", "puppeteer-cluster": "^0.23.0", "typescript": "~4.7.4" @@ -374,7 +376,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -405,6 +406,11 @@ "node": ">=8" } }, + "node_modules/async": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -566,7 +572,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -587,7 +592,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -598,8 +602,7 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/concat-map": { "version": "0.0.1", @@ -646,6 +649,25 @@ "node-fetch": "2.6.7" } }, + "node_modules/cross-fetch/node_modules/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -749,6 +771,20 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "node_modules/ejs": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz", + "integrity": "sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==", + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -1154,6 +1190,28 @@ "pend": "~1.2.0" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -1166,6 +1224,33 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "dependencies": { + "minimatch": "^5.0.1" + } + }, + "node_modules/filelist/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/filelist/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -1239,6 +1324,17 @@ "integrity": "sha512-0sQoMh9s0BYsm+12Huy/rkKxVu4R1+r96YX5cG44rHV0pQ6iC3Q+mkoMFaGWObMFYQxCVT+ssG1ksneA2MI9KQ==", "dev": true }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -1392,7 +1488,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, "engines": { "node": ">=8" } @@ -1590,6 +1685,23 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, + "node_modules/jake": { + "version": "10.8.7", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.7.tgz", + "integrity": "sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w==", + "dependencies": { + "async": "^3.2.3", + "chalk": "^4.0.2", + "filelist": "^1.0.4", + "minimatch": "^3.1.2" + }, + "bin": { + "jake": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -1764,23 +1876,38 @@ "node": ">= 0.6" } }, - "node_modules/node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch-commonjs": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/node-fetch-commonjs/-/node-fetch-commonjs-3.3.1.tgz", + "integrity": "sha512-kN5TfyrGRWJOZEAx7GvDu7kjvMKjmt4VmaE3dVeDWwzqzNFYgYroRLvTRJAEQ9/Ka3TumTwDrCZ5u3FJUqFb8A==", "dependencies": { - "whatwg-url": "^5.0.0" + "formdata-polyfill": "^4.0.10", + "web-streams-polyfill": "^3.1.1" }, "engines": { - "node": "4.x || >=6.0.0" + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" } }, "node_modules/object-inspect": { @@ -2391,7 +2518,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "dependencies": { "has-flag": "^4.0.0" }, @@ -2583,6 +2709,14 @@ "node": ">= 0.8" } }, + "node_modules/web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -2890,7 +3024,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "requires": { "color-convert": "^2.0.1" } @@ -2912,6 +3045,11 @@ "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", "dev": true }, + "async": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" + }, "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -3025,7 +3163,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -3040,7 +3177,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "requires": { "color-name": "~1.1.4" } @@ -3048,8 +3184,7 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "concat-map": { "version": "0.0.1", @@ -3085,6 +3220,16 @@ "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", "requires": { "node-fetch": "2.6.7" + }, + "dependencies": { + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + } + } } }, "cross-spawn": { @@ -3160,6 +3305,14 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "ejs": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz", + "integrity": "sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==", + "requires": { + "jake": "^10.8.5" + } + }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -3483,6 +3636,15 @@ "pend": "~1.2.0" } }, + "fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "requires": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + } + }, "file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -3492,6 +3654,32 @@ "flat-cache": "^3.0.4" } }, + "filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "requires": { + "minimatch": "^5.0.1" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, + "minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "requires": { + "brace-expansion": "^2.0.1" + } + } + } + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -3555,6 +3743,14 @@ "integrity": "sha512-0sQoMh9s0BYsm+12Huy/rkKxVu4R1+r96YX5cG44rHV0pQ6iC3Q+mkoMFaGWObMFYQxCVT+ssG1ksneA2MI9KQ==", "dev": true }, + "formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "requires": { + "fetch-blob": "^3.1.2" + } + }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -3662,8 +3858,7 @@ "has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" }, "has-property-descriptors": { "version": "1.0.2", @@ -3793,6 +3988,17 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, + "jake": { + "version": "10.8.7", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.7.tgz", + "integrity": "sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w==", + "requires": { + "async": "^3.2.3", + "chalk": "^4.0.2", + "filelist": "^1.0.4", + "minimatch": "^3.1.2" + } + }, "js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -3925,12 +4131,18 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" }, - "node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==" + }, + "node-fetch-commonjs": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/node-fetch-commonjs/-/node-fetch-commonjs-3.3.1.tgz", + "integrity": "sha512-kN5TfyrGRWJOZEAx7GvDu7kjvMKjmt4VmaE3dVeDWwzqzNFYgYroRLvTRJAEQ9/Ka3TumTwDrCZ5u3FJUqFb8A==", "requires": { - "whatwg-url": "^5.0.0" + "formdata-polyfill": "^4.0.10", + "web-streams-polyfill": "^3.1.1" } }, "object-inspect": { @@ -4351,7 +4563,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "requires": { "has-flag": "^4.0.0" } @@ -4497,6 +4708,11 @@ "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" }, + "web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==" + }, "webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/unfurler/package.json b/unfurler/package.json index 5c61f9233..ad0c2b7fe 100644 --- a/unfurler/package.json +++ b/unfurler/package.json @@ -19,6 +19,8 @@ "dependencies": { "@types/node": "^16.11.41", "express": "^4.19.2", + "ejs": "^3.1.9", + "node-fetch-commonjs": "^3.3.1", "puppeteer": "^15.3.2", "puppeteer-cluster": "^0.23.0", "typescript": "~4.7.4" diff --git a/unfurler/src/config.ts b/unfurler/src/config.ts index 445ae4514..9af2b070d 100644 --- a/unfurler/src/config.ts +++ b/unfurler/src/config.ts @@ -30,6 +30,10 @@ interface IConfig { MAX_PAGE_AGE?: number; RENDER_TIMEOUT?: number; }; + API: { + MEMPOOL: string; + ESPLORA: string; + } SYSLOG: { ENABLED: boolean; HOST: string; @@ -53,6 +57,10 @@ const defaults: IConfig = { 'ENABLED': true, 'CLUSTER_SIZE': 1, }, + 'API': { + 'MEMPOOL': 'https://mempool.space/api/v1', + 'ESPLORA': 'https://mempool.space/api', + }, 'SYSLOG': { 'ENABLED': true, 'HOST': '127.0.0.1', @@ -66,6 +74,7 @@ class Config implements IConfig { SERVER: IConfig['SERVER']; MEMPOOL: IConfig['MEMPOOL']; PUPPETEER: IConfig['PUPPETEER']; + API: IConfig['API']; SYSLOG: IConfig['SYSLOG']; constructor() { @@ -73,6 +82,7 @@ class Config implements IConfig { this.SERVER = configs.SERVER; this.MEMPOOL = configs.MEMPOOL; this.PUPPETEER = configs.PUPPETEER; + this.API = configs.API; this.SYSLOG = configs.SYSLOG; } diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 162525dca..1ce5a0a72 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -113,6 +113,8 @@ class Server { } setUpRoutes() { + this.app.set('view engine', 'ejs'); + if (puppeteerEnabled) { this.app.get('/unfurl/render*', async (req, res) => { return this.renderPreview(req, res) }) this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) }) @@ -122,6 +124,7 @@ class Server { } this.app.get('/unfurl*', (req, res) => { return this.renderHTML(req, res, true) }) this.app.get('/slurp*', (req, res) => { return this.renderHTML(req, res, false) }) + this.app.get('/sip*', (req, res) => { return this.renderSip(req, res) }) this.app.get('*', (req, res) => { return this.renderHTML(req, res, false) }) } @@ -371,6 +374,38 @@ class Server { } return html; } + + async renderSip(req, res): Promise { + const start = Date.now(); + const rawPath = req.params[0]; + const { lang, path } = parseLanguageUrl(rawPath); + const matchedRoute = matchRoute(this.network, path, 'sip'); + + let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg); + let ogTitle = 'The Mempool Open Source Project®'; + + const canonical = this.canonicalHost + rawPath; + + if (matchedRoute.render) { + ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`; + ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`; + } + + if (matchedRoute.sip) { + logger.info(`sipping "${req.url}"`); + try { + const data = await matchedRoute.sip.getData(matchedRoute.params); + logger.info(`sip data fetched for "${req.url}" in ${Date.now() - start}ms`); + res.render(matchedRoute.sip.template, { canonical, ogImageUrl, ogTitle, matchedRoute, data }); + logger.info(`sip returned "${req.url}" in ${Date.now() - start}ms`); + } catch (e) { + logger.err(`failed to sip ${req.url}: ` + (e instanceof Error ? e.message : `${e}`)); + res.status(500).send(); + } + } else { + return this.renderHTML(req, res, false); + } + } } const server = new Server(); diff --git a/unfurler/src/routes.ts b/unfurler/src/routes.ts index dca64193c..00833315d 100644 --- a/unfurler/src/routes.ts +++ b/unfurler/src/routes.ts @@ -1,9 +1,19 @@ +import fetch from 'node-fetch-commonjs'; +import config from './config'; + interface Match { render: boolean; title: string; fallbackImg: string; staticImg?: string; networkMode: string; + params?: string[]; + sip?: SipTemplate; +} + +interface SipTemplate { + template: string; + getData: Function; } const routes = { @@ -19,18 +29,37 @@ const routes = { title: "Mempool Accelerator", fallbackImg: '/resources/previews/accelerator.jpg', }, - address: { - render: true, - params: 1, - getTitle(path) { - return `Address: ${path[0]}`; - } - }, block: { render: true, params: 1, getTitle(path) { return `Block: ${path[0]}`; + }, + sip: { + template: 'block', + async getData (params: string[]) { + if (params?.length) { + let blockId = params[0]; + if (blockId.length !== 64) { + blockId = await (await fetch(config.API.ESPLORA + `/block-height/${blockId}`)).text(); + } + const [block, transactions] = await Promise.all([ + (await fetch(config.API.MEMPOOL + `/block/${blockId}`)).json(), + (await fetch(config.API.ESPLORA + `/block/${blockId}/txids`)).json() + ]) + return { + block, + transactions, + }; + } + } + } + }, + address: { + render: true, + params: 1, + getTitle(path) { + return `Address: ${path[0]}`; } }, blocks: { @@ -162,7 +191,7 @@ const networks = { } }; -export function matchRoute(network: string, path: string): Match { +export function matchRoute(network: string, path: string, matchFor: string = 'render'): Match { const match: Match = { render: false, title: '', @@ -183,7 +212,7 @@ export function matchRoute(network: string, path: string): Match { match.fallbackImg = route.fallbackImg; // traverse the route tree until we run out of route or tree, or hit a renderable match - while (route.routes && parts.length && route.routes[parts[0]]) { + while (!route[matchFor] && route.routes && parts.length && route.routes[parts[0]]) { route = route.routes[parts[0]]; parts.shift(); if (route.fallbackImg) { @@ -192,8 +221,10 @@ export function matchRoute(network: string, path: string): Match { } // enough route parts left for title & rendering - if (route.render && parts.length >= route.params) { - match.render = true; + if (route[matchFor] && parts.length >= route.params) { + match.render = route.render; + match.sip = route.sip; + match.params = parts; } // only use set a static image for exact matches if (!parts.length && route.staticImg) { diff --git a/unfurler/views/block.ejs b/unfurler/views/block.ejs new file mode 100644 index 000000000..92133b56e --- /dev/null +++ b/unfurler/views/block.ejs @@ -0,0 +1,91 @@ + + + <%- include('head'); %> + + <%- include('header'); %> +
+

Block <%- data.block.height %>

+

Previous block Next block

+

Details

+ + + + + + + + + + + + + + + + + + + + + + <% if (data.block.extras) { %> + <% if (data.block.extras.feeRange && data.block.extras.feeRange.length) { %> + + + + + <% } %> + + + + + + + + + + + + + <% if (data.block.extras.matchRate != null) { %> + + + + + <% } %> + + + + + <% } %> + + + + + + + + + + + + + + + + + + + + +
Height<%= data.block.height %>
Hash<%= data.block.id %>
Timestamp<%= (new Date(data.block.timestamp * 1000)).toISOString() %>
Size<%= data.block.size / 1_000_000 %> MB
Weight<%= data.block.weight / 1_000_000 %> MWU
Fee span<%= (data.block.extras.feeRange[0]).toFixed(2) %> - <%= (data.block.extras.feeRange[data.block.extras.feeRange.length - 1]).toFixed(2) %> sat/vB
Median fee<%= (data.block.extras.medianFee).toFixed(2) %> sat/vB
Total fees<%= (data.block.extras.totalFees / 100_000_000).toFixed(8) %> BTC
Subsidy + fees<%= (data.block.extras.reward / 100_000_000).toFixed(8) %> BTC
Health<%= data.block.extras.matchRate %>%
Miner<%= data.block.extras.pool.name || 'Unknown' %>
Version<%= "0x" + data.block.version.toString(16) %>
Difficulty<%= data.block.difficulty %>
Bits<%= "0x" + data.block.bits.toString(16) %>
Nonce<%= "0x" + data.block.nonce.toString(16) %>
Merkle root<%= data.block.merkle_root %>
+

<%= data.transactions.length %> Transactions

+
    + <% data.transactions.forEach((txid, i) => { %> +
  1. <%= txid %><% if (i === 0) { %> (Coinbase)<% } %>
    2. + <% }); %> +
+
+ <%- include('footer'); %> + + \ No newline at end of file diff --git a/unfurler/views/footer.ejs b/unfurler/views/footer.ejs new file mode 100644 index 000000000..8857b08ad --- /dev/null +++ b/unfurler/views/footer.ejs @@ -0,0 +1,130 @@ + + + \ No newline at end of file diff --git a/unfurler/views/head.ejs b/unfurler/views/head.ejs new file mode 100644 index 000000000..8cbdc8a13 --- /dev/null +++ b/unfurler/views/head.ejs @@ -0,0 +1,23 @@ + + + <%= ogTitle %> + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/unfurler/views/header.ejs b/unfurler/views/header.ejs new file mode 100644 index 000000000..073e3efb3 --- /dev/null +++ b/unfurler/views/header.ejs @@ -0,0 +1,79 @@ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+

The Mempool Open Source Project®

+

Explore the full Bitcoin ecosystem with mempool.space™

+
+
\ No newline at end of file