diff --git a/.changeset/gentle-fishes-enjoy.md b/.changeset/gentle-fishes-enjoy.md new file mode 100644 index 000000000..b4c9b8f87 --- /dev/null +++ b/.changeset/gentle-fishes-enjoy.md @@ -0,0 +1,5 @@ +--- +"nostrudel": minor +--- + +Fetch open graph metadata for links diff --git a/.changeset/soft-coins-beg.md b/.changeset/soft-coins-beg.md new file mode 100644 index 000000000..c30e0e8c0 --- /dev/null +++ b/.changeset/soft-coins-beg.md @@ -0,0 +1,5 @@ +--- +"nostrudel": minor +--- + +Add CORS proxy diff --git a/package.json b/package.json index abb5a176c..080b9d81d 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "@emotion/react": "^11.11.0", "@emotion/styled": "^11.11.0", "bech32": "^2.0.0", + "cheerio": "^1.0.0-rc.12", "dayjs": "^1.11.8", "framer-motion": "^7.10.3", "idb": "^7.1.1", diff --git a/src/components/embed-types/common.tsx b/src/components/embed-types/common.tsx index 497d6730a..6ef5bec8d 100644 --- a/src/components/embed-types/common.tsx +++ b/src/components/embed-types/common.tsx @@ -3,6 +3,7 @@ import appSettings from "../../services/app-settings"; import { ImageGalleryLink } from "../image-gallery"; import { useIsMobile } from "../../hooks/use-is-mobile"; import { useTrusted } from "../note/trust"; +import OpenGraphCard from "../open-graph-card"; const BlurredImage = (props: ImageProps) => { const { isOpen, onOpen } = useDisclosure(); @@ -44,9 +45,5 @@ export function renderVideoUrl(match: URL) { } export function renderGenericUrl(match: URL) { - return ( - - {match.toString()} - - ); + return ; } diff --git a/src/components/embed-types/emoji.tsx b/src/components/embed-types/emoji.tsx index cc37491f2..706c78f52 100644 --- a/src/components/embed-types/emoji.tsx +++ b/src/components/embed-types/emoji.tsx @@ -6,8 +6,6 @@ export function embedEmoji(content: EmbedableContent, note: NostrEvent | DraftNo return embedJSX(content, { regexp: /:([a-zA-Z0-9]+):/i, render: (match) => { - console.log(match); - const emojiTag = note.tags.find( (tag) => tag[0] === "emoji" && tag[1].toLowerCase() === match[1].toLowerCase() && tag[2] ); diff --git a/src/components/open-graph-card.tsx b/src/components/open-graph-card.tsx new file mode 100644 index 000000000..b61498799 --- /dev/null +++ b/src/components/open-graph-card.tsx @@ -0,0 +1,32 @@ +import { Box, CardProps, Code, Heading, Image, Link, LinkBox, LinkOverlay, Text } from "@chakra-ui/react"; +import useOpenGraphData from "../hooks/use-open-graph-data"; + +export default function OpenGraphCard({ url, ...props }: { url: URL } & Omit) { + const { value: data, loading } = useOpenGraphData(url); + + const link = ( + + {url.toString()} + + ); + + if (!data) return link; + + return ( + + {data.ogImage?.map((ogImage) => ( + + ))} + + + + + {data.ogTitle ?? data.dcTitle} + + + {data.ogDescription || data.dcDescription} + {link} + + + ); +} diff --git a/src/helpers/cors.ts b/src/helpers/cors.ts new file mode 100644 index 000000000..7108fd2b9 --- /dev/null +++ b/src/helpers/cors.ts @@ -0,0 +1,17 @@ +import appSettings from "../services/app-settings"; +import { convertToUrl } from "./url"; + +const corsFailedHosts = new Set(); + +export function fetchWithCorsFallback(url: URL | string, opts?: RequestInit) { + if (!appSettings.value.corsProxy) return fetch(url, opts); + + if (corsFailedHosts.has(convertToUrl(url).host)) { + return fetch(appSettings.value.corsProxy + url, opts); + } + + return fetch(url, opts).catch((e) => { + corsFailedHosts.add(convertToUrl(url).host); + return fetch(appSettings.value.corsProxy + url, opts); + }); +} diff --git a/src/helpers/url.ts b/src/helpers/url.ts index 90965977c..314f14adc 100644 --- a/src/helpers/url.ts +++ b/src/helpers/url.ts @@ -1,4 +1,4 @@ -const convertToUrl = (url: string | URL) => (url instanceof URL ? url : new URL(url)); +export const convertToUrl = (url: string | URL) => (url instanceof URL ? url : new URL(url)); export function normalizeRelayUrl(relayUrl: string) { const url = new URL(relayUrl); diff --git a/src/hooks/use-open-graph-data.ts b/src/hooks/use-open-graph-data.ts new file mode 100644 index 000000000..a1f66347c --- /dev/null +++ b/src/hooks/use-open-graph-data.ts @@ -0,0 +1,13 @@ +import { useAsync } from "react-use"; +import extractMetaTags from "../lib/open-graph-scraper/extract"; +import { fetchWithCorsFallback } from "../helpers/cors"; + +export default function useOpenGraphData(url: URL) { + return useAsync(async () => { + try { + const html = await fetchWithCorsFallback(url).then((res) => res.text()); + return extractMetaTags(html); + } catch (e) {} + return null; + }, [url]); +} diff --git a/src/lib/open-graph-scraper/README.md b/src/lib/open-graph-scraper/README.md new file mode 100644 index 000000000..dd85c0e10 --- /dev/null +++ b/src/lib/open-graph-scraper/README.md @@ -0,0 +1 @@ +open graph parser code copied from https://github.com/jshemas/openGraphScraper/blob/master/lib/extract.ts and modified to work in the browser diff --git a/src/lib/open-graph-scraper/extract.ts b/src/lib/open-graph-scraper/extract.ts new file mode 100644 index 000000000..5f08183ac --- /dev/null +++ b/src/lib/open-graph-scraper/extract.ts @@ -0,0 +1,54 @@ +import { load } from "cheerio"; + +import fallback from "./fallback"; +import fields from "./fields"; +import mediaSetup from "./media"; + +import type { OgObjectInteral } from "./types"; + +/** + * extract all of the meta tags needed for ogs + * + * @param {sting} body - the body of the fetch request + * @param {object} options - options for ogs + * @return {object} object with ogs results + * + */ +export default function extractMetaTags(body: string, useFallbacks = true) { + let ogObject: OgObjectInteral = {}; + const $ = load(body); + const metaFields = fields; + + // find all of the open graph info in the meta tags + $("meta").each((index, meta) => { + if (!meta.attribs || (!meta.attribs.property && !meta.attribs.name)) return; + const property = meta.attribs.property || meta.attribs.name; + const content = meta.attribs.content || meta.attribs.value; + metaFields.forEach((item) => { + if (item && property.toLowerCase() === item.property.toLowerCase()) { + if (!item.multiple) { + // @ts-ignore + ogObject[item.fieldName] = content; + // @ts-ignore + } else if (!ogObject[item.fieldName]) { + // @ts-ignore + ogObject[item.fieldName] = [content]; + // @ts-ignore + } else if (Array.isArray(ogObject[item.fieldName])) { + // @ts-ignore + ogObject[item.fieldName].push(content); + } + } + }); + }); + + // formats the multiple media values + ogObject = mediaSetup(ogObject); + + // if onlyGetOpenGraphInfo isn't set, run the open graph fallbacks + if (useFallbacks) { + ogObject = fallback(ogObject, $); + } + + return ogObject; +} diff --git a/src/lib/open-graph-scraper/fallback.ts b/src/lib/open-graph-scraper/fallback.ts new file mode 100644 index 000000000..c05204f6e --- /dev/null +++ b/src/lib/open-graph-scraper/fallback.ts @@ -0,0 +1,172 @@ +import type { CheerioAPI } from "cheerio"; + +import { findImageTypeFromUrl, isImageTypeValid, isUrlValid } from "./utils"; +import type { ImageObject, OgObjectInteral } from "./types"; + +const doesElementExist = (selector: string, attribute: string, $: CheerioAPI) => + $(selector).attr(attribute) && ($(selector).attr(attribute)?.length || 0) > 0; + +/** + * ogs fallbacks + * + * @param {object} ogObject - the current ogObject + * @param {object} $ - cheerio.load() of the current html + * @return {object} object with ogs results with updated fallback values + * + */ +export function fallback(ogObject: OgObjectInteral, $: CheerioAPI) { + // title fallback + if (!ogObject.ogTitle) { + if ($("title").text() && $("title").text().length > 0) { + ogObject.ogTitle = $("title").first().text(); + } else if ( + $('head > meta[name="title"]').attr("content") && + ($('head > meta[name="title"]').attr("content")?.length || 0) > 0 + ) { + ogObject.ogTitle = $('head > meta[name="title"]').attr("content"); + } else if ($(".post-title").text() && $(".post-title").text().length > 0) { + ogObject.ogTitle = $(".post-title").text(); + } else if ($(".entry-title").text() && $(".entry-title").text().length > 0) { + ogObject.ogTitle = $(".entry-title").text(); + } else if ($('h1[class*="title" i] a').text() && $('h1[class*="title" i] a').text().length > 0) { + ogObject.ogTitle = $('h1[class*="title" i] a').text(); + } else if ($('h1[class*="title" i]').text() && $('h1[class*="title" i]').text().length > 0) { + ogObject.ogTitle = $('h1[class*="title" i]').text(); + } + } + + // Get meta description tag if og description was not provided + if (!ogObject.ogDescription) { + if (doesElementExist('head > meta[name="description"]', "content", $)) { + ogObject.ogDescription = $('head > meta[name="description"]').attr("content"); + } else if (doesElementExist('head > meta[itemprop="description"]', "content", $)) { + ogObject.ogDescription = $('head > meta[itemprop="description"]').attr("content"); + } else if ($("#description").text() && $("#description").text().length > 0) { + ogObject.ogDescription = $("#description").text(); + } + } + + // Get all of images if there is no og:image info + if (!ogObject.ogImage) { + ogObject.ogImage = []; + $("img").map((index, imageElement) => { + const source: string = $(imageElement).attr("src") || ""; + if (!source) return false; + const type = findImageTypeFromUrl(source); + if (!isUrlValid(source) || !isImageTypeValid(type)) return false; + const fallbackImage: ImageObject = { + url: source, + type, + }; + if ($(imageElement).attr("width") && Number($(imageElement).attr("width"))) + fallbackImage.width = Number($(imageElement).attr("width")); + if ($(imageElement).attr("height") && Number($(imageElement).attr("height"))) + fallbackImage.height = Number($(imageElement).attr("height")); + ogObject.ogImage?.push(fallbackImage); + return false; + }); + ogObject.ogImage = ogObject.ogImage + .filter((value) => value.url !== undefined && value.url !== "") + .filter((value, index) => index < 10); + if (ogObject.ogImage.length === 0) delete ogObject.ogImage; + } else if (ogObject.ogImage) { + ogObject.ogImage.map((image) => { + if (image.url && !image.type) { + const type = findImageTypeFromUrl(image.url); + if (isImageTypeValid(type)) image.type = type; + } + return false; + }); + } + + // audio fallback + if (!ogObject.ogAudioURL && !ogObject.ogAudioSecureURL) { + const audioElementValue: string = $("audio").attr("src") || ""; + const audioSourceElementValue: string = $("audio > source").attr("src") || ""; + if (doesElementExist("audio", "src", $)) { + if (audioElementValue.startsWith("https")) { + ogObject.ogAudioSecureURL = audioElementValue; + } else { + ogObject.ogAudioURL = audioElementValue; + } + const audioElementTypeValue: string = $("audio").attr("type") || ""; + if (!ogObject.ogAudioType && doesElementExist("audio", "type", $)) ogObject.ogAudioType = audioElementTypeValue; + } else if (doesElementExist("audio > source", "src", $)) { + if (audioSourceElementValue.startsWith("https")) { + ogObject.ogAudioSecureURL = audioSourceElementValue; + } else { + ogObject.ogAudioURL = audioSourceElementValue; + } + const audioSourceElementTypeValue: string = $("audio > source").attr("type") || ""; + if (!ogObject.ogAudioType && doesElementExist("audio > source", "type", $)) + ogObject.ogAudioType = audioSourceElementTypeValue; + } + } + + // locale fallback + if (!ogObject.ogLocale) { + if (doesElementExist("html", "lang", $)) { + ogObject.ogLocale = $("html").attr("lang"); + } else if (doesElementExist('head > meta[itemprop="inLanguage"]', "content", $)) { + ogObject.ogLocale = $('head > meta[itemprop="inLanguage"]').attr("content"); + } + } + + // logo fallback + if (!ogObject.ogLogo) { + if (doesElementExist('meta[itemprop="logo"]', "content", $)) { + ogObject.ogLogo = $('meta[itemprop="logo"]').attr("content"); + } else if (doesElementExist('img[itemprop="logo"]', "src", $)) { + ogObject.ogLogo = $('img[itemprop="logo"]').attr("src"); + } + } + + // url fallback + if (!ogObject.ogUrl) { + if (doesElementExist('link[rel="canonical"]', "href", $)) { + ogObject.ogUrl = $('link[rel="canonical"]').attr("href"); + } else if (doesElementExist('link[rel="alternate"][hreflang="x-default"]', "href", $)) { + ogObject.ogUrl = $('link[rel="alternate"][hreflang="x-default"]').attr("href"); + } + } + + // date fallback + if (!ogObject.ogDate) { + if (doesElementExist('head > meta[name="date"]', "content", $)) { + ogObject.ogDate = $('head > meta[name="date"]').attr("content"); + } else if (doesElementExist('[itemprop*="datemodified" i]', "content", $)) { + ogObject.ogDate = $('[itemprop*="datemodified" i]').attr("content"); + } else if (doesElementExist('[itemprop="datepublished" i]', "content", $)) { + ogObject.ogDate = $('[itemprop="datepublished" i]').attr("content"); + } else if (doesElementExist('[itemprop*="date" i]', "content", $)) { + ogObject.ogDate = $('[itemprop*="date" i]').attr("content"); + } else if (doesElementExist('time[itemprop*="date" i]', "datetime", $)) { + ogObject.ogDate = $('time[itemprop*="date" i]').attr("datetime"); + } else if (doesElementExist("time[datetime]", "datetime", $)) { + ogObject.ogDate = $("time[datetime]").attr("datetime"); + } + } + + // favicon fallback + if (!ogObject.favicon) { + if (doesElementExist('link[rel="shortcut icon"]', "href", $)) { + ogObject.favicon = $('link[rel="shortcut icon"]').attr("href"); + } else if (doesElementExist('link[rel="icon"]', "href", $)) { + ogObject.favicon = $('link[rel="icon"]').attr("href"); + } else if (doesElementExist('link[rel="mask-icon"]', "href", $)) { + ogObject.favicon = $('link[rel="mask-icon"]').attr("href"); + } else if (doesElementExist('link[rel="apple-touch-icon"]', "href", $)) { + ogObject.favicon = $('link[rel="apple-touch-icon"]').attr("href"); + } else if (doesElementExist('link[type="image/png"]', "href", $)) { + ogObject.favicon = $('link[type="image/png"]').attr("href"); + } else if (doesElementExist('link[type="image/ico"]', "href", $)) { + ogObject.favicon = $('link[type="image/ico"]').attr("href"); + } else if (doesElementExist('link[type="image/x-icon"]', "href", $)) { + ogObject.favicon = $('link[type="image/x-icon"]').attr("href"); + } + } + + return ogObject; +} + +export default fallback; diff --git a/src/lib/open-graph-scraper/fields.ts b/src/lib/open-graph-scraper/fields.ts new file mode 100644 index 000000000..96dcc5ce1 --- /dev/null +++ b/src/lib/open-graph-scraper/fields.ts @@ -0,0 +1,855 @@ +/** + * array of meta tags ogs is looking for + * + * @return {array} array of meta tags + * + */ +const fields = [ + { + multiple: false, + property: "og:title", + fieldName: "ogTitle", + }, + { + multiple: false, + property: "og:type", + fieldName: "ogType", + }, + { + multiple: false, + property: "og:logo", + fieldName: "ogLogo", + }, + { + multiple: true, + property: "og:image", + fieldName: "ogImageProperty", + }, + { + multiple: true, + property: "og:image:url", + fieldName: "ogImageURL", + }, + { + multiple: true, + property: "og:image:secure_url", + fieldName: "ogImageSecureURL", + }, + { + multiple: true, + property: "og:image:width", + fieldName: "ogImageWidth", + }, + { + multiple: true, + property: "og:image:height", + fieldName: "ogImageHeight", + }, + { + multiple: true, + property: "og:image:type", + fieldName: "ogImageType", + }, + { + multiple: false, + property: "og:url", + fieldName: "ogUrl", + }, + { + multiple: false, + property: "og:audio", + fieldName: "ogAudio", + }, + { + multiple: false, + property: "og:audio:url", + fieldName: "ogAudioURL", + }, + { + multiple: false, + property: "og:audio:secure_url", + fieldName: "ogAudioSecureURL", + }, + { + multiple: false, + property: "og:audio:type", + fieldName: "ogAudioType", + }, + { + multiple: false, + property: "og:description", + fieldName: "ogDescription", + }, + { + multiple: false, + property: "og:determiner", + fieldName: "ogDeterminer", + }, + { + multiple: false, + property: "og:locale", + fieldName: "ogLocale", + }, + { + multiple: false, + property: "og:locale:alternate", + fieldName: "ogLocaleAlternate", + }, + { + multiple: false, + property: "og:site_name", + fieldName: "ogSiteName", + }, + { + multiple: false, + property: "og:product:retailer_item_id", + fieldName: "ogProductRetailerItemId", + }, + { + multiple: false, + property: "og:product:price:amount", + fieldName: "ogProductPriceAmount", + }, + { + multiple: false, + property: "og:product:price:currency", + fieldName: "ogProductPriceCurrency", + }, + { + multiple: false, + property: "og:product:availability", + fieldName: "ogProductAvailability", + }, + { + multiple: false, + property: "og:product:condition", + fieldName: "ogProductCondition", + }, + { + multiple: false, + property: "og:price:amount", + fieldName: "ogPriceAmount", + }, + { + multiple: false, + property: "og:price:currency", + fieldName: "ogPriceCurrency", + }, + { + multiple: false, + property: "og:availability", + fieldName: "ogAvailability", + }, + { + multiple: true, + property: "og:video", + fieldName: "ogVideoProperty", + }, + { + multiple: true, + property: "og:video:url", // An alternative to 'og:video' + fieldName: "ogVideoProperty", + }, + { + multiple: true, + property: "og:video:secure_url", + fieldName: "ogVideoSecureURL", + }, + { + multiple: true, + property: "og:video:actor:id", + fieldName: "ogVideoActorId", + }, + { + multiple: true, + property: "og:video:width", + fieldName: "ogVideoWidth", + }, + { + multiple: true, + property: "og:video:height", + fieldName: "ogVideoHeight", + }, + { + multiple: true, + property: "og:video:type", + fieldName: "ogVideoType", + }, + { + multiple: false, + property: "twitter:card", + fieldName: "twitterCard", + }, + { + multiple: false, + property: "twitter:url", + fieldName: "twitterUrl", + }, + { + multiple: false, + property: "twitter:site", + fieldName: "twitterSite", + }, + { + multiple: false, + property: "twitter:site:id", + fieldName: "twitterSiteId", + }, + { + multiple: false, + property: "twitter:creator", + fieldName: "twitterCreator", + }, + { + multiple: false, + property: "twitter:creator:id", + fieldName: "twitterCreatorId", + }, + { + multiple: false, + property: "twitter:title", + fieldName: "twitterTitle", + }, + { + multiple: false, + property: "twitter:description", + fieldName: "twitterDescription", + }, + { + multiple: true, + property: "twitter:image", + fieldName: "twitterImageProperty", + }, + { + multiple: true, + property: "twitter:image:height", + fieldName: "twitterImageHeight", + }, + { + multiple: true, + property: "twitter:image:width", + fieldName: "twitterImageWidth", + }, + { + multiple: true, + property: "twitter:image:src", + fieldName: "twitterImageSrc", + }, + { + multiple: true, + property: "twitter:image:alt", + fieldName: "twitterImageAlt", + }, + { + multiple: true, + property: "twitter:player", + fieldName: "twitterPlayerProperty", + }, + { + multiple: true, + property: "twitter:player:width", + fieldName: "twitterPlayerWidth", + }, + { + multiple: true, + property: "twitter:player:height", + fieldName: "twitterPlayerHeight", + }, + { + multiple: true, + property: "twitter:player:stream", + fieldName: "twitterPlayerStream", + }, + { + multiple: true, + property: "twitter:player:stream:content_type", + fieldName: "twitterPlayerStreamContentType", + }, + { + multiple: false, + property: "twitter:app:name:iphone", + fieldName: "twitterAppNameiPhone", + }, + { + multiple: false, + property: "twitter:app:id:iphone", + fieldName: "twitterAppIdiPhone", + }, + { + multiple: false, + property: "twitter:app:url:iphone", + fieldName: "twitterAppUrliPhone", + }, + { + multiple: false, + property: "twitter:app:name:ipad", + fieldName: "twitterAppNameiPad", + }, + { + multiple: false, + property: "twitter:app:id:ipad", + fieldName: "twitterAppIdiPad", + }, + { + multiple: false, + property: "twitter:app:url:ipad", + fieldName: "twitterAppUrliPad", + }, + { + multiple: false, + property: "twitter:app:name:googleplay", + fieldName: "twitterAppNameGooglePlay", + }, + { + multiple: false, + property: "twitter:app:id:googleplay", + fieldName: "twitterAppIdGooglePlay", + }, + { + multiple: false, + property: "twitter:app:url:googleplay", + fieldName: "twitterAppUrlGooglePlay", + }, + { + multiple: true, + property: "music:song", + fieldName: "musicSongProperty", + }, + { + multiple: true, + property: "music:song:disc", + fieldName: "musicSongDisc", + }, + { + multiple: true, + property: "music:song:track", + fieldName: "musicSongTrack", + }, + { + multiple: true, + property: "music:song:url", + fieldName: "musicSongUrl", + }, + { + multiple: true, + property: "music:musician", + fieldName: "musicMusician", + }, + { + multiple: false, + property: "music:release_date", + fieldName: "musicReleaseDate", + }, + { + multiple: false, + property: "music:duration", + fieldName: "musicDuration", + }, + { + multiple: true, + property: "music:creator", + fieldName: "musicCreator", + }, + { + multiple: true, + property: "music:album", + fieldName: "musicAlbum", + }, + { + multiple: false, + property: "music:album:disc", + fieldName: "musicAlbumDisc", + }, + { + multiple: false, + property: "music:album:track", + fieldName: "musicAlbumTrack", + }, + { + multiple: false, + property: "music:album:url", + fieldName: "musicAlbumUrl", + }, + { + multiple: false, + property: "article:published_date", + fieldName: "articlePublishedDate", + }, + { + multiple: false, + property: "article:published_time", + fieldName: "articlePublishedTime", + }, + { + multiple: false, + property: "article:modified_date", + fieldName: "articleModifiedDate", + }, + { + multiple: false, + property: "article:modified_time", + fieldName: "articleModifiedTime", + }, + { + multiple: false, + property: "article:expiration_time", + fieldName: "articleExpirationTime", + }, + { + multiple: false, + property: "article:author", + fieldName: "articleAuthor", + }, + { + multiple: false, + property: "article:section", + fieldName: "articleSection", + }, + { + multiple: false, + property: "article:tag", + fieldName: "articleTag", + }, + { + multiple: false, + property: "article:publisher", + fieldName: "articlePublisher", + }, + { + multiple: false, + property: "og:article:published_time", + fieldName: "ogArticlePublishedTime", + }, + { + multiple: false, + property: "og:article:modified_time", + fieldName: "ogArticleModifiedTime", + }, + { + multiple: false, + property: "og:article:expiration_time", + fieldName: "ogArticleExpirationTime", + }, + { + multiple: false, + property: "og:article:author", + fieldName: "ogArticleAuthor", + }, + { + multiple: false, + property: "og:article:section", + fieldName: "ogArticleSection", + }, + { + multiple: false, + property: "og:article:tag", + fieldName: "ogArticleTag", + }, + { + multiple: false, + property: "og:article:publisher", + fieldName: "ogArticlePublisher", + }, + { + multiple: false, + property: "books:book", + fieldName: "booksBook", + }, + { + multiple: false, + property: "book:author", + fieldName: "bookAuthor", + }, + { + multiple: false, + property: "book:isbn", + fieldName: "bookIsbn", + }, + { + multiple: false, + property: "book:release_date", + fieldName: "bookReleaseDate", + }, + { + multiple: false, + property: "book:canonical_name", + fieldName: "bookCanonicalName", + }, + { + multiple: false, + property: "book:tag", + fieldName: "bookTag", + }, + { + multiple: false, + property: "books:rating:value", + fieldName: "booksRatingValue", + }, + { + multiple: false, + property: "books:rating:scale", + fieldName: "booksRatingScale", + }, + { + multiple: false, + property: "profile:first_name", + fieldName: "profileFirstName", + }, + { + multiple: false, + property: "profile:last_name", + fieldName: "profileLastName", + }, + { + multiple: false, + property: "profile:username", + fieldName: "profileUsername", + }, + { + multiple: false, + property: "profile:gender", + fieldName: "profileGender", + }, + { + multiple: false, + property: "business:contact_data:street_address", + fieldName: "businessContactDataStreetAddress", + }, + { + multiple: false, + property: "business:contact_data:locality", + fieldName: "businessContactDataLocality", + }, + { + multiple: false, + property: "business:contact_data:region", + fieldName: "businessContactDataRegion", + }, + { + multiple: false, + property: "business:contact_data:postal_code", + fieldName: "businessContactDataPostalCode", + }, + { + multiple: false, + property: "business:contact_data:country_name", + fieldName: "businessContactDataCountryName", + }, + { + multiple: false, + property: "restaurant:menu", + fieldName: "restaurantMenu", + }, + { + multiple: false, + property: "restaurant:restaurant", + fieldName: "restaurantRestaurant", + }, + { + multiple: false, + property: "restaurant:section", + fieldName: "restaurantSection", + }, + { + multiple: false, + property: "restaurant:variation:price:amount", + fieldName: "restaurantVariationPriceAmount", + }, + { + multiple: false, + property: "restaurant:variation:price:currency", + fieldName: "restaurantVariationPriceCurrency", + }, + { + multiple: false, + property: "restaurant:contact_info:website", + fieldName: "restaurantContactInfoWebsite", + }, + { + multiple: false, + property: "restaurant:contact_info:street_address", + fieldName: "restaurantContactInfoStreetAddress", + }, + { + multiple: false, + property: "restaurant:contact_info:locality", + fieldName: "restaurantContactInfoLocality", + }, + { + multiple: false, + property: "restaurant:contact_info:region", + fieldName: "restaurantContactInfoRegion", + }, + { + multiple: false, + property: "restaurant:contact_info:postal_code", + fieldName: "restaurantContactInfoPostalCode", + }, + { + multiple: false, + property: "restaurant:contact_info:country_name", + fieldName: "restaurantContactInfoCountryName", + }, + { + multiple: false, + property: "restaurant:contact_info:email", + fieldName: "restaurantContactInfoEmail", + }, + { + multiple: false, + property: "restaurant:contact_info:phone_number", + fieldName: "restaurantContactInfoPhoneNumber", + }, + { + multiple: false, + property: "place:location:latitude", + fieldName: "placeLocationLatitude", + }, + { + multiple: false, + property: "place:location:longitude", + fieldName: "placeLocationLongitude", + }, + { + multiple: false, + property: "og:date", + fieldName: "ogDate", + }, + { + multiple: false, + property: "author", + fieldName: "author", + }, + { + multiple: false, + property: "updated_time", + fieldName: "updatedTime", + }, + { + multiple: false, + property: "modified_time", + fieldName: "modifiedTime", + }, + { + multiple: false, + property: "published_time", + fieldName: "publishedTime", + }, + { + multiple: false, + property: "release_date", + fieldName: "releaseDate", + }, + { + multiple: false, + property: "dc.source", + fieldName: "dcSource", + }, + { + multiple: false, + property: "dc.subject", + fieldName: "dcSubject", + }, + { + multiple: false, + property: "dc.title", + fieldName: "dcTitle", + }, + { + multiple: false, + property: "dc.type", + fieldName: "dcType", + }, + { + multiple: false, + property: "dc.creator", + fieldName: "dcCreator", + }, + { + multiple: false, + property: "dc.coverage", + fieldName: "dcCoverage", + }, + { + multiple: false, + property: "dc.language", + fieldName: "dcLanguage", + }, + { + multiple: false, + property: "dc.contributor", + fieldName: "dcContributor", + }, + { + multiple: false, + property: "dc.date", + fieldName: "dcDate", + }, + { + multiple: false, + property: "dc.date.issued", + fieldName: "dcDateIssued", + }, + { + multiple: false, + property: "dc.date.created", + fieldName: "dcDateCreated", + }, + { + multiple: false, + property: "dc.description", + fieldName: "dcDescription", + }, + { + multiple: false, + property: "dc.identifier", + fieldName: "dcIdentifier", + }, + { + multiple: false, + property: "dc.publisher", + fieldName: "dcPublisher", + }, + { + multiple: false, + property: "dc.rights", + fieldName: "dcRights", + }, + { + multiple: false, + property: "dc.relation", + fieldName: "dcRelation", + }, + { + multiple: false, + property: "dc.format.media", + fieldName: "dcFormatMedia", + }, + { + multiple: false, + property: "dc.format.size", + fieldName: "dcFormatSize", + }, + { + multiple: false, + property: "al:ios:url", + fieldName: "alIosUrl", + }, + { + multiple: false, + property: "al:ios:app_store_id", + fieldName: "alIosAppStoreId", + }, + { + multiple: false, + property: "al:ios:app_name", + fieldName: "alIosAppName", + }, + { + multiple: false, + property: "al:iphone:url", + fieldName: "alIphoneUrl", + }, + { + multiple: false, + property: "al:iphone:app_store_id", + fieldName: "alIphoneAppStoreId", + }, + { + multiple: false, + property: "al:iphone:app_name", + fieldName: "alIphoneAppName", + }, + { + multiple: false, + property: "al:ipad:url", + fieldName: "alIpadUrl", + }, + { + multiple: false, + property: "al:ipad:app_store_id", + fieldName: "alIpadAppStoreId", + }, + { + multiple: false, + property: "al:ipad:app_name", + fieldName: "alIpadAppName", + }, + { + multiple: false, + property: "al:android:url", + fieldName: "alAndroidUrl", + }, + { + multiple: false, + property: "al:android:package", + fieldName: "alAndroidPackage", + }, + { + multiple: false, + property: "al:android:class", + fieldName: "alAndroidClass", + }, + { + multiple: false, + property: "al:android:app_name", + fieldName: "alAndroidAppName", + }, + { + multiple: false, + property: "al:windows_phone:url", + fieldName: "alWindowsPhoneUrl", + }, + { + multiple: false, + property: "al:windows_phone:app_id", + fieldName: "alWindowsPhoneAppId", + }, + { + multiple: false, + property: "al:windows_phone:app_name", + fieldName: "alWindowsPhoneAppName", + }, + { + multiple: false, + property: "al:windows:url", + fieldName: "alWindowsUrl", + }, + { + multiple: false, + property: "al:windows:app_id", + fieldName: "alWindowsAppId", + }, + { + multiple: false, + property: "al:windows:app_name", + fieldName: "alWindowsAppName", + }, + { + multiple: false, + property: "al:windows_universal:url", + fieldName: "alWindowsUniversalUrl", + }, + { + multiple: false, + property: "al:windows_universal:app_id", + fieldName: "alWindowsUniversalAppId", + }, + { + multiple: false, + property: "al:windows_universal:app_name", + fieldName: "alWindowsUniversalAppName", + }, + { + multiple: false, + property: "al:web:url", + fieldName: "alWebUrl", + }, + { + multiple: false, + property: "al:web:should_fallback", + fieldName: "alWebShouldFallback", + }, +]; + +export default fields; diff --git a/src/lib/open-graph-scraper/media.ts b/src/lib/open-graph-scraper/media.ts new file mode 100644 index 000000000..a0f94e2da --- /dev/null +++ b/src/lib/open-graph-scraper/media.ts @@ -0,0 +1,234 @@ +import fields from "./fields"; +import { removeNestedUndefinedValues } from "./utils"; +import type { + ImageObject, + MusicSongObject, + OgObjectInteral, + TwitterImageObject, + TwitterPlayerObject, + VideoObject, +} from "./types"; + +const mediaMapperTwitterImage = (item: TwitterImageObject[]) => ({ + alt: item[3], + height: item[2], + url: item[0], + width: item[1], +}); + +const mediaMapperTwitterPlayer = (item: TwitterPlayerObject[]) => ({ + height: item[2], + stream: item[3], + url: item[0], + width: item[1], +}); + +const mediaMapperMusicSong = (item: MusicSongObject[]) => ({ + disc: item[2], + track: item[1], + url: item[0], +}); + +const mediaMapper = (item: ImageObject[] | VideoObject[]) => ({ + height: item[2], + type: item[3], + url: item[0], + width: item[1], +}); + +const mediaSorter = ( + a: ImageObject | TwitterImageObject | VideoObject | TwitterPlayerObject, + b: ImageObject | TwitterImageObject | VideoObject | TwitterPlayerObject +) => { + if (!(a.url && b.url)) { + return 0; + } + + const aRes = a.url.match(/\.(\w{2,5})$/); + const aExt = (aRes && aRes[1].toLowerCase()) || null; + const bRes = b.url.match(/\.(\w{2,5})$/); + const bExt = (bRes && bRes[1].toLowerCase()) || null; + + if (aExt === "gif" && bExt !== "gif") { + return -1; + } + if (aExt !== "gif" && bExt === "gif") { + return 1; + } + return Math.max(b.width || 0, b.height || 0) - Math.max(a.width || 0, a.height || 0); +}; + +const mediaSorterMusicSong = (a: MusicSongObject, b: MusicSongObject) => { + if (!(a.track && b.track)) { + return 0; + } + if ((a.disc || 0) > (b.disc || 0)) { + return 1; + } + if ((a.disc || 0) < (b.disc || 0)) { + return -1; + } + return a.track - b.track; +}; + +// lodash zip replacement +const zip = (array: any, ...args: any) => { + if (array === undefined) return []; + return array.map((value: any, idx: number) => [value, ...args.map((arr: []) => arr[idx])]); +}; + +/** + * formats the multiple media values + * + * @param {object} ogObject - the current ogObject + * @param {object} options - options for ogs + * @return {object} object with ogs results with updated media values + * + */ +export function mediaSetup(ogObject: OgObjectInteral) { + // sets ogImage property/width/height/type to empty array if one these exists + if ( + ogObject.ogImageSecureURL || + ogObject.ogImageURL || + ogObject.ogImageProperty || + ogObject.ogImageWidth || + ogObject.ogImageHeight || + ogObject.ogImageType + ) { + ogObject.ogImageSecureURL = ogObject.ogImageSecureURL ? ogObject.ogImageSecureURL : []; + ogObject.ogImageURL = ogObject.ogImageURL ? ogObject.ogImageURL : []; + ogObject.ogImageProperty = ogObject.ogImageProperty ? ogObject.ogImageProperty : []; + // set ogImageProperty to ogImageSecureURL if it exists + // eslint-disable-next-line max-len + ogObject.ogImageProperty = + ogObject.ogImageSecureURL.length !== 0 ? ogObject.ogImageSecureURL : ogObject.ogImageProperty; + // fall back to ogImageURL if ogImageProperty isn't set + ogObject.ogImageProperty = ogObject.ogImageProperty.length !== 0 ? ogObject.ogImageProperty : ogObject.ogImageURL; + ogObject.ogImageWidth = ogObject.ogImageWidth ? ogObject.ogImageWidth : []; + ogObject.ogImageHeight = ogObject.ogImageHeight ? ogObject.ogImageHeight : []; + ogObject.ogImageType = ogObject.ogImageType ? ogObject.ogImageType : []; + } + + // format images and limit to 10 + const ogImages: ImageObject[] = zip( + ogObject.ogImageProperty, + ogObject.ogImageWidth, + ogObject.ogImageHeight, + ogObject.ogImageType + ) + .map(mediaMapper) + .filter((value: ImageObject) => value.url !== undefined && value.url !== "") + .filter((value: ImageObject, index: number) => index < 10) + .sort(mediaSorter); + + // sets ogVideo property/width/height/type to empty array if one these exists + if (ogObject.ogVideoProperty || ogObject.ogVideoWidth || ogObject.ogVideoHeight || ogObject.ogVideoType) { + ogObject.ogVideoProperty = ogObject.ogVideoProperty ? ogObject.ogVideoProperty : []; + ogObject.ogVideoWidth = ogObject.ogVideoWidth ? ogObject.ogVideoWidth : []; + ogObject.ogVideoHeight = ogObject.ogVideoHeight ? ogObject.ogVideoHeight : []; + ogObject.ogVideoType = ogObject.ogVideoType ? ogObject.ogVideoType : []; + } + + // format videos and limit to 10 + const ogVideos: VideoObject[] = zip( + ogObject.ogVideoProperty, + ogObject.ogVideoWidth, + ogObject.ogVideoHeight, + ogObject.ogVideoType + ) + .map(mediaMapper) + .filter((value: VideoObject) => value.url !== undefined && value.url !== "") + .filter((value: VideoObject, index: number) => index < 10) + .sort(mediaSorter); + + // sets twitter image src/property/width/height/alt to empty array if one these exists + if ( + ogObject.twitterImageSrc || + ogObject.twitterImageProperty || + ogObject.twitterImageWidth || + ogObject.twitterImageHeight || + ogObject.twitterImageAlt + ) { + ogObject.twitterImageSrc = ogObject.twitterImageSrc ? ogObject.twitterImageSrc : []; + // eslint-disable-next-line max-len + ogObject.twitterImageProperty = ogObject.twitterImageProperty + ? ogObject.twitterImageProperty + : ogObject.twitterImageSrc; // deafult to twitterImageSrc + ogObject.twitterImageWidth = ogObject.twitterImageWidth ? ogObject.twitterImageWidth : []; + ogObject.twitterImageHeight = ogObject.twitterImageHeight ? ogObject.twitterImageHeight : []; + ogObject.twitterImageAlt = ogObject.twitterImageAlt ? ogObject.twitterImageAlt : []; + } + + // format twitter images and limit to 10 + const twitterImages: TwitterImageObject[] = zip( + ogObject.twitterImageProperty, + ogObject.twitterImageWidth, + ogObject.twitterImageHeight, + ogObject.twitterImageAlt + ) + .map(mediaMapperTwitterImage) + .filter((value: TwitterImageObject) => value.url !== undefined && value.url !== "") + .filter((value: TwitterImageObject, index: number) => index < 10) + .sort(mediaSorter); + + // sets twitter property/width/height/stream to empty array if one these exists + if ( + ogObject.twitterPlayerProperty || + ogObject.twitterPlayerWidth || + ogObject.twitterPlayerHeight || + ogObject.twitterPlayerStream + ) { + ogObject.twitterPlayerProperty = ogObject.twitterPlayerProperty ? ogObject.twitterPlayerProperty : []; + ogObject.twitterPlayerWidth = ogObject.twitterPlayerWidth ? ogObject.twitterPlayerWidth : []; + ogObject.twitterPlayerHeight = ogObject.twitterPlayerHeight ? ogObject.twitterPlayerHeight : []; + ogObject.twitterPlayerStream = ogObject.twitterPlayerStream ? ogObject.twitterPlayerStream : []; + } + + // format twitter player and limit to 10 + const twitterPlayers: TwitterPlayerObject[] = zip( + ogObject.twitterPlayerProperty, + ogObject.twitterPlayerWidth, + ogObject.twitterPlayerHeight, + ogObject.twitterPlayerStream + ) + .map(mediaMapperTwitterPlayer) + .filter((value: TwitterPlayerObject) => value.url !== undefined && value.url !== "") + .filter((value: TwitterPlayerObject, index: number) => index < 10) + .sort(mediaSorter); + + // sets music property/songTrack/songDisc to empty array if one these exists + if (ogObject.musicSongProperty || ogObject.musicSongTrack || ogObject.musicSongDisc || ogObject.musicSongUrl) { + ogObject.musicSongUrl = ogObject.musicSongUrl ? ogObject.musicSongUrl : []; + ogObject.musicSongProperty = ogObject.musicSongProperty ? ogObject.musicSongProperty : ogObject.musicSongUrl; // deafult to musicSongUrl + ogObject.musicSongTrack = ogObject.musicSongTrack ? ogObject.musicSongTrack : []; + ogObject.musicSongDisc = ogObject.musicSongDisc ? ogObject.musicSongDisc : []; + } + + // format music songs and limit to 10 + const musicSongs: MusicSongObject[] = zip(ogObject.musicSongProperty, ogObject.musicSongTrack, ogObject.musicSongDisc) + .map(mediaMapperMusicSong) + .filter((value: MusicSongObject) => value.url !== undefined && value.url !== "") + .filter((value: MusicSongObject, index: number) => index < 10) + .sort(mediaSorterMusicSong); + + // remove old values since everything will live under the main property + fields + .filter((item) => item.multiple && item.fieldName && item.fieldName.match("(ogImage|ogVideo|twitter|musicSong).*")) + .forEach((item) => { + // @ts-ignore + delete ogObject[item.fieldName]; + }); + + if (ogImages.length) ogObject.ogImage = ogImages; + if (ogVideos.length) ogObject.ogVideo = ogVideos; + if (twitterImages.length) ogObject.twitterImage = twitterImages; + if (twitterPlayers.length) ogObject.twitterPlayer = twitterPlayers; + if (musicSongs.length) ogObject.musicSong = musicSongs; + + // removes any undefs + ogObject = removeNestedUndefinedValues(ogObject); + + return ogObject; +} + +export default mediaSetup; diff --git a/src/lib/open-graph-scraper/types.ts b/src/lib/open-graph-scraper/types.ts new file mode 100644 index 000000000..2a75371b5 --- /dev/null +++ b/src/lib/open-graph-scraper/types.ts @@ -0,0 +1,240 @@ +export type TwitterImageObject = { + alt?: string; + height?: number; + url: string; + width?: number; +}; + +export type TwitterPlayerObject = { + height?: number; + stream?: string; + url: string; + width?: number; +}; + +export type ImageObject = { + height?: number; + type: string; + url: string; + width?: number; +}; + +export type VideoObject = { + height?: number; + type?: string; + url: string; + width?: number; +}; + +export type MusicSongObject = { + disc?: string; + track?: number; + url: string; +}; + +export type OgObjectInteral = { + alAndroidAppName?: string; + alAndroidClass?: string; + alAndroidPackage?: string; + alAndroidUrl?: string; + alIosAppName?: string; + alIosAppStoreId?: string; + alIosUrl?: string; + alIpadAppName?: string; + alIpadAppStoreId?: string; + alIpadUrl?: string; + alIphoneAppName?: string; + alIphoneAppStoreId?: string; + alIphoneUrl?: string; + alWebShouldFallback?: string; + alWebUrl?: string; + alWindowsAppId?: string; + alWindowsAppName?: string; + alWindowsPhoneAppId?: string; + alWindowsPhoneAppName?: string; + alWindowsPhoneUrl?: string; + alWindowsUniversalAppId?: string; + alWindowsUniversalAppName?: string; + alWindowsUniversalUrl?: string; + alWindowsUrl?: string; + articleAuthor?: string; + articleExpirationTime?: string; + articleModifiedTime?: string; + articlePublishedTime?: string; + articlePublisher?: string; + articleSection?: string; + articleTag?: string; + author?: string; + bookAuthor?: string; + bookCanonicalName?: string; + bookIsbn?: string; + bookReleaseDate?: string; + booksBook?: string; + booksRatingScale?: string; + booksRatingValue?: string; + bookTag?: string; + businessContactDataCountryName?: string; + businessContactDataLocality?: string; + businessContactDataPostalCode?: string; + businessContactDataRegion?: string; + businessContactDataStreetAddress?: string; + charset?: string; + dcContributor?: string; + dcCoverage?: string; + dcCreator?: string; + dcDate?: string; + dcDateCreated?: string; + dcDateIssued?: string; + dcDescription?: string; + dcFormatMedia?: string; + dcFormatSize?: string; + dcIdentifier?: string; + dcLanguage?: string; + dcPublisher?: string; + dcRelation?: string; + dcRights?: string; + dcSource?: string; + dcSubject?: string; + dcTitle?: string; + dcType?: string; + error?: string; + errorDetails?: Error; + favicon?: string; + modifiedTime?: string; + musicAlbum?: string; + musicAlbumDisc?: string; + musicAlbumTrack?: string; + musicAlbumUrl?: string; + musicCreator?: string; + musicDuration?: string; + musicMusician?: string; + musicReleaseDate?: string; + musicSong?: MusicSongObject[]; + musicSongDisc?: string | null[]; + musicSongProperty?: string | null[]; + musicSongTrack?: number | null[]; + musicSongUrl?: string | null[]; + ogArticleAuthor?: string; + ogArticleExpirationTime?: string; + ogArticleModifiedTime?: string; + ogArticlePublishedTime?: string; + ogArticlePublisher?: string; + ogArticleSection?: string; + ogArticleTag?: string; + ogAudio?: string; + ogAudioSecureURL?: string; + ogAudioType?: string; + ogAudioURL?: string; + ogAvailability?: string; + ogDate?: string; + ogDescription?: string; + ogDeterminer?: string; + ogImage?: ImageObject[]; + ogImageHeight?: string | null[]; + ogImageProperty?: string | null[]; + ogImageSecureURL?: string | null[]; + ogImageType?: string | null[]; + ogImageURL?: string | null[]; + ogImageWidth?: string | null[]; + ogLocale?: string; + ogLocaleAlternate?: string; + ogLogo?: string; + ogPriceAmount?: string; + ogPriceCurrency?: string; + ogProductAvailability?: string; + ogProductCondition?: string; + ogProductPriceAmount?: string; + ogProductPriceCurrency?: string; + ogProductRetailerItemId?: string; + ogSiteName?: string; + ogTitle?: string; + ogType?: string; + ogUrl?: string; + ogVideo?: VideoObject[]; + ogVideoActorId?: string; + ogVideoHeight?: string | null[]; + ogVideoProperty?: string | null[]; + ogVideoSecureURL?: string; + ogVideoType?: string | null[]; + ogVideoWidth?: string | null[]; + placeLocationLatitude?: string; + placeLocationLongitude?: string; + profileFirstName?: string; + profileGender?: string; + profileLastName?: string; + profileUsername?: string; + publishedTime?: string; + releaseDate?: string; + requestUrl?: string; + restaurantContactInfoCountryName?: string; + restaurantContactInfoEmail?: string; + restaurantContactInfoLocality?: string; + restaurantContactInfoPhoneNumber?: string; + restaurantContactInfoPostalCode?: string; + restaurantContactInfoRegion?: string; + restaurantContactInfoStreetAddress?: string; + restaurantContactInfoWebsite?: string; + restaurantMenu?: string; + restaurantRestaurant?: string; + restaurantSection?: string; + restaurantVariationPriceAmount?: string; + restaurantVariationPriceCurrency?: string; + success?: boolean; + twitterAppIdGooglePlay?: string; + twitterAppIdiPad?: string; + twitterAppIdiPhone?: string; + twitterAppNameGooglePlay?: string; + twitterAppNameiPad?: string; + twitterAppNameiPhone?: string; + twitterAppUrlGooglePlay?: string; + twitterAppUrliPad?: string; + twitterAppUrliPhone?: string; + twitterCard?: string; + twitterCreator?: string; + twitterCreatorId?: string; + twitterDescription?: string; + twitterImage?: TwitterImageObject[]; + twitterImageAlt?: string | null[]; + twitterImageHeight?: string | null[]; + twitterImageProperty?: string | null[]; + twitterImageSrc?: string | null[]; + twitterImageWidth?: string | null[]; + twitterPlayer?: TwitterPlayerObject[]; + twitterPlayerHeight?: string | null[]; + twitterPlayerProperty?: string | null[]; + twitterPlayerStream?: string | null[]; + twitterPlayerStreamContentType?: string; + twitterPlayerWidth?: string | null[]; + twitterSite?: string; + twitterSiteId?: string; + twitterTitle?: string; + twitterUrl?: string; + updatedTime?: string; +}; + +export type OgObject = Omit< + OgObjectInteral, + | "musicSongDisc" + | "musicSongProperty" + | "musicSongTrack" + | "musicSongUrl" + | "ogImageHeight" + | "ogImageProperty" + | "ogImageSecureURL" + | "ogImageType" + | "ogImageURL" + | "ogImageWidth" + | "ogVideoHeight" + | "ogVideoProperty" + | "ogVideoType" + | "ogVideoWidth" + | "twitterImageAlt" + | "twitterImageHeight" + | "twitterImageProperty" + | "twitterImageSrc" + | "twitterImageWidth" + | "twitterPlayerHeight" + | "twitterPlayerProperty" + | "twitterPlayerStream" + | "twitterPlayerWidth" +>; diff --git a/src/lib/open-graph-scraper/utils.ts b/src/lib/open-graph-scraper/utils.ts new file mode 100644 index 000000000..690a32e67 --- /dev/null +++ b/src/lib/open-graph-scraper/utils.ts @@ -0,0 +1,139 @@ +/** + * Checks if URL is valid + * + * @param {string} url - url to be checked + * @return {boolean} boolean value if the url is valid + * + */ +export function isUrlValid(url: string): boolean { + try { + new URL(url); + return true; + } catch (e) {} + return false; +} + +/** + * Forces url to start with http:// if it doesn't + * + * @param {string} url - url to be updated + * @return {string} url that starts with http + * + */ +const coerceUrl = (url: string): string => (/^(f|ht)tps?:\/\//i.test(url) ? url : `http://${url}`); + +/** + * Validates and formats url + * + * @param {string} url - url to be checked and formatted + * @return {string} proper url or null + * + */ +export function validateAndFormatURL(url: string): { url: string | null } { + return { url: isUrlValid(url) ? coerceUrl(url) : null }; +} + +/** + * Finds the image type from a given url + * + * @param {string} url - url to be checked + * @return {string} image type from url + * + */ +export function findImageTypeFromUrl(url: string): string { + let type: string = url.split(".").pop() || ""; + [type] = type.split("?"); + return type; +} + +/** + * Checks if image type is valid + * + * @param {string} type - type to be checked + * @return {boolean} boolean value if type is value + * + */ +export function isImageTypeValid(type: string): boolean { + const validImageTypes: string[] = [ + "apng", + "bmp", + "gif", + "ico", + "cur", + "jpg", + "jpeg", + "jfif", + "pjpeg", + "pjp", + "png", + "svg", + "tif", + "tiff", + "webp", + ]; + return validImageTypes.includes(type); +} + +/** + * Checks if URL is a non html page + * + * @param {string} url - url to be checked + * @return {boolean} boolean value if url is non html + * + */ +export function isThisANonHTMLUrl(url: string): boolean { + const invalidImageTypes: string[] = [ + ".doc", + ".docx", + ".xls", + ".xlsx", + ".ppt", + ".pptx", + ".3gp", + ".avi", + ".mov", + ".mp4", + ".m4v", + ".m4a", + ".mp3", + ".mkv", + ".ogv", + ".ogm", + ".ogg", + ".oga", + ".webm", + ".wav", + ".bmp", + ".gif", + ".jpg", + ".jpeg", + ".png", + ".webp", + ".zip", + ".rar", + ".tar", + ".tar.gz", + ".tgz", + ".tar.bz2", + ".tbz2", + ".txt", + ".pdf", + ]; + const extension: string = findImageTypeFromUrl(url); + return invalidImageTypes.some((type: string): boolean => `.${extension}`.includes(type)); +} + +/** + * Find and delete nested undefs + * + * @param {object} object - object to be cleaned + * @return {object} object without nested undefs + * + */ +export function removeNestedUndefinedValues(object: { [key: string]: any }): { [key: string]: any } { + Object.entries(object).forEach(([key, value]) => { + if (value && typeof value === "object") removeNestedUndefinedValues(value); + else if (value === undefined) delete object[key]; + }); + return object; +} diff --git a/src/services/dns-identity.ts b/src/services/dns-identity.ts index a3f53601b..a9e8babe0 100644 --- a/src/services/dns-identity.ts +++ b/src/services/dns-identity.ts @@ -1,5 +1,6 @@ import dayjs from "dayjs"; import db from "./db"; +import { fetchWithCorsFallback } from "../helpers/cors"; function parseAddress(address: string): { name?: string; domain?: string } { const parts = address.trim().toLowerCase().split("@"); @@ -26,7 +27,9 @@ function getIdentityFromJson(name: string, domain: string, json: IdentityJson): } async function fetchAllIdentities(domain: string) { - const json = await fetch(`//${domain}/.well-known/nostr.json`).then((res) => res.json() as Promise); + const json = await fetchWithCorsFallback(`//${domain}/.well-known/nostr.json`).then( + (res) => res.json() as Promise + ); await addToCache(domain, json); } @@ -35,7 +38,7 @@ async function fetchIdentity(address: string) { const { name, domain } = parseAddress(address); if (!name || !domain) throw new Error("invalid address"); - const json = await fetch(`https://${domain}/.well-known/nostr.json?name=${name}`) + const json = await fetchWithCorsFallback(`https://${domain}/.well-known/nostr.json?name=${name}`) .then((res) => res.json() as Promise) .then((json) => { // convert all keys in names, and relays to lower case diff --git a/src/services/user-app-settings.ts b/src/services/user-app-settings.ts index 97e6285a7..bd932acf1 100644 --- a/src/services/user-app-settings.ts +++ b/src/services/user-app-settings.ts @@ -26,6 +26,7 @@ export type AppSettings = { zapAmounts: number[]; primaryColor: string; imageProxy: string; + corsProxy: string; showContentWarning: boolean; twitterRedirect?: string; redditRedirect?: string; @@ -43,6 +44,7 @@ export const defaultSettings: AppSettings = { zapAmounts: [50, 200, 500, 1000], primaryColor: "#8DB600", imageProxy: "", + corsProxy: "", showContentWarning: true, twitterRedirect: undefined, redditRedirect: undefined, diff --git a/src/views/settings/privacy-settings.tsx b/src/views/settings/privacy-settings.tsx index 5b7558fc9..98a187846 100644 --- a/src/views/settings/privacy-settings.tsx +++ b/src/views/settings/privacy-settings.tsx @@ -29,7 +29,7 @@ async function validateInvidiousUrl(url?: string) { } export default function PrivacySettings() { - const { youtubeRedirect, twitterRedirect, redditRedirect, updateSettings } = useAppSettings(); + const { youtubeRedirect, twitterRedirect, redditRedirect, corsProxy, updateSettings } = useAppSettings(); const { register, handleSubmit, formState } = useForm({ mode: "onBlur", @@ -37,6 +37,7 @@ export default function PrivacySettings() { youtubeRedirect, twitterRedirect, redditRedirect, + corsProxy, }, }); @@ -113,6 +114,19 @@ export default function PrivacySettings() { + + CORS Proxy + + {formState.errors.corsProxy && {formState.errors.corsProxy.message}} + + This is used as a fallback when verifying NIP-05 ids and fetching open-graph metadata. URL to an + instance of{" "} + + cors-anywhere + + + +