highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-06-28 17:01:10 +02:00
Code Issues Packages Projects Releases Wiki Activity

connector: ensure absolute URL integrity (#1196)

Browse Source
This commit is contained in:
teocns 2024-03-10 03:04:05 -06:00 committed by GitHub
parent f292ede85a
commit 19c7ebdc26
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/danswer/connectors/web/connector.py
View File

@ -100,10 +100,12 @@ def extract_urls_from_sitemap(sitemap_url: str) -> list[str]:
response.raise_for_status() response.raise_for_status()
soup = BeautifulSoup(response.content, "html.parser") soup = BeautifulSoup(response.content, "html.parser")
urls = [loc_tag.text for loc_tag in soup.find_all("loc")] return [_ensure_absolute_url(sitemap_url, loc_tag.text) for loc_tag in soup.find_all("loc")]
return urls
def _ensure_absolute_url(source_url:str, maybe_relative_url: str) -> str:
if not urlparse(maybe_relative_url).netloc:
return urljoin(source_url, maybe_relative_url)
return maybe_relative_url
def _ensure_valid_url(url: str) -> str: def _ensure_valid_url(url: str) -> str:
if "://" not in url: if "://" not in url: