highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-08-03 21:52:29 +02:00
Code Issues Packages Projects Releases Wiki Activity

Double Check Files/URLs (#1344)

Browse Source
This commit is contained in:
Yuhong Sun
2024-04-17 21:15:16 -07:00
committed by Weves
parent 26f8d884e1
commit be12e4fa64
1 changed files with 15 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
backend/danswer/connectors/web/connector.py
View File

@@ -42,6 +42,19 @@ class WEB_CONNECTOR_VALID_SETTINGS(str, Enum):
UPLOAD = "upload"
def protected_url_check(url: str) -> None:
parse = urlparse(url)
if parse.scheme == "file":
raise ValueError("Not permitted to read local files via Web Connector.")
if (
parse.scheme == "localhost"
or parse.scheme == "127.0.0.1"
or parse.hostname == "localhost"
or parse.hostname == "127.0.0.1"
):
raise ValueError("Not permitted to read localhost urls.")
def check_internet_connection(url: str) -> None:
try:
response = requests.get(url, timeout=3)
@@ -189,6 +202,8 @@ class WebConnector(LoadConnector):
continue
visited_links.add(current_url)
protected_url_check(current_url)
logger.info(f"Visiting {current_url}")
try: