highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-10-09 20:55:06 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix web scrolling

Browse Source
This commit is contained in:
Yuhong Sun
2025-03-03 09:00:05 -08:00
parent 7f0653d173
commit 39fd6919ad
1 changed files with 7 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
backend/onyx/connectors/web/connector.py
View File

@@ -16,7 +16,6 @@ from oauthlib.oauth2 import BackendApplicationClient
from playwright.sync_api import BrowserContext from playwright.sync_api import BrowserContext
from playwright.sync_api import Playwright from playwright.sync_api import Playwright
from playwright.sync_api import sync_playwright from playwright.sync_api import sync_playwright
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
from requests_oauthlib import OAuth2Session # type:ignore from requests_oauthlib import OAuth2Session # type:ignore
from urllib3.exceptions import MaxRetryError from urllib3.exceptions import MaxRetryError
@@ -354,19 +353,13 @@ class WebConnector(LoadConnector):
continue continue
page = context.new_page() page = context.new_page()
# wait_until="networkidle" is used to wait for the page to load completely which is necessary
# for the javascript heavy websites # Can't use wait_until="networkidle" because it interferes with the scrolling behavior
try: page_response = page.goto(
page_response = page.goto( initial_url,
initial_url, timeout=30000, # 30 seconds
wait_until="networkidle", )
timeout=30000, # 30 seconds
)
except PlaywrightTimeoutError:
logger.warning(
f"NetworkIdle timeout for {initial_url}, falling back to default load"
)
page_response = page.goto(initial_url)
last_modified = ( last_modified = (
page_response.header_value("Last-Modified") page_response.header_value("Last-Modified")
if page_response if page_response