highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-04-07 11:28:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix issue with Confluence errors not being ignored

Browse Source
This commit is contained in:
Weves 2023-08-21 08:47:53 -07:00 committed by Chris Weaver
parent 9f1898c384
commit e5352b6af8
1 changed files with 14 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
backend/danswer/connectors/confluence/connector.py
View File

@ -3,6 +3,7 @@ from collections.abc import Collection
from datetime import datetime
from datetime import timezone
from typing import Any
from typing import cast
from urllib.parse import urlparse
from atlassian import Confluence # type:ignore
@ -125,17 +126,19 @@ class ConfluenceConnector(LoadConnector, PollConnector):
return pages
def _fetch_comments(
self, confluence_client: Confluence, page_id: str
) -> Collection[dict[str, Any]]:
def _fetch_comments(self, confluence_client: Confluence, page_id: str) -> str:
try:
return confluence_client.get_page_child_by_type(
page_id,
type="comment",
start=None,
limit=None,
expand="body.storage.value",
comment_pages = cast(
Collection[dict[str, Any]],
confluence_client.get_page_child_by_type(
page_id,
type="comment",
start=None,
limit=None,
expand="body.storage.value",
),
)
return _comment_dfs("", comment_pages, confluence_client)
except Exception as e:
if not self.continue_on_failure:
raise e
@ -143,7 +146,7 @@ class ConfluenceConnector(LoadConnector, PollConnector):
logger.exception(
"Ran into exception when fetching comments from Confluence"
)
return []
return ""
def _get_doc_batch(
self, start_ind: int, time_filter: Callable[[datetime], bool] | None = None
@ -163,8 +166,7 @@ class ConfluenceConnector(LoadConnector, PollConnector):
page_text = (
page.get("title", "") + "\n" + parse_html_page_basic(page_html)
)
comment_pages = self._fetch_comments(self.confluence_client, page["id"])
comments_text = _comment_dfs("", comment_pages, self.confluence_client)
comments_text = self._fetch_comments(self.confluence_client, page["id"])
page_text += comments_text
page_url = self.wiki_base + page["_links"]["webui"]