highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-06-01 10:40:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

Added filter to slim connector and logging for space permissions

Browse Source
This commit is contained in:
hagen-danswer 2024-12-06 07:55:54 -08:00
parent c50cd20156
commit 7c6981e052
3 changed files with 20 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/danswer/connectors/confluence/connector.py
View File

@ -13,6 +13,7 @@ from danswer.connectors.confluence.onyx_confluence import build_confluence_clien
from danswer.connectors.confluence.onyx_confluence import OnyxConfluence
from danswer.connectors.confluence.utils import attachment_to_content
from danswer.connectors.confluence.utils import build_confluence_document_id
from danswer.connectors.confluence.utils import check_attachment_filetype
from danswer.connectors.confluence.utils import datetime_from_string
from danswer.connectors.confluence.utils import extract_text_from_confluence_html
from danswer.connectors.interfaces import GenerateDocumentsOutput
@ -298,6 +299,8 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
expand=restrictions_expand,
limit=_SLIM_DOC_BATCH_SIZE,
):
if not check_attachment_filetype(attachment):
continue
doc_metadata_list.append(
SlimDocument(
id=build_confluence_document_id(

20
backend/danswer/connectors/confluence/utils.py
View File

@ -177,19 +177,23 @@ def extract_text_from_confluence_html(
return format_document_soup(soup)
def attachment_to_content(
confluence_client: OnyxConfluence,
attachment: dict[str, Any],
) -> str | None:
"""If it returns None, assume that we should skip this attachment."""
if attachment["metadata"]["mediaType"] in [
def check_attachment_filetype(attachment: dict[str, Any]) -> bool:
return attachment["metadata"]["mediaType"] in [
"image/jpeg",
"image/png",
"image/gif",
"image/svg+xml",
"video/mp4",
"video/quicktime",
]:
]
def attachment_to_content(
confluence_client: OnyxConfluence,
attachment: dict[str, Any],
) -> str | None:
"""If it returns None, assume that we should skip this attachment."""
if not check_attachment_filetype(attachment):
return None
download_link = confluence_client.url + attachment["_links"]["download"]
@ -245,7 +249,7 @@ def build_confluence_document_id(
return f"{base_url}{content_url}"
def extract_referenced_attachment_names(page_text: str) -> list[str]:
def _extract_referenced_attachment_names(page_text: str) -> list[str]:
"""Parse a Confluence html page to generate a list of current
attachments in use

12
backend/ee/danswer/external_permissions/confluence/doc_sync.py
View File

@ -95,7 +95,6 @@ def _get_cloud_space_permissions(
def _get_space_permissions(
confluence_client: OnyxConfluence,
is_cloud: bool,
) -> dict[str, ExternalAccess]:
logger.debug("Getting space permissions")
# Gets all the spaces in the Confluence instance
@ -117,7 +116,7 @@ def _get_space_permissions(
logger.debug(f"Got {len(all_space_keys)} spaces from confluence")
space_permissions_by_space_key: dict[str, ExternalAccess] = {}
for space_key in all_space_keys:
if is_cloud:
if confluence_client.cloud:
space_permissions = _get_cloud_space_permissions(
confluence_client=confluence_client, space_key=space_key
)
@ -242,7 +241,9 @@ def _fetch_all_page_restrictions_for_space(
)
continue
logger.warning(f"No permissions found for document {slim_doc.id}")
logger.warning(
f"No permissions found for document {slim_doc.id} in space {space_key}"
)
logger.debug("Finished fetching all page restrictions for space")
return document_restrictions
@ -263,11 +264,8 @@ def confluence_doc_sync(
)
confluence_connector.load_credentials(cc_pair.credential.credential_json)
is_cloud = cc_pair.connector.connector_specific_config.get("is_cloud", False)
space_permissions_by_space_key = _get_space_permissions(
confluence_client=confluence_connector.confluence_client,
is_cloud=is_cloud,
confluence_client=confluence_connector.confluence_client
)
slim_docs = []