tygod for tests

This commit is contained in:
hagen-danswer
2024-12-06 08:03:34 -08:00
parent 7c6981e052
commit 32ec4efc7a
3 changed files with 11 additions and 7 deletions

View File

@@ -13,9 +13,9 @@ from danswer.connectors.confluence.onyx_confluence import build_confluence_clien
from danswer.connectors.confluence.onyx_confluence import OnyxConfluence
from danswer.connectors.confluence.utils import attachment_to_content
from danswer.connectors.confluence.utils import build_confluence_document_id
from danswer.connectors.confluence.utils import check_attachment_filetype
from danswer.connectors.confluence.utils import datetime_from_string
from danswer.connectors.confluence.utils import extract_text_from_confluence_html
from danswer.connectors.confluence.utils import validate_attachment_filetype
from danswer.connectors.interfaces import GenerateDocumentsOutput
from danswer.connectors.interfaces import GenerateSlimDocumentOutput
from danswer.connectors.interfaces import LoadConnector
@@ -299,7 +299,7 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
expand=restrictions_expand,
limit=_SLIM_DOC_BATCH_SIZE,
):
if not check_attachment_filetype(attachment):
if not validate_attachment_filetype(attachment):
continue
doc_metadata_list.append(
SlimDocument(

View File

@@ -177,8 +177,8 @@ def extract_text_from_confluence_html(
return format_document_soup(soup)
def check_attachment_filetype(attachment: dict[str, Any]) -> bool:
return attachment["metadata"]["mediaType"] in [
def validate_attachment_filetype(attachment: dict[str, Any]) -> bool:
return attachment["metadata"]["mediaType"] not in [
"image/jpeg",
"image/png",
"image/gif",
@@ -193,7 +193,7 @@ def attachment_to_content(
attachment: dict[str, Any],
) -> str | None:
"""If it returns None, assume that we should skip this attachment."""
if not check_attachment_filetype(attachment):
if not validate_attachment_filetype(attachment):
return None
download_link = confluence_client.url + attachment["_links"]["download"]

View File

@@ -95,6 +95,7 @@ def _get_cloud_space_permissions(
def _get_space_permissions(
confluence_client: OnyxConfluence,
is_cloud: bool,
) -> dict[str, ExternalAccess]:
logger.debug("Getting space permissions")
# Gets all the spaces in the Confluence instance
@@ -116,7 +117,7 @@ def _get_space_permissions(
logger.debug(f"Got {len(all_space_keys)} spaces from confluence")
space_permissions_by_space_key: dict[str, ExternalAccess] = {}
for space_key in all_space_keys:
if confluence_client.cloud:
if is_cloud:
space_permissions = _get_cloud_space_permissions(
confluence_client=confluence_client, space_key=space_key
)
@@ -264,8 +265,11 @@ def confluence_doc_sync(
)
confluence_connector.load_credentials(cc_pair.credential.credential_json)
is_cloud = cc_pair.connector.connector_specific_config.get("is_cloud", False)
space_permissions_by_space_key = _get_space_permissions(
confluence_client=confluence_connector.confluence_client
confluence_client=confluence_connector.confluence_client,
is_cloud=is_cloud,
)
slim_docs = []