highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-10-02 17:38:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

Increased slim doc batch size for confluence connector (#3221)

Browse Source
This commit is contained in:
hagen-danswer
2024-11-22 16:42:15 -08:00
committed by GitHub
parent 9dc23bf3e7
commit 85cbd9caed
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/danswer/connectors/confluence/connector.py
View File

@@ -51,6 +51,8 @@ _RESTRICTIONS_EXPANSION_FIELDS = [
"restrictions.read.restrictions.group", "restrictions.read.restrictions.group",
] ]
_SLIM_DOC_BATCH_SIZE = 1000
class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector): class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
def __init__( def __init__(
@@ -263,6 +265,7 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
for page in self.confluence_client.cql_paginate_all_expansions( for page in self.confluence_client.cql_paginate_all_expansions(
cql=page_query, cql=page_query,
expand=restrictions_expand, expand=restrictions_expand,
limit=_SLIM_DOC_BATCH_SIZE,
): ):
# If the page has restrictions, add them to the perm_sync_data # If the page has restrictions, add them to the perm_sync_data
# These will be used by doc_sync.py to sync permissions # These will be used by doc_sync.py to sync permissions
@@ -286,6 +289,7 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
for attachment in self.confluence_client.cql_paginate_all_expansions( for attachment in self.confluence_client.cql_paginate_all_expansions(
cql=attachment_cql, cql=attachment_cql,
expand=restrictions_expand, expand=restrictions_expand,
limit=_SLIM_DOC_BATCH_SIZE,
): ):
doc_metadata_list.append( doc_metadata_list.append(
SlimDocument( SlimDocument(