diff --git a/backend/ee/onyx/external_permissions/sync_params.py b/backend/ee/onyx/external_permissions/sync_params.py index fa063c171..7c50273fd 100644 --- a/backend/ee/onyx/external_permissions/sync_params.py +++ b/backend/ee/onyx/external_permissions/sync_params.py @@ -50,6 +50,12 @@ DOC_PERMISSIONS_FUNC_MAP: dict[DocumentSource, DocSyncFuncType] = { DocumentSource.GMAIL: gmail_doc_sync, } + +def source_requires_doc_sync(source: DocumentSource) -> bool: + """Checks if the given DocumentSource requires doc syncing.""" + return source in DOC_PERMISSIONS_FUNC_MAP + + # These functions update: # - the user_email <-> external_user_group_id mapping # in postgres without committing @@ -61,6 +67,11 @@ GROUP_PERMISSIONS_FUNC_MAP: dict[DocumentSource, GroupSyncFuncType] = { } +def source_requires_external_group_sync(source: DocumentSource) -> bool: + """Checks if the given DocumentSource requires external group syncing.""" + return source in GROUP_PERMISSIONS_FUNC_MAP + + GROUP_PERMISSIONS_IS_CC_PAIR_AGNOSTIC: set[DocumentSource] = { DocumentSource.CONFLUENCE, } diff --git a/backend/onyx/background/indexing/run_indexing.py b/backend/onyx/background/indexing/run_indexing.py index d7af55aab..16756368b 100644 --- a/backend/onyx/background/indexing/run_indexing.py +++ b/backend/onyx/background/indexing/run_indexing.py @@ -407,8 +407,12 @@ def _run_indexing( # don't use a checkpoint if we're explicitly indexing from # the beginning in order to avoid weird interactions between - # checkpointing / failure handling. - if index_attempt.from_beginning: + # checkpointing / failure handling + # OR + # if the last attempt was successful + if index_attempt.from_beginning or ( + most_recent_attempt and most_recent_attempt.status.is_successful() + ): checkpoint = connector_runner.connector.build_dummy_checkpoint() else: checkpoint = get_latest_valid_checkpoint( diff --git a/backend/onyx/server/documents/models.py b/backend/onyx/server/documents/models.py index 653c077d3..365094b35 100644 --- a/backend/onyx/server/documents/models.py +++ b/backend/onyx/server/documents/models.py @@ -1,5 +1,6 @@ from datetime import datetime from datetime import timezone +from datetime import UTC from typing import Any from typing import Generic from typing import TypeVar @@ -25,6 +26,7 @@ from onyx.db.models import TaskStatus from onyx.server.models import FullUserSnapshot from onyx.server.models import InvitedUserSnapshot from onyx.server.utils import mask_credential_dict +from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop class DocumentSyncStatus(BaseModel): @@ -227,10 +229,49 @@ class CCPairFullInfo(BaseModel): # information on syncing/indexing last_indexed: datetime | None last_pruned: datetime | None - last_permission_sync: datetime | None + # accounts for both doc sync and group sync + last_full_permission_sync: datetime | None overall_indexing_speed: float | None latest_checkpoint_description: str | None + @classmethod + def _get_last_full_permission_sync( + cls, cc_pair_model: ConnectorCredentialPair + ) -> datetime | None: + check_if_source_requires_external_group_sync = fetch_ee_implementation_or_noop( + "onyx.external_permissions.sync_params", + "source_requires_external_group_sync", + noop_return_value=False, + ) + check_if_source_requires_doc_sync = fetch_ee_implementation_or_noop( + "onyx.external_permissions.sync_params", + "source_requires_doc_sync", + noop_return_value=False, + ) + + needs_group_sync = check_if_source_requires_external_group_sync( + cc_pair_model.connector.source + ) + needs_doc_sync = check_if_source_requires_doc_sync( + cc_pair_model.connector.source + ) + + last_group_sync = ( + cc_pair_model.last_time_external_group_sync + if needs_group_sync + else datetime.now(UTC) + ) + last_doc_sync = ( + cc_pair_model.last_time_perm_sync if needs_doc_sync else datetime.now(UTC) + ) + + # if either is still None at this point, it means sync is necessary but + # has never completed. + if last_group_sync is None or last_doc_sync is None: + return None + + return min(last_group_sync, last_doc_sync) + @classmethod def from_models( cls, @@ -292,9 +333,7 @@ class CCPairFullInfo(BaseModel): last_index_attempt.time_started if last_index_attempt else None ), last_pruned=cc_pair_model.last_pruned, - last_permission_sync=( - last_index_attempt.time_started if last_index_attempt else None - ), + last_full_permission_sync=cls._get_last_full_permission_sync(cc_pair_model), overall_indexing_speed=overall_indexing_speed, latest_checkpoint_description=None, ) diff --git a/web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx b/web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx index 35e770334..2ebe86b47 100644 --- a/web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx +++ b/web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx @@ -24,7 +24,7 @@ function convertObjectToString(obj: any): string | any { return obj; } -function buildConfigEntries( +export function buildConfigEntries( obj: any, sourceType: ValidSources ): { [key: string]: string } { @@ -189,24 +189,15 @@ export function AdvancedConfigDisplay({ } export function ConfigDisplay({ - connectorSpecificConfig, - sourceType, + configEntries, onEdit, }: { - connectorSpecificConfig: any; - sourceType: ValidSources; + configEntries: { [key: string]: string }; onEdit?: (key: string) => void; }) { - const configEntries = Object.entries( - buildConfigEntries(connectorSpecificConfig, sourceType) - ); - if (!configEntries.length) { - return null; - } - return (