Implement source testing framework + Slack (#2650)

* Added permission sync tests for Slack

* moved folders

* prune test + mypy

* added wait for indexing to cc_pair creation

* commented out check

* should fix other tests

* added slack channel pool

* fixed everything and mypy

* reduced flake

backend/ee/danswer/background/celery_utils.py

@@ -1,3 +1,6 @@
+from datetime import datetime
+from datetime import timezone
+
 from sqlalchemy.orm import Session
 
 from danswer.db.enums import AccessType
@@ -12,10 +15,32 @@ from ee.danswer.background.task_name_builders import (
 from ee.danswer.background.task_name_builders import (
     name_sync_external_group_permissions_task,
 )
+from ee.danswer.external_permissions.sync_params import PERMISSION_SYNC_PERIODS
 
 logger = setup_logger()
 
 
+def _is_time_to_run_sync(cc_pair: ConnectorCredentialPair) -> bool:
+    source_sync_period = PERMISSION_SYNC_PERIODS.get(cc_pair.connector.source)
+
+    # If RESTRICTED_FETCH_PERIOD[source] is None, we always run the sync.
+    if not source_sync_period:
+        return True
+
+    # If the last sync is None, it has never been run so we run the sync
+    if cc_pair.last_time_perm_sync is None:
+        return True
+
+    last_sync = cc_pair.last_time_perm_sync.replace(tzinfo=timezone.utc)
+    current_time = datetime.now(timezone.utc)
+
+    # If the last sync is greater than the full fetch period, we run the sync
+    if (current_time - last_sync).total_seconds() > source_sync_period:
+        return True
+
+    return False
+
+
 def should_perform_chat_ttl_check(
     retention_limit_days: int | None, db_session: Session
 ) -> bool:
@@ -28,7 +53,7 @@ def should_perform_chat_ttl_check(
     if not latest_task:
         return True
 
-    if latest_task and check_task_is_live_and_not_timed_out(latest_task, db_session):
+    if check_task_is_live_and_not_timed_out(latest_task, db_session):
         logger.debug(f"{task_name} is already being performed. Skipping.")
         return False
     return True
@@ -50,6 +75,9 @@ def should_perform_external_doc_permissions_check(
         logger.debug(f"{task_name} is already being performed. Skipping.")
         return False
 
+    if not _is_time_to_run_sync(cc_pair):
+        return False
+
     return True
 
 
@@ -69,4 +97,7 @@ def should_perform_external_group_permissions_check(
         logger.debug(f"{task_name} is already being performed. Skipping.")
         return False
 
+    if not _is_time_to_run_sync(cc_pair):
+        return False
+
     return True

backend/ee/danswer/external_permissions/permission_sync.py

@@ -6,38 +6,15 @@ from sqlalchemy.orm import Session
 from danswer.access.access import get_access_for_documents
 from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
 from danswer.db.document import get_document_ids_for_connector_credential_pair
-from danswer.db.models import ConnectorCredentialPair
 from danswer.document_index.factory import get_current_primary_default_document_index
 from danswer.document_index.interfaces import UpdateRequest
 from danswer.utils.logger import setup_logger
 from ee.danswer.external_permissions.sync_params import DOC_PERMISSIONS_FUNC_MAP
 from ee.danswer.external_permissions.sync_params import GROUP_PERMISSIONS_FUNC_MAP
-from ee.danswer.external_permissions.sync_params import PERMISSION_SYNC_PERIODS
 
 logger = setup_logger()
 
 
-def _is_time_to_run_sync(cc_pair: ConnectorCredentialPair) -> bool:
-    source_sync_period = PERMISSION_SYNC_PERIODS.get(cc_pair.connector.source)
-
-    # If RESTRICTED_FETCH_PERIOD[source] is None, we always run the sync.
-    if not source_sync_period:
-        return True
-
-    # If the last sync is None, it has never been run so we run the sync
-    if cc_pair.last_time_perm_sync is None:
-        return True
-
-    last_sync = cc_pair.last_time_perm_sync.replace(tzinfo=timezone.utc)
-    current_time = datetime.now(timezone.utc)
-
-    # If the last sync is greater than the full fetch period, we run the sync
-    if (current_time - last_sync).total_seconds() > source_sync_period:
-        return True
-
-    return False
-
-
 def run_external_group_permission_sync(
     db_session: Session,
     cc_pair_id: int,
@@ -53,9 +30,6 @@ def run_external_group_permission_sync(
         # Not all sync connectors support group permissions so this is fine
         return
 
-    if not _is_time_to_run_sync(cc_pair):
-        return
-
     try:
         # This function updates:
         # - the user_email <-> external_user_group_id mapping
@@ -91,9 +65,6 @@ def run_external_doc_permission_sync(
             f"No permission sync function found for source type: {source_type}"
         )
 
-    if not _is_time_to_run_sync(cc_pair):
-        return
-
     try:
         # This function updates:
         # - the user_email <-> document mapping