From 8eb4320f76f2f4f698fd46c5703e858ebc6e67d8 Mon Sep 17 00:00:00 2001
From: Weves <chrisweaver101@gmail.com>
Date: Tue, 4 Feb 2025 17:44:59 -0800
Subject: [PATCH] Support not pausing connectors on initialization failure

---
 .../onyx/background/indexing/run_indexing.py  | 29 ++++++++++++-------
 backend/onyx/configs/app_configs.py           |  5 ++++
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/backend/onyx/background/indexing/run_indexing.py b/backend/onyx/background/indexing/run_indexing.py
index 72fd8e4fe..d502679f0 100644
--- a/backend/onyx/background/indexing/run_indexing.py
+++ b/backend/onyx/background/indexing/run_indexing.py
@@ -11,6 +11,7 @@ from onyx.background.indexing.checkpointing import get_time_windows_for_index_at
 from onyx.background.indexing.tracer import OnyxTracer
 from onyx.configs.app_configs import INDEXING_SIZE_WARNING_THRESHOLD
 from onyx.configs.app_configs import INDEXING_TRACER_INTERVAL
+from onyx.configs.app_configs import LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE
 from onyx.configs.app_configs import POLL_CONNECTOR_OFFSET
 from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import MilestoneRecordType
@@ -55,6 +56,7 @@ def _get_connector_runner(
     start_time: datetime,
     end_time: datetime,
     tenant_id: str | None,
+    leave_connector_active: bool = LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE,
 ) -> ConnectorRunner:
     """
     NOTE: `start_time` and `end_time` are only used for poll connectors
@@ -76,20 +78,25 @@ def _get_connector_runner(
         )
     except Exception as e:
         logger.exception(f"Unable to instantiate connector due to {e}")
-        # since we failed to even instantiate the connector, we pause the CCPair since
-        # it will never succeed
 
-        cc_pair = get_connector_credential_pair_from_id(
-            db_session=db_session,
-            cc_pair_id=attempt.connector_credential_pair.id,
-        )
-        if cc_pair and cc_pair.status == ConnectorCredentialPairStatus.ACTIVE:
-            update_connector_credential_pair(
+        # since we failed to even instantiate the connector, we pause the CCPair since
+        # it will never succeed. Sometimes there are cases where the connector will
+        # intermittently fail to initialize in which case we should pass in
+        # leave_connector_active=True to allow it to continue.
+        # For example, if there is nightly maintenance on a Confluence Server instance,
+        # the connector will fail to initialize every night.
+        if not leave_connector_active:
+            cc_pair = get_connector_credential_pair_from_id(
                 db_session=db_session,
-                connector_id=attempt.connector_credential_pair.connector.id,
-                credential_id=attempt.connector_credential_pair.credential.id,
-                status=ConnectorCredentialPairStatus.PAUSED,
+                cc_pair_id=attempt.connector_credential_pair.id,
             )
+            if cc_pair and cc_pair.status == ConnectorCredentialPairStatus.ACTIVE:
+                update_connector_credential_pair(
+                    db_session=db_session,
+                    connector_id=attempt.connector_credential_pair.connector.id,
+                    credential_id=attempt.connector_credential_pair.credential.id,
+                    status=ConnectorCredentialPairStatus.PAUSED,
+                )
         raise e
 
     return ConnectorRunner(
diff --git a/backend/onyx/configs/app_configs.py b/backend/onyx/configs/app_configs.py
index 67f106b00..4de5e2f74 100644
--- a/backend/onyx/configs/app_configs.py
+++ b/backend/onyx/configs/app_configs.py
@@ -409,6 +409,11 @@ EXPERIMENTAL_CHECKPOINTING_ENABLED = (
     os.environ.get("EXPERIMENTAL_CHECKPOINTING_ENABLED", "").lower() == "true"
 )
 
+LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE = (
+    os.environ.get("LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE", "").lower()
+    == "true"
+)
+
 PRUNING_DISABLED = -1
 DEFAULT_PRUNING_FREQ = 60 * 60 * 24  # Once a day