diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py index 197856d2d0a3..99acb591df76 100644 --- a/backend/danswer/background/celery/tasks/indexing/tasks.py +++ b/backend/danswer/background/celery/tasks/indexing/tasks.py @@ -598,7 +598,7 @@ def connector_indexing_proxy_task( db_session, "Connector termination signal detected", ) - finally: + except Exception: # if the DB exceptions, we'll just get an unfriendly failure message # in the UI instead of the cancellation message logger.exception( diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py index 1670dcd84c33..be9e82f63651 100644 --- a/backend/danswer/background/celery/tasks/vespa/tasks.py +++ b/backend/danswer/background/celery/tasks/vespa/tasks.py @@ -680,17 +680,28 @@ def monitor_ccpair_indexing_taskset( ) task_logger.warning(msg) - index_attempt = get_index_attempt(db_session, payload.index_attempt_id) - if index_attempt: - if ( - index_attempt.status != IndexingStatus.CANCELED - and index_attempt.status != IndexingStatus.FAILED - ): - mark_attempt_failed( - index_attempt_id=payload.index_attempt_id, - db_session=db_session, - failure_reason=msg, - ) + try: + index_attempt = get_index_attempt( + db_session, payload.index_attempt_id + ) + if index_attempt: + if ( + index_attempt.status != IndexingStatus.CANCELED + and index_attempt.status != IndexingStatus.FAILED + ): + mark_attempt_failed( + index_attempt_id=payload.index_attempt_id, + db_session=db_session, + failure_reason=msg, + ) + except Exception: + task_logger.exception( + "monitor_ccpair_indexing_taskset - transient exception marking index attempt as failed: " + f"attempt={payload.index_attempt_id} " + f"tenant={tenant_id} " + f"cc_pair={cc_pair_id} " + f"search_settings={search_settings_id}" + ) redis_connector_index.reset() return diff --git a/backend/danswer/db/index_attempt.py b/backend/danswer/db/index_attempt.py index 06bbee10559c..2c8ccd99aa0e 100644 --- a/backend/danswer/db/index_attempt.py +++ b/backend/danswer/db/index_attempt.py @@ -522,12 +522,16 @@ def expire_index_attempts( search_settings_id: int, db_session: Session, ) -> None: - delete_query = ( - delete(IndexAttempt) + not_started_query = ( + update(IndexAttempt) .where(IndexAttempt.search_settings_id == search_settings_id) .where(IndexAttempt.status == IndexingStatus.NOT_STARTED) + .values( + status=IndexingStatus.CANCELED, + error_msg="Canceled, likely due to model swap", + ) ) - db_session.execute(delete_query) + db_session.execute(not_started_query) update_query = ( update(IndexAttempt) @@ -549,9 +553,14 @@ def cancel_indexing_attempts_for_ccpair( include_secondary_index: bool = False, ) -> None: stmt = ( - delete(IndexAttempt) + update(IndexAttempt) .where(IndexAttempt.connector_credential_pair_id == cc_pair_id) .where(IndexAttempt.status == IndexingStatus.NOT_STARTED) + .values( + status=IndexingStatus.CANCELED, + error_msg="Canceled by user", + time_started=datetime.now(timezone.utc), + ) ) if not include_secondary_index: diff --git a/backend/danswer/server/documents/cc_pair.py b/backend/danswer/server/documents/cc_pair.py index 424ae2564627..88c812b19e21 100644 --- a/backend/danswer/server/documents/cc_pair.py +++ b/backend/danswer/server/documents/cc_pair.py @@ -33,8 +33,6 @@ from danswer.db.engine import get_current_tenant_id from danswer.db.engine import get_session from danswer.db.enums import AccessType from danswer.db.enums import ConnectorCredentialPairStatus -from danswer.db.index_attempt import cancel_indexing_attempts_for_ccpair -from danswer.db.index_attempt import cancel_indexing_attempts_past_model from danswer.db.index_attempt import count_index_attempts_for_connector from danswer.db.index_attempt import get_latest_index_attempt_for_cc_pair_id from danswer.db.index_attempt import get_paginated_index_attempts_for_cc_pair_id @@ -193,9 +191,6 @@ def update_cc_pair_status( db_session ) - cancel_indexing_attempts_for_ccpair(cc_pair_id, db_session) - cancel_indexing_attempts_past_model(db_session) - redis_connector = RedisConnector(tenant_id, cc_pair_id) try: