improve messaging and UI around cleanup of leftover index attempts (#3247)

* improve messaging and UI around cleanup of leftover index attempts

* add tag on init

backend/danswer/background/celery/apps/primary.py

@@ -24,7 +24,7 @@ from danswer.configs.constants import POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME
 from danswer.db.engine import get_session_with_default_tenant
 from danswer.db.engine import SqlEngine
 from danswer.db.index_attempt import get_index_attempt
-from danswer.db.index_attempt import mark_attempt_failed
+from danswer.db.index_attempt import mark_attempt_canceled
 from danswer.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
 from danswer.redis.redis_connector_delete import RedisConnectorDelete
 from danswer.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync
@@ -165,13 +165,13 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
                 continue
 
             failure_reason = (
-                f"Orphaned index attempt found on startup: "
+                f"Canceling leftover index attempt found on startup: "
                 f"index_attempt={attempt.id} "
                 f"cc_pair={attempt.connector_credential_pair_id} "
                 f"search_settings={attempt.search_settings_id}"
             )
             logger.warning(failure_reason)
-            mark_attempt_failed(attempt.id, db_session, failure_reason)
+            mark_attempt_canceled(attempt.id, db_session, failure_reason)
 
 
 @worker_ready.connect

backend/danswer/background/celery/tasks/indexing/tasks.py

@@ -77,7 +77,7 @@ class IndexingCallback(IndexingHeartbeatInterface):
         self.started: datetime = datetime.now(timezone.utc)
         self.redis_lock.reacquire()
 
-        self.last_tag: str = ""
+        self.last_tag: str = "IndexingCallback.__init__"
         self.last_lock_reacquire: datetime = datetime.now(timezone.utc)
 
     def should_stop(self) -> bool:

backend/danswer/db/enums.py

@@ -5,6 +5,7 @@ class IndexingStatus(str, PyEnum):
     NOT_STARTED = "not_started"
     IN_PROGRESS = "in_progress"
     SUCCESS = "success"
+    CANCELED = "canceled"
     FAILED = "failed"
     COMPLETED_WITH_ERRORS = "completed_with_errors"
 
@@ -12,6 +13,7 @@ class IndexingStatus(str, PyEnum):
         terminal_states = {
             IndexingStatus.SUCCESS,
             IndexingStatus.COMPLETED_WITH_ERRORS,
+            IndexingStatus.CANCELED,
             IndexingStatus.FAILED,
         }
         return self in terminal_states

backend/danswer/db/index_attempt.py

@@ -225,6 +225,28 @@ def mark_attempt_partially_succeeded(
         raise
 
 
+def mark_attempt_canceled(
+    index_attempt_id: int,
+    db_session: Session,
+    reason: str = "Unknown",
+) -> None:
+    try:
+        attempt = db_session.execute(
+            select(IndexAttempt)
+            .where(IndexAttempt.id == index_attempt_id)
+            .with_for_update()
+        ).scalar_one()
+
+        if not attempt.time_started:
+            attempt.time_started = datetime.now(timezone.utc)
+        attempt.status = IndexingStatus.CANCELED
+        attempt.error_msg = reason
+        db_session.commit()
+    except Exception:
+        db_session.rollback()
+        raise
+
+
 def mark_attempt_failed(
     index_attempt_id: int,
     db_session: Session,

web/src/app/admin/configuration/search/UpgradingPage.tsx

@@ -68,10 +68,11 @@ export default function UpgradingPage({
   const statusOrder: Record<ValidStatuses, number> = useMemo(
     () => ({
       failed: 0,
-      completed_with_errors: 1,
-      not_started: 2,
-      in_progress: 3,
-      success: 4,
+      canceled: 1,
+      completed_with_errors: 2,
+      not_started: 3,
+      in_progress: 4,
+      success: 5,
     }),
     []
   );

web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx

@@ -322,7 +322,8 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
                       </Text>
                     )}
 
-                    {indexAttempt.status === "failed" &&
+                    {(indexAttempt.status === "failed" ||
+                      indexAttempt.status === "canceled") &&
                       indexAttempt.error_msg && (
                         <Text className="flex flex-wrap whitespace-normal">
                           {indexAttempt.error_msg}

web/src/components/Status.tsx

@@ -52,7 +52,7 @@ export function IndexAttemptStatus({
         popupContent={
           <div className="w-64 p-2 break-words overflow-hidden whitespace-normal">
             The indexing attempt completed, but some errors were encountered
-            during the rrun.
+            during the run.
             <br />
             <br />
             Click View Errors for more details.
@@ -78,6 +78,12 @@ export function IndexAttemptStatus({
         Scheduled
       </Badge>
     );
+  } else if (status === "canceled") {
+    badge = (
+      <Badge variant="canceled" icon={FiClock}>
+        Canceled
+      </Badge>
+    );
   } else {
     badge = (
       <Badge variant="outline" icon={FiMinus}>

web/src/components/ui/badge.tsx

@@ -8,6 +8,8 @@ const badgeVariants = cva(
   {
     variants: {
       variant: {
+        canceled:
+          "border-gray-200 bg-gray-50 text-gray-600 hover:bg-gray-75 dark:bg-gray-900 dark:text-neutral-50 dark:hover:bg-gray-850",
         orange:
           "border-orange-200 bg-orange-50 text-orange-600 hover:bg-orange-75 dark:bg-orange-900 dark:text-neutral-50 dark:hover:bg-orange-850",
         paused:

web/src/lib/types.ts

@@ -72,6 +72,7 @@ export type ValidInputTypes = "load_state" | "poll" | "event";
 export type ValidStatuses =
   | "success"
   | "completed_with_errors"
+  | "canceled"
   | "failed"
   | "in_progress"
   | "not_started";