Add full exception trace to UI

backend/alembic/versions/8987770549c0_add_full_exception_stack_trace.py

@@ -0,0 +1,25 @@
+"""Add full exception stack trace
+
+Revision ID: 8987770549c0
+Revises: ec3ec2eabf7b
+Create Date: 2024-02-10 19:31:28.339135
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "8987770549c0"
+down_revision = "ec3ec2eabf7b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "index_attempt", sa.Column("full_exception_trace", sa.Text(), nullable=True)
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("index_attempt", "full_exception_trace")

backend/danswer/background/indexing/run_indexing.py

@@ -1,4 +1,5 @@
 import time
+import traceback
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
@@ -243,7 +244,12 @@ def _run_indexing(
                 or db_connector.disabled
                 or index_attempt.status != IndexingStatus.IN_PROGRESS
             ):
-                mark_attempt_failed(index_attempt, db_session, failure_reason=str(e))
+                mark_attempt_failed(
+                    index_attempt,
+                    db_session,
+                    failure_reason=str(e),
+                    full_exception_trace=traceback.format_exc(),
+                )
                 if is_primary:
                     update_connector_credential_pair(
                         db_session=db_session,

backend/danswer/db/index_attempt.py

@@ -95,10 +95,14 @@ def mark_attempt_succeeded(
 
 
 def mark_attempt_failed(
-    index_attempt: IndexAttempt, db_session: Session, failure_reason: str = "Unknown"
+    index_attempt: IndexAttempt,
+    db_session: Session,
+    failure_reason: str = "Unknown",
+    full_exception_trace: str | None = None,
 ) -> None:
     index_attempt.status = IndexingStatus.FAILED
     index_attempt.error_msg = failure_reason
+    index_attempt.full_exception_trace = full_exception_trace
     db_session.add(index_attempt)
     db_session.commit()
 

backend/danswer/db/models.py

@@ -427,9 +427,10 @@ class IndexAttempt(Base):
     # The two below may be slightly out of sync if user switches Embedding Model
     new_docs_indexed: Mapped[int | None] = mapped_column(Integer, default=0)
     total_docs_indexed: Mapped[int | None] = mapped_column(Integer, default=0)
-    error_msg: Mapped[str | None] = mapped_column(
-        Text, default=None
-    )  # only filled if status = "failed"
+    # only filled if status = "failed"
+    error_msg: Mapped[str | None] = mapped_column(Text, default=None)
+    # only filled if status = "failed" AND an unhandled exception caused the failure
+    full_exception_trace: Mapped[str | None] = mapped_column(Text, default=None)
     # Nullable because in the past, we didn't allow swapping out embedding models live
     embedding_model_id: Mapped[int] = mapped_column(
         ForeignKey("embedding_model.id"),

backend/danswer/server/documents/models.py

@@ -32,6 +32,7 @@ class IndexAttemptSnapshot(BaseModel):
     new_docs_indexed: int  # only includes completely new docs
     total_docs_indexed: int  # includes docs that are updated
     error_msg: str | None
+    full_exception_trace: str | None
     time_started: str | None
     time_updated: str
 
@@ -45,6 +46,7 @@ class IndexAttemptSnapshot(BaseModel):
             new_docs_indexed=index_attempt.new_docs_indexed or 0,
             total_docs_indexed=index_attempt.total_docs_indexed or 0,
             error_msg=index_attempt.error_msg,
+            full_exception_trace=index_attempt.full_exception_trace,
             time_started=index_attempt.time_started.isoformat()
             if index_attempt.time_started
             else None,

web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx

@@ -8,6 +8,8 @@ import {
   TableBody,
   TableCell,
   Text,
+  Button,
+  Divider,
 } from "@tremor/react";
 import { IndexAttemptStatus } from "@/components/Status";
 import { CCPairFullInfo } from "./types";
@@ -15,14 +17,63 @@ import { useState } from "react";
 import { PageSelector } from "@/components/PageSelector";
 import { localizeAndPrettify } from "@/lib/time";
 import { getDocsProcessedPerMinute } from "@/lib/indexAttempt";
+import { Modal } from "@/components/Modal";
+import { CheckmarkIcon, CopyIcon } from "@/components/icons/icons";
 
 const NUM_IN_PAGE = 8;
 
 export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
   const [page, setPage] = useState(1);
+  const [indexAttemptTracePopupId, setIndexAttemptTracePopupId] = useState<
+    number | null
+  >(null);
+  const indexAttemptToDisplayTraceFor = ccPair.index_attempts.find(
+    (indexAttempt) => indexAttempt.id === indexAttemptTracePopupId
+  );
+  const [copyClicked, setCopyClicked] = useState(false);
 
   return (
     <>
+      {indexAttemptToDisplayTraceFor &&
+        indexAttemptToDisplayTraceFor.full_exception_trace && (
+          <Modal
+            width="w-4/6"
+            className="h-5/6 overflow-y-hidden flex flex-col"
+            title="Full Exception Trace"
+            onOutsideClick={() => setIndexAttemptTracePopupId(null)}
+          >
+            <div className="overflow-y-auto mb-6">
+              <div className="mb-6">
+                {!copyClicked ? (
+                  <div
+                    onClick={() => {
+                      navigator.clipboard.writeText(
+                        indexAttemptToDisplayTraceFor.full_exception_trace!
+                      );
+                      setCopyClicked(true);
+                      setTimeout(() => setCopyClicked(false), 2000);
+                    }}
+                    className="flex w-fit cursor-pointer hover:bg-hover-light p-2 border-border border rounded"
+                  >
+                    Copy full trace
+                    <CopyIcon className="ml-2 my-auto" />
+                  </div>
+                ) : (
+                  <div className="flex w-fit hover:bg-hover-light p-2 border-border border rounded cursor-default">
+                    Copied to clipboard
+                    <CheckmarkIcon
+                      className="my-auto ml-2 flex flex-shrink-0 text-success"
+                      size={16}
+                    />
+                  </div>
+                )}
+              </div>
+              <div className="whitespace-pre-wrap">
+                {indexAttemptToDisplayTraceFor.full_exception_trace}
+              </div>
+            </div>
+          </Modal>
+        )}
       <Table>
         <TableHead>
           <TableRow>
@@ -61,7 +112,17 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
                   <TableCell>{indexAttempt.total_docs_indexed}</TableCell>
                   <TableCell>
                     <Text className="flex flex-wrap whitespace-normal">
-                      {indexAttempt.error_msg || "-"}
+                      <div>{indexAttempt.error_msg || "-"}</div>
+                      {indexAttempt.full_exception_trace && (
+                        <div
+                          onClick={() =>
+                            setIndexAttemptTracePopupId(indexAttempt.id)
+                          }
+                          className="mt-2 text-link cursor-pointer"
+                        >
+                          View Full Trace
+                        </div>
+                      )}
                     </Text>
                   </TableCell>
                 </TableRow>

web/src/lib/types.ts

@@ -167,6 +167,7 @@ export interface IndexAttemptSnapshot {
   new_docs_indexed: number;
   total_docs_indexed: number;
   error_msg: string | null;
+  full_exception_trace: string | null;
   time_started: string | null;
   time_updated: string;
 }