Improve chat page performance (#3347)

* Simplify /manage/indexing-status

* Rename endpoint

backend/danswer/server/documents/connector.py

@@ -1010,37 +1010,18 @@ def get_connector_by_id(
 
 
 class BasicCCPairInfo(BaseModel):
-    docs_indexed: int
     has_successful_run: bool
     source: DocumentSource
 
 
-@router.get("/indexing-status")
+@router.get("/connector-status")
 def get_basic_connector_indexing_status(
     _: User = Depends(current_user),
     db_session: Session = Depends(get_session),
 ) -> list[BasicCCPairInfo]:
     cc_pairs = get_connector_credential_pairs(db_session)
-    cc_pair_identifiers = [
-        ConnectorCredentialPairIdentifier(
-            connector_id=cc_pair.connector_id, credential_id=cc_pair.credential_id
-        )
-        for cc_pair in cc_pairs
-    ]
-    document_count_info = get_document_counts_for_cc_pairs(
-        db_session=db_session,
-        cc_pair_identifiers=cc_pair_identifiers,
-    )
-    cc_pair_to_document_cnt = {
-        (connector_id, credential_id): cnt
-        for connector_id, credential_id, cnt in document_count_info
-    }
     return [
         BasicCCPairInfo(
-            docs_indexed=cc_pair_to_document_cnt.get(
-                (cc_pair.connector_id, cc_pair.credential_id)
-            )
-            or 0,
             has_successful_run=cc_pair.last_successful_index_time is not None,
             source=cc_pair.connector.source,
         )

web/src/components/initialSetup/search/NoCompleteSourceModal.tsx

@@ -1,71 +0,0 @@
-"use client";
-
-import { Modal } from "../../Modal";
-import Link from "next/link";
-import { useEffect, useState } from "react";
-import { CCPairBasicInfo } from "@/lib/types";
-import { useRouter } from "next/navigation";
-
-export function NoCompleteSourcesModal({
-  ccPairs,
-}: {
-  ccPairs: CCPairBasicInfo[];
-}) {
-  const router = useRouter();
-  const [isHidden, setIsHidden] = useState(false);
-
-  useEffect(() => {
-    const interval = setInterval(() => {
-      router.refresh();
-    }, 5000);
-
-    return () => clearInterval(interval);
-  }, [router]);
-
-  if (isHidden) {
-    return null;
-  }
-
-  const totalDocs = ccPairs.reduce(
-    (acc, ccPair) => acc + ccPair.docs_indexed,
-    0
-  );
-
-  return (
-    <Modal
-      className="max-w-4xl"
-      title="⏳ None of your connectors have finished a full sync yet"
-      onOutsideClick={() => setIsHidden(true)}
-    >
-      <div className="text-sm">
-        <div>
-          <div>
-            You&apos;ve connected some sources, but none of them have finished
-            syncing. Depending on the size of the knowledge base(s) you&apos;ve
-            connected to Danswer, it can take anywhere between 30 seconds to a
-            few days for the initial sync to complete. So far we&apos;ve synced{" "}
-            <b>{totalDocs}</b> documents.
-            <br />
-            <br />
-            To view the status of your syncing connectors, head over to the{" "}
-            <Link className="text-link" href="admin/indexing/status">
-              Existing Connectors page
-            </Link>
-            .
-            <br />
-            <br />
-            <p
-              className="text-link cursor-pointer inline"
-              onClick={() => {
-                setIsHidden(true);
-              }}
-            >
-              Or, click here to continue and ask questions on the partially
-              synced knowledge set.
-            </p>
-          </div>
-        </div>
-      </div>
-    </Modal>
-  );
-}

web/src/lib/assistants/fetchPersonaEditorInfoSS.ts

@@ -26,7 +26,7 @@ export async function fetchAssistantEditorInfoSS(
   | [null, string]
 > {
   const tasks = [
-    fetchSS("/manage/indexing-status"),
+    fetchSS("/manage/connector-status"),
     fetchSS("/manage/document-set"),
     fetchSS("/llm/provider"),
     // duplicate fetch, but shouldn't be too big of a deal

web/src/lib/chat/fetchAssistantdata.ts

@@ -30,7 +30,7 @@ export async function fetchAssistantData(): Promise<AssistantData> {
 
     // Parallel fetch of additional data
     const [ccPairsResponse, llmProviders] = await Promise.all([
-      fetchSS("/manage/indexing-status").catch((error) => {
+      fetchSS("/manage/connector-status").catch((error) => {
         console.error("Failed to fetch connectors:", error);
         return null;
       }),

web/src/lib/chat/fetchChatData.ts

@@ -52,7 +52,7 @@ export async function fetchChatData(searchParams: {
   const tasks = [
     getAuthTypeMetadataSS(),
     getCurrentUserSS(),
-    fetchSS("/manage/indexing-status"),
+    fetchSS("/manage/connector-status"),
     fetchSS("/manage/document-set"),
     fetchSS("/chat/get-user-chat-sessions"),
     fetchSS("/query/valid-tags"),

web/src/lib/chat/fetchSomeChatData.ts

@@ -70,7 +70,7 @@ export async function fetchSomeChatData(
   const taskMap: Record<FetchOption, () => Promise<any>> = {
     user: getCurrentUserSS,
     chatSessions: () => fetchSS("/chat/get-user-chat-sessions"),
-    ccPairs: () => fetchSS("/manage/indexing-status"),
+    ccPairs: () => fetchSS("/manage/connector-status"),
     documentSets: () => fetchSS("/manage/document-set"),
     assistants: fetchAssistantsSS,
     tags: () => fetchSS("/query/valid-tags"),
@@ -229,9 +229,7 @@ export async function fetchSomeChatData(
     result.shouldDisplaySourcesIncompleteModal =
       hasAnyConnectors &&
       !result.shouldShowWelcomeModal &&
-      !result.ccPairs.some(
+      !result.ccPairs.some((ccPair) => ccPair.has_successful_run) &&
-        (ccPair) => ccPair.has_successful_run && ccPair.docs_indexed > 0
-      ) &&
       (!user || user.role === "admin");
   }
 

web/src/lib/types.ts

@@ -136,7 +136,6 @@ export interface ConnectorIndexingStatus<
 }
 
 export interface CCPairBasicInfo {
-  docs_indexed: number;
   has_successful_run: boolean;
   source: ValidSources;
 }