Speed up admin pages (#3623)

* ni

* speed up pages

* minor nit

* nit

backend/onyx/server/documents/connector.py

@@ -17,7 +17,6 @@ from onyx.auth.users import current_admin_user
 from onyx.auth.users import current_chat_accesssible_user
 from onyx.auth.users import current_curator_or_admin_user
 from onyx.auth.users import current_user
-from onyx.background.celery.celery_utils import get_deletion_attempt_snapshot
 from onyx.background.celery.versioned_apps.primary import app as primary_app
 from onyx.configs.app_configs import ENABLED_CONNECTOR_TYPES
 from onyx.configs.constants import DocumentSource
@@ -97,6 +96,7 @@ from onyx.server.documents.models import AuthUrl
 from onyx.server.documents.models import ConnectorCredentialPairIdentifier
 from onyx.server.documents.models import ConnectorIndexingStatus
 from onyx.server.documents.models import ConnectorSnapshot
+from onyx.server.documents.models import ConnectorStatus
 from onyx.server.documents.models import ConnectorUpdateRequest
 from onyx.server.documents.models import CredentialBase
 from onyx.server.documents.models import CredentialSnapshot
@@ -497,6 +497,40 @@ def get_currently_failed_indexing_status(
     return indexing_statuses
 
 
+@router.get("/admin/connector/status")
+def get_connector_status(
+    user: User = Depends(current_curator_or_admin_user),
+    db_session: Session = Depends(get_session),
+) -> list[ConnectorStatus]:
+    cc_pairs = get_connector_credential_pairs(
+        db_session=db_session,
+        user=user,
+    )
+
+    group_cc_pair_relationships = get_cc_pair_groups_for_ids(
+        db_session=db_session,
+        cc_pair_ids=[cc_pair.id for cc_pair in cc_pairs],
+    )
+    group_cc_pair_relationships_dict: dict[int, list[int]] = {}
+    for relationship in group_cc_pair_relationships:
+        group_cc_pair_relationships_dict.setdefault(relationship.cc_pair_id, []).append(
+            relationship.user_group_id
+        )
+
+    return [
+        ConnectorStatus(
+            cc_pair_id=cc_pair.id,
+            name=cc_pair.name,
+            connector=ConnectorSnapshot.from_connector_db_model(cc_pair.connector),
+            credential=CredentialSnapshot.from_credential_db_model(cc_pair.credential),
+            access_type=cc_pair.access_type,
+            groups=group_cc_pair_relationships_dict.get(cc_pair.id, []),
+        )
+        for cc_pair in cc_pairs
+        if cc_pair.name != "DefaultCCPair" and cc_pair.connector and cc_pair.credential
+    ]
+
+
 @router.get("/admin/connector/indexing-status")
 def get_connector_indexing_status(
     secondary_index: bool = False,
@@ -599,6 +633,7 @@ def get_connector_indexing_status(
             ConnectorIndexingStatus(
                 cc_pair_id=cc_pair.id,
                 name=cc_pair.name,
+                in_progress=in_progress,
                 cc_pair_status=cc_pair.status,
                 connector=ConnectorSnapshot.from_connector_db_model(connector),
                 credential=CredentialSnapshot.from_credential_db_model(credential),
@@ -615,9 +650,6 @@ def get_connector_indexing_status(
                 docs_indexed=cc_pair_to_document_cnt.get(
                     (connector.id, credential.id), 0
                 ),
-                error_msg=(
-                    latest_index_attempt.error_msg if latest_index_attempt else None
-                ),
                 latest_index_attempt=(
                     IndexAttemptSnapshot.from_index_attempt_db_model(
                         latest_index_attempt
@@ -625,20 +657,6 @@ def get_connector_indexing_status(
                     if latest_index_attempt
                     else None
                 ),
-                deletion_attempt=get_deletion_attempt_snapshot(
-                    connector_id=connector.id,
-                    credential_id=credential.id,
-                    db_session=db_session,
-                    tenant_id=tenant_id,
-                ),
-                is_deletable=check_deletion_attempt_is_allowed(
-                    connector_credential_pair=cc_pair,
-                    db_session=db_session,
-                    # allow scheduled indexing attempts here, since on deletion request we will cancel them
-                    allow_scheduled=True,
-                )
-                is None,
-                in_progress=in_progress,
             )
         )
 

backend/onyx/server/documents/models.py

@@ -307,28 +307,30 @@ class FailedConnectorIndexingStatus(BaseModel):
     credential_id: int
 
 
-class ConnectorIndexingStatus(BaseModel):
-    """Represents the latest indexing status of a connector"""
+class ConnectorStatus(BaseModel):
+    """
+    Represents the status of a connector,
+    including indexing status elated information
+    """
 
     cc_pair_id: int
     name: str | None
-    cc_pair_status: ConnectorCredentialPairStatus
     connector: ConnectorSnapshot
     credential: CredentialSnapshot
-    owner: str
-    groups: list[int]
     access_type: AccessType
+    groups: list[int]
+
+
+class ConnectorIndexingStatus(ConnectorStatus):
+    """Represents the full indexing status of a connector"""
+
+    cc_pair_status: ConnectorCredentialPairStatus
+    owner: str
     last_finished_status: IndexingStatus | None
     last_status: IndexingStatus | None
     last_success: datetime | None
-    docs_indexed: int
-    error_msg: str | None
     latest_index_attempt: IndexAttemptSnapshot | None
-    deletion_attempt: DeletionAttemptSnapshot | None
-    is_deletable: bool
-
-    # index attempt in db can be marked successful while celery/redis
-    # is stil running/cleaning up
+    docs_indexed: int
     in_progress: bool
 
 

web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx

@@ -6,11 +6,8 @@ import { FetchError, errorHandlingFetcher } from "@/lib/fetcher";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { LoadingAnimation } from "@/components/Loading";
 import { usePopup } from "@/components/admin/connectors/Popup";
-import { ConnectorIndexingStatus, ValidSources } from "@/lib/types";
-import {
-  usePublicCredentials,
-  useConnectorCredentialIndexingStatus,
-} from "@/lib/hooks";
+import { ValidSources } from "@/lib/types";
+import { usePublicCredentials } from "@/lib/hooks";
 import Title from "@/components/ui/title";
 import { DriveJsonUploadSection, DriveAuthSection } from "./Credential";
 import {
@@ -18,13 +15,9 @@ import {
   GoogleDriveCredentialJson,
   GoogleDriveServiceAccountCredentialJson,
 } from "@/lib/connectors/credentials";
-import {
-  ConnectorSnapshot,
-  GoogleDriveConfig,
-} from "@/lib/connectors/connectors";
+import { ConnectorSnapshot } from "@/lib/connectors/connectors";
 import { useUser } from "@/components/user/UserProvider";
 import { buildSimilarCredentialInfoURL } from "@/app/admin/connector/[ccPairId]/lib";
-import { fetchConnectors } from "@/lib/connector";
 
 const useConnectorsByCredentialId = (credential_id: number | null) => {
   let url: string | null = null;

web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx

@@ -4,19 +4,15 @@ import useSWR from "swr";
 import { errorHandlingFetcher } from "@/lib/fetcher";
 import { LoadingAnimation } from "@/components/Loading";
 import { usePopup } from "@/components/admin/connectors/Popup";
-import { ConnectorIndexingStatus } from "@/lib/types";
+import { CCPairBasicInfo } from "@/lib/types";
 import {
   Credential,
   GmailCredentialJson,
   GmailServiceAccountCredentialJson,
 } from "@/lib/connectors/credentials";
 import { GmailAuthSection, GmailJsonUploadSection } from "./Credential";
-import {
-  usePublicCredentials,
-  useConnectorCredentialIndexingStatus,
-} from "@/lib/hooks";
+import { usePublicCredentials, useBasicConnectorStatus } from "@/lib/hooks";
 import Title from "@/components/ui/title";
-import { GmailConfig } from "@/lib/connectors/connectors";
 import { useUser } from "@/components/user/UserProvider";
 
 export const GmailMain = () => {
@@ -42,7 +38,7 @@ export const GmailMain = () => {
     data: connectorIndexingStatuses,
     isLoading: isConnectorIndexingStatusesLoading,
     error: connectorIndexingStatusesError,
-  } = useConnectorCredentialIndexingStatus();
+  } = useBasicConnectorStatus();
 
   const {
     data: credentialsData,
@@ -116,13 +112,11 @@ export const GmailMain = () => {
       credential.credential_json?.google_service_account_key &&
       credential.source === "gmail"
   );
-  const gmailConnectorIndexingStatuses: ConnectorIndexingStatus<
-    GmailConfig,
-    GmailCredentialJson
-  >[] = connectorIndexingStatuses.filter(
-    (connectorIndexingStatus) =>
-      connectorIndexingStatus.connector.source === "gmail"
-  );
+
+  const gmailConnectorIndexingStatuses: CCPairBasicInfo[] =
+    connectorIndexingStatuses.filter(
+      (connectorIndexingStatus) => connectorIndexingStatus.source === "gmail"
+    );
 
   return (
     <>

web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx

@@ -8,12 +8,7 @@ import {
   updateDocumentSet,
   DocumentSetCreationRequest,
 } from "./lib";
-import {
-  ConnectorIndexingStatus,
-  DocumentSet,
-  UserGroup,
-  UserRole,
-} from "@/lib/types";
+import { ConnectorStatus, DocumentSet, UserGroup, UserRole } from "@/lib/types";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle";
 import { Separator } from "@/components/ui/separator";
@@ -24,7 +19,7 @@ import React, { useEffect, useState } from "react";
 import { useUser } from "@/components/user/UserProvider";
 
 interface SetCreationPopupProps {
-  ccPairs: ConnectorIndexingStatus<any, any>[];
+  ccPairs: ConnectorStatus<any, any>[];
   userGroups: UserGroup[] | undefined;
   onClose: () => void;
   setPopup: (popupSpec: PopupSpec | null) => void;

web/src/app/admin/documents/sets/[documentSetId]/page.tsx

@@ -3,10 +3,7 @@ import { use } from "react";
 
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { refreshDocumentSets, useDocumentSets } from "../hooks";
-import {
-  useConnectorCredentialIndexingStatus,
-  useUserGroups,
-} from "@/lib/hooks";
+import { useConnectorStatus, useUserGroups } from "@/lib/hooks";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { BookmarkIcon } from "@/components/icons/icons";
@@ -30,7 +27,7 @@ function Main({ documentSetId }: { documentSetId: number }) {
     data: ccPairs,
     isLoading: isCCPairsLoading,
     error: ccPairsError,
-  } = useConnectorCredentialIndexingStatus();
+  } = useConnectorStatus();
 
   // EE only
   const { data: userGroups, isLoading: userGroupsIsLoading } = useUserGroups();

web/src/app/admin/documents/sets/new/page.tsx

@@ -3,10 +3,7 @@
 import { AdminPageTitle } from "@/components/admin/Title";
 import { BookmarkIcon } from "@/components/icons/icons";
 import { DocumentSetCreationForm } from "../DocumentSetCreationForm";
-import {
-  useConnectorCredentialIndexingStatus,
-  useUserGroups,
-} from "@/lib/hooks";
+import { useConnectorStatus, useUserGroups } from "@/lib/hooks";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { BackButton } from "@/components/BackButton";
@@ -23,7 +20,7 @@ function Main() {
     data: ccPairs,
     isLoading: isCCPairsLoading,
     error: ccPairsError,
-  } = useConnectorCredentialIndexingStatus();
+  } = useConnectorStatus();
 
   // EE only
   const { data: userGroups, isLoading: userGroupsIsLoading } = useUserGroups();

web/src/app/admin/documents/sets/page.tsx

@@ -14,8 +14,7 @@ import Text from "@/components/ui/text";
 import Title from "@/components/ui/title";
 import { Separator } from "@/components/ui/separator";
 import { Button } from "@/components/ui/button";
-import { useConnectorCredentialIndexingStatus } from "@/lib/hooks";
-import { ConnectorIndexingStatus, DocumentSet } from "@/lib/types";
+import { DocumentSet } from "@/lib/types";
 import { useState } from "react";
 import { useDocumentSets } from "./hooks";
 import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle";
@@ -99,7 +98,6 @@ const EditRow = ({
 
 interface DocumentFeedbackTableProps {
   documentSets: DocumentSet[];
-  ccPairs: ConnectorIndexingStatus<any, any>[];
   refresh: () => void;
   refreshEditable: () => void;
   setPopup: (popupSpec: PopupSpec | null) => void;
@@ -275,6 +273,7 @@ const Main = () => {
     error: documentSetsError,
     refreshDocumentSets,
   } = useDocumentSets();
+
   const {
     data: editableDocumentSets,
     isLoading: isEditableDocumentSetsLoading,
@@ -282,17 +281,7 @@ const Main = () => {
     refreshDocumentSets: refreshEditableDocumentSets,
   } = useDocumentSets(true);
 
-  const {
-    data: ccPairs,
-    isLoading: isCCPairsLoading,
-    error: ccPairsError,
-  } = useConnectorCredentialIndexingStatus();
-
-  if (
-    isDocumentSetsLoading ||
-    isCCPairsLoading ||
-    isEditableDocumentSetsLoading
-  ) {
+  if (isDocumentSetsLoading || isEditableDocumentSetsLoading) {
     return <ThreeDotsLoader />;
   }
 
@@ -304,10 +293,6 @@ const Main = () => {
     return <div>Error: {editableDocumentSetsError}</div>;
   }
 
-  if (ccPairsError || !ccPairs) {
-    return <div>Error: {ccPairsError}</div>;
-  }
-
   return (
     <div className="mb-8">
       {popup}
@@ -331,7 +316,6 @@ const Main = () => {
           <DocumentSetTable
             documentSets={documentSets}
             editableDocumentSets={editableDocumentSets}
-            ccPairs={ccPairs}
             refresh={refreshDocumentSets}
             refreshEditable={refreshEditableDocumentSets}
             setPopup={setPopup}

web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx

@@ -411,11 +411,6 @@ export function CCPairIndexingStatusTable({
             last_success: "2023-07-01T12:00:00Z",
             last_finished_status: "success",
             latest_index_attempt: null,
-            owner: "1",
-            error_msg: "",
-            deletion_attempt: null,
-            is_deletable: true,
-            in_progress: false,
             groups: [], // Add this line
           }}
           isEditable={false}

web/src/app/admin/indexing/status/page.tsx

@@ -16,6 +16,7 @@ function Main() {
     isLoading: indexAttemptIsLoading,
     error: indexAttemptError,
   } = useConnectorCredentialIndexingStatus();
+
   const {
     data: editableIndexAttemptData,
     isLoading: editableIndexAttemptIsLoading,

web/src/app/ee/admin/groups/ConnectorEditor.tsx

@@ -1,10 +1,10 @@
-import { ConnectorIndexingStatus } from "@/lib/types";
+import { ConnectorIndexingStatus, ConnectorStatus } from "@/lib/types";
 import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle";
 
 interface ConnectorEditorProps {
   selectedCCPairIds: number[];
   setSetCCPairIds: (ccPairId: number[]) => void;
-  allCCPairs: ConnectorIndexingStatus<any, any>[];
+  allCCPairs: ConnectorStatus<any, any>[];
 }
 
 export const ConnectorEditor = ({

web/src/app/ee/admin/groups/UserGroupCreationForm.tsx

@@ -1,7 +1,7 @@
 import { Form, Formik } from "formik";
 import * as Yup from "yup";
 import { PopupSpec } from "@/components/admin/connectors/Popup";
-import { ConnectorIndexingStatus, User, UserGroup } from "@/lib/types";
+import { ConnectorStatus, User, UserGroup } from "@/lib/types";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { createUserGroup } from "./lib";
 import { UserEditor } from "./UserEditor";
@@ -14,7 +14,7 @@ interface UserGroupCreationFormProps {
   onClose: () => void;
   setPopup: (popupSpec: PopupSpec | null) => void;
   users: User[];
-  ccPairs: ConnectorIndexingStatus<any, any>[];
+  ccPairs: ConnectorStatus<any, any>[];
   existingUserGroup?: UserGroup;
 }
 

web/src/app/ee/admin/groups/[groupId]/AddConnectorForm.tsx

@@ -5,11 +5,11 @@ import { useState } from "react";
 import { FiPlus, FiX } from "react-icons/fi";
 import { updateUserGroup } from "./lib";
 import { PopupSpec } from "@/components/admin/connectors/Popup";
-import { ConnectorIndexingStatus, UserGroup } from "@/lib/types";
+import { ConnectorStatus, UserGroup } from "@/lib/types";
 import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle";
 import { Connector } from "@/lib/connectors/connectors";
 interface AddConnectorFormProps {
-  ccPairs: ConnectorIndexingStatus<any, any>[];
+  ccPairs: ConnectorStatus<any, any>[];
   userGroup: UserGroup;
   onClose: () => void;
   setPopup: (popupSpec: PopupSpec) => void;

web/src/app/ee/admin/groups/[groupId]/GroupDisplay.tsx

@@ -12,6 +12,7 @@ import {
   UserGroup,
   UserRole,
   USER_ROLE_LABELS,
+  ConnectorStatus,
 } from "@/lib/types";
 import { AddConnectorForm } from "./AddConnectorForm";
 import { Separator } from "@/components/ui/separator";
@@ -42,7 +43,7 @@ import { GenericConfirmModal } from "@/components/modals/GenericConfirmModal";
 
 interface GroupDisplayProps {
   users: User[];
-  ccPairs: ConnectorIndexingStatus<any, any>[];
+  ccPairs: ConnectorStatus<any, any>[];
   userGroup: UserGroup;
   refreshUserGroup: () => void;
 }

web/src/app/ee/admin/groups/[groupId]/page.tsx

@@ -5,7 +5,11 @@ import { GroupsIcon } from "@/components/icons/icons";
 import { GroupDisplay } from "./GroupDisplay";
 import { useSpecificUserGroup } from "./hook";
 import { ThreeDotsLoader } from "@/components/Loading";
-import { useConnectorCredentialIndexingStatus, useUsers } from "@/lib/hooks";
+import {
+  useConnectorCredentialIndexingStatus,
+  useConnectorStatus,
+  useUsers,
+} from "@/lib/hooks";
 import { useRouter } from "next/navigation";
 import { BackButton } from "@/components/BackButton";
 import { AdminPageTitle } from "@/components/admin/Title";
@@ -29,7 +33,7 @@ const Page = (props: { params: Promise<{ groupId: string }> }) => {
     data: ccPairs,
     isLoading: isCCPairsLoading,
     error: ccPairsError,
-  } = useConnectorCredentialIndexingStatus();
+  } = useConnectorStatus();
 
   if (userGroupIsLoading || userIsLoading || isCCPairsLoading) {
     return (

web/src/app/ee/admin/groups/page.tsx

@@ -6,11 +6,7 @@ import { UserGroupCreationForm } from "./UserGroupCreationForm";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { useState } from "react";
 import { ThreeDotsLoader } from "@/components/Loading";
-import {
-  useConnectorCredentialIndexingStatus,
-  useUserGroups,
-  useUsers,
-} from "@/lib/hooks";
+import { useConnectorStatus, useUserGroups, useUsers } from "@/lib/hooks";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { Button } from "@/components/ui/button";
 
@@ -26,7 +22,7 @@ const Main = () => {
     data: ccPairs,
     isLoading: isCCPairsLoading,
     error: ccPairsError,
-  } = useConnectorCredentialIndexingStatus();
+  } = useConnectorStatus();
 
   const {
     data: users,

web/src/lib/hooks.ts

@@ -5,6 +5,8 @@ import {
   DocumentBoostStatus,
   Tag,
   UserGroup,
+  ConnectorStatus,
+  CCPairBasicInfo,
 } from "@/lib/types";
 import useSWR, { mutate, useSWRConfig } from "swr";
 import { errorHandlingFetcher } from "./fetcher";
@@ -71,6 +73,7 @@ export const useObjectState = <T>(
 };
 
 const INDEXING_STATUS_URL = "/api/manage/admin/connector/indexing-status";
+const CONNECTOR_STATUS_URL = "/api/manage/admin/connector/status";
 
 export const useConnectorCredentialIndexingStatus = (
   refreshInterval = 30000, // 30 seconds
@@ -92,6 +95,30 @@ export const useConnectorCredentialIndexingStatus = (
   };
 };
 
+export const useConnectorStatus = (refreshInterval = 30000) => {
+  const { mutate } = useSWRConfig();
+  const url = CONNECTOR_STATUS_URL;
+  const swrResponse = useSWR<ConnectorStatus<any, any>[]>(
+    url,
+    errorHandlingFetcher,
+    { refreshInterval: refreshInterval }
+  );
+
+  return {
+    ...swrResponse,
+    refreshIndexingStatus: () => mutate(url),
+  };
+};
+
+export const useBasicConnectorStatus = () => {
+  const url = "/api/manage/admin/connector-status";
+  const swrResponse = useSWR<CCPairBasicInfo[]>(url, errorHandlingFetcher);
+  return {
+    ...swrResponse,
+    refreshIndexingStatus: () => mutate(url),
+  };
+};
+
 export const useCategories = () => {
   const { mutate } = useSWRConfig();
   const swrResponse = useSWR<PersonaCategory[]>(

web/src/lib/types.ts

@@ -130,27 +130,26 @@ export interface IndexAttemptSnapshot {
   time_updated: string;
 }
 
-export interface ConnectorIndexingStatus<
-  ConnectorConfigType,
-  ConnectorCredentialType,
-> {
+export interface ConnectorStatus<ConnectorConfigType, ConnectorCredentialType> {
   cc_pair_id: number;
   name: string | null;
-  cc_pair_status: ConnectorCredentialPairStatus;
   connector: Connector<ConnectorConfigType>;
   credential: Credential<ConnectorCredentialType>;
   access_type: AccessType;
-  owner: string;
   groups: number[];
-  last_finished_status: ValidStatuses | null;
-  last_status: ValidStatuses | null;
+}
+
+export interface ConnectorIndexingStatus<
+  ConnectorConfigType,
+  ConnectorCredentialType,
+> extends ConnectorStatus<ConnectorConfigType, ConnectorCredentialType> {
+  // Inlcude data only necessary for indexing statuses in admin page
   last_success: string | null;
-  docs_indexed: number;
-  error_msg: string;
+  last_status: ValidStatuses | null;
+  last_finished_status: ValidStatuses | null;
+  cc_pair_status: ConnectorCredentialPairStatus;
   latest_index_attempt: IndexAttemptSnapshot | null;
-  deletion_attempt: DeletionAttemptSnapshot | null;
-  is_deletable: boolean;
-  in_progress: boolean;
+  docs_indexed: number;
 }
 
 export interface OAuthPrepareAuthorizationResponse {