diff --git a/backend/onyx/server/documents/connector.py b/backend/onyx/server/documents/connector.py index c93d5249908..46c74942e1e 100644 --- a/backend/onyx/server/documents/connector.py +++ b/backend/onyx/server/documents/connector.py @@ -17,7 +17,6 @@ from onyx.auth.users import current_admin_user from onyx.auth.users import current_chat_accesssible_user from onyx.auth.users import current_curator_or_admin_user from onyx.auth.users import current_user -from onyx.background.celery.celery_utils import get_deletion_attempt_snapshot from onyx.background.celery.versioned_apps.primary import app as primary_app from onyx.configs.app_configs import ENABLED_CONNECTOR_TYPES from onyx.configs.constants import DocumentSource @@ -97,6 +96,7 @@ from onyx.server.documents.models import AuthUrl from onyx.server.documents.models import ConnectorCredentialPairIdentifier from onyx.server.documents.models import ConnectorIndexingStatus from onyx.server.documents.models import ConnectorSnapshot +from onyx.server.documents.models import ConnectorStatus from onyx.server.documents.models import ConnectorUpdateRequest from onyx.server.documents.models import CredentialBase from onyx.server.documents.models import CredentialSnapshot @@ -497,6 +497,40 @@ def get_currently_failed_indexing_status( return indexing_statuses +@router.get("/admin/connector/status") +def get_connector_status( + user: User = Depends(current_curator_or_admin_user), + db_session: Session = Depends(get_session), +) -> list[ConnectorStatus]: + cc_pairs = get_connector_credential_pairs( + db_session=db_session, + user=user, + ) + + group_cc_pair_relationships = get_cc_pair_groups_for_ids( + db_session=db_session, + cc_pair_ids=[cc_pair.id for cc_pair in cc_pairs], + ) + group_cc_pair_relationships_dict: dict[int, list[int]] = {} + for relationship in group_cc_pair_relationships: + group_cc_pair_relationships_dict.setdefault(relationship.cc_pair_id, []).append( + relationship.user_group_id + ) + + return [ + ConnectorStatus( + cc_pair_id=cc_pair.id, + name=cc_pair.name, + connector=ConnectorSnapshot.from_connector_db_model(cc_pair.connector), + credential=CredentialSnapshot.from_credential_db_model(cc_pair.credential), + access_type=cc_pair.access_type, + groups=group_cc_pair_relationships_dict.get(cc_pair.id, []), + ) + for cc_pair in cc_pairs + if cc_pair.name != "DefaultCCPair" and cc_pair.connector and cc_pair.credential + ] + + @router.get("/admin/connector/indexing-status") def get_connector_indexing_status( secondary_index: bool = False, @@ -599,6 +633,7 @@ def get_connector_indexing_status( ConnectorIndexingStatus( cc_pair_id=cc_pair.id, name=cc_pair.name, + in_progress=in_progress, cc_pair_status=cc_pair.status, connector=ConnectorSnapshot.from_connector_db_model(connector), credential=CredentialSnapshot.from_credential_db_model(credential), @@ -615,9 +650,6 @@ def get_connector_indexing_status( docs_indexed=cc_pair_to_document_cnt.get( (connector.id, credential.id), 0 ), - error_msg=( - latest_index_attempt.error_msg if latest_index_attempt else None - ), latest_index_attempt=( IndexAttemptSnapshot.from_index_attempt_db_model( latest_index_attempt @@ -625,20 +657,6 @@ def get_connector_indexing_status( if latest_index_attempt else None ), - deletion_attempt=get_deletion_attempt_snapshot( - connector_id=connector.id, - credential_id=credential.id, - db_session=db_session, - tenant_id=tenant_id, - ), - is_deletable=check_deletion_attempt_is_allowed( - connector_credential_pair=cc_pair, - db_session=db_session, - # allow scheduled indexing attempts here, since on deletion request we will cancel them - allow_scheduled=True, - ) - is None, - in_progress=in_progress, ) ) diff --git a/backend/onyx/server/documents/models.py b/backend/onyx/server/documents/models.py index 9098e3b3831..64880587bb7 100644 --- a/backend/onyx/server/documents/models.py +++ b/backend/onyx/server/documents/models.py @@ -307,28 +307,30 @@ class FailedConnectorIndexingStatus(BaseModel): credential_id: int -class ConnectorIndexingStatus(BaseModel): - """Represents the latest indexing status of a connector""" +class ConnectorStatus(BaseModel): + """ + Represents the status of a connector, + including indexing status elated information + """ cc_pair_id: int name: str | None - cc_pair_status: ConnectorCredentialPairStatus connector: ConnectorSnapshot credential: CredentialSnapshot - owner: str - groups: list[int] access_type: AccessType + groups: list[int] + + +class ConnectorIndexingStatus(ConnectorStatus): + """Represents the full indexing status of a connector""" + + cc_pair_status: ConnectorCredentialPairStatus + owner: str last_finished_status: IndexingStatus | None last_status: IndexingStatus | None last_success: datetime | None - docs_indexed: int - error_msg: str | None latest_index_attempt: IndexAttemptSnapshot | None - deletion_attempt: DeletionAttemptSnapshot | None - is_deletable: bool - - # index attempt in db can be marked successful while celery/redis - # is stil running/cleaning up + docs_indexed: int in_progress: bool diff --git a/backend/test b/backend/test new file mode 100644 index 00000000000..e69de29bb2d diff --git a/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx b/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx index c92c401c3cd..e1ed28b25cb 100644 --- a/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx +++ b/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx @@ -6,11 +6,8 @@ import { FetchError, errorHandlingFetcher } from "@/lib/fetcher"; import { ErrorCallout } from "@/components/ErrorCallout"; import { LoadingAnimation } from "@/components/Loading"; import { usePopup } from "@/components/admin/connectors/Popup"; -import { ConnectorIndexingStatus, ValidSources } from "@/lib/types"; -import { - usePublicCredentials, - useConnectorCredentialIndexingStatus, -} from "@/lib/hooks"; +import { ValidSources } from "@/lib/types"; +import { usePublicCredentials } from "@/lib/hooks"; import Title from "@/components/ui/title"; import { DriveJsonUploadSection, DriveAuthSection } from "./Credential"; import { @@ -18,13 +15,9 @@ import { GoogleDriveCredentialJson, GoogleDriveServiceAccountCredentialJson, } from "@/lib/connectors/credentials"; -import { - ConnectorSnapshot, - GoogleDriveConfig, -} from "@/lib/connectors/connectors"; +import { ConnectorSnapshot } from "@/lib/connectors/connectors"; import { useUser } from "@/components/user/UserProvider"; import { buildSimilarCredentialInfoURL } from "@/app/admin/connector/[ccPairId]/lib"; -import { fetchConnectors } from "@/lib/connector"; const useConnectorsByCredentialId = (credential_id: number | null) => { let url: string | null = null; diff --git a/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx b/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx index ec98bfcdbd5..28af99441f1 100644 --- a/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx +++ b/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx @@ -4,19 +4,15 @@ import useSWR from "swr"; import { errorHandlingFetcher } from "@/lib/fetcher"; import { LoadingAnimation } from "@/components/Loading"; import { usePopup } from "@/components/admin/connectors/Popup"; -import { ConnectorIndexingStatus } from "@/lib/types"; +import { CCPairBasicInfo } from "@/lib/types"; import { Credential, GmailCredentialJson, GmailServiceAccountCredentialJson, } from "@/lib/connectors/credentials"; import { GmailAuthSection, GmailJsonUploadSection } from "./Credential"; -import { - usePublicCredentials, - useConnectorCredentialIndexingStatus, -} from "@/lib/hooks"; +import { usePublicCredentials, useBasicConnectorStatus } from "@/lib/hooks"; import Title from "@/components/ui/title"; -import { GmailConfig } from "@/lib/connectors/connectors"; import { useUser } from "@/components/user/UserProvider"; export const GmailMain = () => { @@ -42,7 +38,7 @@ export const GmailMain = () => { data: connectorIndexingStatuses, isLoading: isConnectorIndexingStatusesLoading, error: connectorIndexingStatusesError, - } = useConnectorCredentialIndexingStatus(); + } = useBasicConnectorStatus(); const { data: credentialsData, @@ -116,13 +112,11 @@ export const GmailMain = () => { credential.credential_json?.google_service_account_key && credential.source === "gmail" ); - const gmailConnectorIndexingStatuses: ConnectorIndexingStatus< - GmailConfig, - GmailCredentialJson - >[] = connectorIndexingStatuses.filter( - (connectorIndexingStatus) => - connectorIndexingStatus.connector.source === "gmail" - ); + + const gmailConnectorIndexingStatuses: CCPairBasicInfo[] = + connectorIndexingStatuses.filter( + (connectorIndexingStatus) => connectorIndexingStatus.source === "gmail" + ); return ( <> diff --git a/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx b/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx index 3f7f4b1b85d..020b5402485 100644 --- a/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx +++ b/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx @@ -8,12 +8,7 @@ import { updateDocumentSet, DocumentSetCreationRequest, } from "./lib"; -import { - ConnectorIndexingStatus, - DocumentSet, - UserGroup, - UserRole, -} from "@/lib/types"; +import { ConnectorStatus, DocumentSet, UserGroup, UserRole } from "@/lib/types"; import { TextFormField } from "@/components/admin/connectors/Field"; import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle"; import { Separator } from "@/components/ui/separator"; @@ -24,7 +19,7 @@ import React, { useEffect, useState } from "react"; import { useUser } from "@/components/user/UserProvider"; interface SetCreationPopupProps { - ccPairs: ConnectorIndexingStatus[]; + ccPairs: ConnectorStatus[]; userGroups: UserGroup[] | undefined; onClose: () => void; setPopup: (popupSpec: PopupSpec | null) => void; diff --git a/web/src/app/admin/documents/sets/[documentSetId]/page.tsx b/web/src/app/admin/documents/sets/[documentSetId]/page.tsx index cfbdc6b58f7..2dcee56c027 100644 --- a/web/src/app/admin/documents/sets/[documentSetId]/page.tsx +++ b/web/src/app/admin/documents/sets/[documentSetId]/page.tsx @@ -3,10 +3,7 @@ import { use } from "react"; import { ErrorCallout } from "@/components/ErrorCallout"; import { refreshDocumentSets, useDocumentSets } from "../hooks"; -import { - useConnectorCredentialIndexingStatus, - useUserGroups, -} from "@/lib/hooks"; +import { useConnectorStatus, useUserGroups } from "@/lib/hooks"; import { ThreeDotsLoader } from "@/components/Loading"; import { AdminPageTitle } from "@/components/admin/Title"; import { BookmarkIcon } from "@/components/icons/icons"; @@ -30,7 +27,7 @@ function Main({ documentSetId }: { documentSetId: number }) { data: ccPairs, isLoading: isCCPairsLoading, error: ccPairsError, - } = useConnectorCredentialIndexingStatus(); + } = useConnectorStatus(); // EE only const { data: userGroups, isLoading: userGroupsIsLoading } = useUserGroups(); diff --git a/web/src/app/admin/documents/sets/new/page.tsx b/web/src/app/admin/documents/sets/new/page.tsx index 828f5128c4b..dd98aa9754c 100644 --- a/web/src/app/admin/documents/sets/new/page.tsx +++ b/web/src/app/admin/documents/sets/new/page.tsx @@ -3,10 +3,7 @@ import { AdminPageTitle } from "@/components/admin/Title"; import { BookmarkIcon } from "@/components/icons/icons"; import { DocumentSetCreationForm } from "../DocumentSetCreationForm"; -import { - useConnectorCredentialIndexingStatus, - useUserGroups, -} from "@/lib/hooks"; +import { useConnectorStatus, useUserGroups } from "@/lib/hooks"; import { ThreeDotsLoader } from "@/components/Loading"; import { usePopup } from "@/components/admin/connectors/Popup"; import { BackButton } from "@/components/BackButton"; @@ -23,7 +20,7 @@ function Main() { data: ccPairs, isLoading: isCCPairsLoading, error: ccPairsError, - } = useConnectorCredentialIndexingStatus(); + } = useConnectorStatus(); // EE only const { data: userGroups, isLoading: userGroupsIsLoading } = useUserGroups(); diff --git a/web/src/app/admin/documents/sets/page.tsx b/web/src/app/admin/documents/sets/page.tsx index d2192929f09..43873016fcb 100644 --- a/web/src/app/admin/documents/sets/page.tsx +++ b/web/src/app/admin/documents/sets/page.tsx @@ -14,8 +14,7 @@ import Text from "@/components/ui/text"; import Title from "@/components/ui/title"; import { Separator } from "@/components/ui/separator"; import { Button } from "@/components/ui/button"; -import { useConnectorCredentialIndexingStatus } from "@/lib/hooks"; -import { ConnectorIndexingStatus, DocumentSet } from "@/lib/types"; +import { DocumentSet } from "@/lib/types"; import { useState } from "react"; import { useDocumentSets } from "./hooks"; import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle"; @@ -99,7 +98,6 @@ const EditRow = ({ interface DocumentFeedbackTableProps { documentSets: DocumentSet[]; - ccPairs: ConnectorIndexingStatus[]; refresh: () => void; refreshEditable: () => void; setPopup: (popupSpec: PopupSpec | null) => void; @@ -275,6 +273,7 @@ const Main = () => { error: documentSetsError, refreshDocumentSets, } = useDocumentSets(); + const { data: editableDocumentSets, isLoading: isEditableDocumentSetsLoading, @@ -282,17 +281,7 @@ const Main = () => { refreshDocumentSets: refreshEditableDocumentSets, } = useDocumentSets(true); - const { - data: ccPairs, - isLoading: isCCPairsLoading, - error: ccPairsError, - } = useConnectorCredentialIndexingStatus(); - - if ( - isDocumentSetsLoading || - isCCPairsLoading || - isEditableDocumentSetsLoading - ) { + if (isDocumentSetsLoading || isEditableDocumentSetsLoading) { return ; } @@ -304,10 +293,6 @@ const Main = () => { return
Error: {editableDocumentSetsError}
; } - if (ccPairsError || !ccPairs) { - return
Error: {ccPairsError}
; - } - return (
{popup} @@ -331,7 +316,6 @@ const Main = () => { void; - allCCPairs: ConnectorIndexingStatus[]; + allCCPairs: ConnectorStatus[]; } export const ConnectorEditor = ({ diff --git a/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx b/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx index 1b7b4bdbf00..32bdf8caf5c 100644 --- a/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx +++ b/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx @@ -1,7 +1,7 @@ import { Form, Formik } from "formik"; import * as Yup from "yup"; import { PopupSpec } from "@/components/admin/connectors/Popup"; -import { ConnectorIndexingStatus, User, UserGroup } from "@/lib/types"; +import { ConnectorStatus, User, UserGroup } from "@/lib/types"; import { TextFormField } from "@/components/admin/connectors/Field"; import { createUserGroup } from "./lib"; import { UserEditor } from "./UserEditor"; @@ -14,7 +14,7 @@ interface UserGroupCreationFormProps { onClose: () => void; setPopup: (popupSpec: PopupSpec | null) => void; users: User[]; - ccPairs: ConnectorIndexingStatus[]; + ccPairs: ConnectorStatus[]; existingUserGroup?: UserGroup; } diff --git a/web/src/app/ee/admin/groups/[groupId]/AddConnectorForm.tsx b/web/src/app/ee/admin/groups/[groupId]/AddConnectorForm.tsx index dee6d5ee17b..c002bbc89ff 100644 --- a/web/src/app/ee/admin/groups/[groupId]/AddConnectorForm.tsx +++ b/web/src/app/ee/admin/groups/[groupId]/AddConnectorForm.tsx @@ -5,11 +5,11 @@ import { useState } from "react"; import { FiPlus, FiX } from "react-icons/fi"; import { updateUserGroup } from "./lib"; import { PopupSpec } from "@/components/admin/connectors/Popup"; -import { ConnectorIndexingStatus, UserGroup } from "@/lib/types"; +import { ConnectorStatus, UserGroup } from "@/lib/types"; import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle"; import { Connector } from "@/lib/connectors/connectors"; interface AddConnectorFormProps { - ccPairs: ConnectorIndexingStatus[]; + ccPairs: ConnectorStatus[]; userGroup: UserGroup; onClose: () => void; setPopup: (popupSpec: PopupSpec) => void; diff --git a/web/src/app/ee/admin/groups/[groupId]/GroupDisplay.tsx b/web/src/app/ee/admin/groups/[groupId]/GroupDisplay.tsx index b6c97e0f1fa..9193ae1fbe2 100644 --- a/web/src/app/ee/admin/groups/[groupId]/GroupDisplay.tsx +++ b/web/src/app/ee/admin/groups/[groupId]/GroupDisplay.tsx @@ -12,6 +12,7 @@ import { UserGroup, UserRole, USER_ROLE_LABELS, + ConnectorStatus, } from "@/lib/types"; import { AddConnectorForm } from "./AddConnectorForm"; import { Separator } from "@/components/ui/separator"; @@ -42,7 +43,7 @@ import { GenericConfirmModal } from "@/components/modals/GenericConfirmModal"; interface GroupDisplayProps { users: User[]; - ccPairs: ConnectorIndexingStatus[]; + ccPairs: ConnectorStatus[]; userGroup: UserGroup; refreshUserGroup: () => void; } diff --git a/web/src/app/ee/admin/groups/[groupId]/page.tsx b/web/src/app/ee/admin/groups/[groupId]/page.tsx index f378473f597..367c7a128df 100644 --- a/web/src/app/ee/admin/groups/[groupId]/page.tsx +++ b/web/src/app/ee/admin/groups/[groupId]/page.tsx @@ -5,7 +5,11 @@ import { GroupsIcon } from "@/components/icons/icons"; import { GroupDisplay } from "./GroupDisplay"; import { useSpecificUserGroup } from "./hook"; import { ThreeDotsLoader } from "@/components/Loading"; -import { useConnectorCredentialIndexingStatus, useUsers } from "@/lib/hooks"; +import { + useConnectorCredentialIndexingStatus, + useConnectorStatus, + useUsers, +} from "@/lib/hooks"; import { useRouter } from "next/navigation"; import { BackButton } from "@/components/BackButton"; import { AdminPageTitle } from "@/components/admin/Title"; @@ -29,7 +33,7 @@ const Page = (props: { params: Promise<{ groupId: string }> }) => { data: ccPairs, isLoading: isCCPairsLoading, error: ccPairsError, - } = useConnectorCredentialIndexingStatus(); + } = useConnectorStatus(); if (userGroupIsLoading || userIsLoading || isCCPairsLoading) { return ( diff --git a/web/src/app/ee/admin/groups/page.tsx b/web/src/app/ee/admin/groups/page.tsx index ada8131f72a..89129a09b91 100644 --- a/web/src/app/ee/admin/groups/page.tsx +++ b/web/src/app/ee/admin/groups/page.tsx @@ -6,11 +6,7 @@ import { UserGroupCreationForm } from "./UserGroupCreationForm"; import { usePopup } from "@/components/admin/connectors/Popup"; import { useState } from "react"; import { ThreeDotsLoader } from "@/components/Loading"; -import { - useConnectorCredentialIndexingStatus, - useUserGroups, - useUsers, -} from "@/lib/hooks"; +import { useConnectorStatus, useUserGroups, useUsers } from "@/lib/hooks"; import { AdminPageTitle } from "@/components/admin/Title"; import { Button } from "@/components/ui/button"; @@ -26,7 +22,7 @@ const Main = () => { data: ccPairs, isLoading: isCCPairsLoading, error: ccPairsError, - } = useConnectorCredentialIndexingStatus(); + } = useConnectorStatus(); const { data: users, diff --git a/web/src/lib/hooks.ts b/web/src/lib/hooks.ts index d8d14e72918..81f425f0dd9 100644 --- a/web/src/lib/hooks.ts +++ b/web/src/lib/hooks.ts @@ -5,6 +5,8 @@ import { DocumentBoostStatus, Tag, UserGroup, + ConnectorStatus, + CCPairBasicInfo, } from "@/lib/types"; import useSWR, { mutate, useSWRConfig } from "swr"; import { errorHandlingFetcher } from "./fetcher"; @@ -71,6 +73,7 @@ export const useObjectState = ( }; const INDEXING_STATUS_URL = "/api/manage/admin/connector/indexing-status"; +const CONNECTOR_STATUS_URL = "/api/manage/admin/connector/status"; export const useConnectorCredentialIndexingStatus = ( refreshInterval = 30000, // 30 seconds @@ -92,6 +95,30 @@ export const useConnectorCredentialIndexingStatus = ( }; }; +export const useConnectorStatus = (refreshInterval = 30000) => { + const { mutate } = useSWRConfig(); + const url = CONNECTOR_STATUS_URL; + const swrResponse = useSWR[]>( + url, + errorHandlingFetcher, + { refreshInterval: refreshInterval } + ); + + return { + ...swrResponse, + refreshIndexingStatus: () => mutate(url), + }; +}; + +export const useBasicConnectorStatus = () => { + const url = "/api/manage/admin/connector-status"; + const swrResponse = useSWR(url, errorHandlingFetcher); + return { + ...swrResponse, + refreshIndexingStatus: () => mutate(url), + }; +}; + export const useCategories = () => { const { mutate } = useSWRConfig(); const swrResponse = useSWR( diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 38cd6742a34..801b3b2f62c 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -130,27 +130,26 @@ export interface IndexAttemptSnapshot { time_updated: string; } -export interface ConnectorIndexingStatus< - ConnectorConfigType, - ConnectorCredentialType, -> { +export interface ConnectorStatus { cc_pair_id: number; name: string | null; - cc_pair_status: ConnectorCredentialPairStatus; connector: Connector; credential: Credential; access_type: AccessType; - owner: string; groups: number[]; - last_finished_status: ValidStatuses | null; - last_status: ValidStatuses | null; +} + +export interface ConnectorIndexingStatus< + ConnectorConfigType, + ConnectorCredentialType, +> extends ConnectorStatus { + // Inlcude data only necessary for indexing statuses in admin page last_success: string | null; - docs_indexed: number; - error_msg: string; + last_status: ValidStatuses | null; + last_finished_status: ValidStatuses | null; + cc_pair_status: ConnectorCredentialPairStatus; latest_index_attempt: IndexAttemptSnapshot | null; - deletion_attempt: DeletionAttemptSnapshot | null; - is_deletable: boolean; - in_progress: boolean; + docs_indexed: number; } export interface OAuthPrepareAuthorizationResponse {