Individual connector page (#640)

This commit is contained in:
Chris Weaver
2023-10-27 21:32:18 -07:00
committed by GitHub
parent ad6ea1679a
commit fcce2b5a60
33 changed files with 1335 additions and 525 deletions

View File

@@ -0,0 +1,70 @@
import { getNameFromPath } from "@/lib/fileUtils";
import { ValidSources } from "@/lib/types";
import { List, ListItem, Card, Title, Divider } from "@tremor/react";
function convertObjectToString(obj: any): string | any {
// Check if obj is an object and not an array or null
if (typeof obj === "object" && obj !== null) {
if (!Array.isArray(obj)) {
return JSON.stringify(obj);
} else {
if (obj.length === 0) {
return null;
}
return obj.map((item) => convertObjectToString(item));
}
}
if (typeof obj === "boolean") {
return obj.toString();
}
return obj;
}
function buildConfigEntries(
obj: any,
sourceType: ValidSources
): { [key: string]: string } {
if (sourceType === "file") {
return obj.file_locations
? {
file_names: obj.file_locations.map(getNameFromPath),
}
: {};
} else if (sourceType === "google_sites") {
return {
base_url: obj.base_url,
};
}
return obj;
}
export function ConfigDisplay({
connectorSpecificConfig,
sourceType,
}: {
connectorSpecificConfig: any;
sourceType: ValidSources;
}) {
const configEntries = Object.entries(
buildConfigEntries(connectorSpecificConfig, sourceType)
);
if (!configEntries.length) {
return null;
}
return (
<>
<Title className="mb-2">Configuration</Title>
<Card>
<List>
{configEntries.map(([key, value]) => (
<ListItem key={key}>
<span>{key}</span>
<span>{convertObjectToString(value) || "-"}</span>
</ListItem>
))}
</List>
</Card>
</>
);
}

View File

@@ -0,0 +1,52 @@
"use client";
import { Button } from "@tremor/react";
import { CCPairFullInfo } from "./types";
import { usePopup } from "@/components/admin/connectors/Popup";
import { useRouter } from "next/navigation";
import { FiTrash } from "react-icons/fi";
import { deleteCCPair } from "@/lib/documentDeletion";
export function DeletionButton({ ccPair }: { ccPair: CCPairFullInfo }) {
const router = useRouter();
const { popup, setPopup } = usePopup();
const isDeleting =
ccPair?.latest_deletion_attempt?.status === "PENDING" ||
ccPair?.latest_deletion_attempt?.status === "STARTED";
let tooltip: string;
if (ccPair.connector.disabled) {
if (isDeleting) {
tooltip = "This connector is currently being deleted";
} else {
tooltip = "Click to delete";
}
} else {
tooltip = "You must disable the connector before deleting it";
}
return (
<div>
{popup}
<Button
variant="secondary"
size="xs"
color="red"
onClick={() =>
deleteCCPair(
ccPair.connector.id,
ccPair.credential.id,
setPopup,
() => router.refresh()
)
}
icon={FiTrash}
disabled={!ccPair.connector.disabled || isDeleting}
tooltip={tooltip}
>
Schedule for Deletion
</Button>
</div>
);
}

View File

@@ -0,0 +1,79 @@
"use client";
import {
Card,
Table,
TableHead,
TableRow,
TableHeaderCell,
TableBody,
TableCell,
Text,
} from "@tremor/react";
import { IndexAttemptStatus } from "@/components/Status";
import { CCPairFullInfo } from "./types";
import { useState } from "react";
import { PageSelector } from "@/components/PageSelector";
import { localizeAndPrettify } from "@/lib/time";
const NUM_IN_PAGE = 8;
export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
const [page, setPage] = useState(1);
return (
<>
<Table>
<TableHead>
<TableRow>
<TableHeaderCell>Time</TableHeaderCell>
<TableHeaderCell>Status</TableHeaderCell>
<TableHeaderCell>Num New Docs</TableHeaderCell>
<TableHeaderCell>Error Msg</TableHeaderCell>
</TableRow>
</TableHead>
<TableBody>
{ccPair.index_attempts
.slice(NUM_IN_PAGE * (page - 1), NUM_IN_PAGE * page)
.map((indexAttempt) => (
<TableRow key={indexAttempt.id}>
<TableCell>
{localizeAndPrettify(indexAttempt.time_updated)}
</TableCell>
<TableCell>
<IndexAttemptStatus
status={indexAttempt.status || "not_started"}
size="xs"
/>
</TableCell>
<TableCell>{indexAttempt.num_docs_indexed}</TableCell>
<TableCell>
<Text className="flex flex-wrap whitespace-normal">
{indexAttempt.error_msg || "-"}
</Text>
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
{ccPair.index_attempts.length > NUM_IN_PAGE && (
<div className="mt-3 flex">
<div className="mx-auto">
<PageSelector
totalPages={Math.ceil(ccPair.index_attempts.length / NUM_IN_PAGE)}
currentPage={page}
onPageChange={(newPage) => {
setPage(newPage);
window.scrollTo({
top: 0,
left: 0,
behavior: "smooth",
});
}}
/>
</div>
</div>
)}
</>
);
}

View File

@@ -0,0 +1,48 @@
"use client";
import { Button } from "@tremor/react";
import { CCPairFullInfo } from "./types";
import { usePopup } from "@/components/admin/connectors/Popup";
import { disableConnector } from "@/lib/connector";
import { useRouter } from "next/navigation";
export function ModifyStatusButtonCluster({
ccPair,
}: {
ccPair: CCPairFullInfo;
}) {
const router = useRouter();
const { popup, setPopup } = usePopup();
return (
<>
{popup}
{ccPair.connector.disabled ? (
<Button
variant="secondary"
size="xs"
onClick={() =>
disableConnector(ccPair.connector, setPopup, () => router.refresh())
}
tooltip="Click to start indexing again!"
>
Re-Enable
</Button>
) : (
<Button
variant="secondary"
size="xs"
onClick={() =>
disableConnector(ccPair.connector, setPopup, () => router.refresh())
}
tooltip={
"When disabled, the connectors documents will still" +
" be visible. However, no new documents will be indexed."
}
>
Disable
</Button>
)}
</>
);
}

View File

@@ -0,0 +1,45 @@
"use client";
import { usePopup } from "@/components/admin/connectors/Popup";
import { runConnector } from "@/lib/connector";
import { Button } from "@tremor/react";
import { useRouter } from "next/navigation";
export function ReIndexButton({
connectorId,
credentialId,
}: {
connectorId: number;
credentialId: number;
}) {
const router = useRouter();
const { popup, setPopup } = usePopup();
return (
<>
{popup}
<Button
className="ml-auto"
variant="secondary"
size="xs"
onClick={async () => {
const errorMsg = await runConnector(connectorId, [credentialId]);
if (errorMsg) {
setPopup({
message: errorMsg,
type: "error",
});
} else {
setPopup({
message: "Triggered connector run",
type: "success",
});
}
router.refresh();
}}
>
Run Indexing
</Button>
</>
);
}

View File

@@ -0,0 +1,110 @@
import { getCCPairSS } from "@/lib/ss/ccPair";
import { CCPairFullInfo } from "./types";
import { getErrorMsg } from "@/lib/fetchUtils";
import { HealthCheckBanner } from "@/components/health/healthcheck";
import { CCPairStatus } from "@/components/Status";
import { BackButton } from "@/components/BackButton";
import { Button, Divider, Title } from "@tremor/react";
import { IndexingAttemptsTable } from "./IndexingAttemptsTable";
import { Text } from "@tremor/react";
import { ConfigDisplay } from "./ConfigDisplay";
import { ModifyStatusButtonCluster } from "./ModifyStatusButtonCluster";
import { DeletionButton } from "./DeletionButton";
import { SSRAutoRefresh } from "@/components/SSRAutoRefresh";
import { ErrorCallout } from "@/components/ErrorCallout";
import { ReIndexButton } from "./ReIndexButton";
export default async function Page({
params,
}: {
params: { ccPairId: string };
}) {
const ccPairId = parseInt(params.ccPairId);
const ccPairResponse = await getCCPairSS(ccPairId);
if (!ccPairResponse.ok) {
const errorMsg = await getErrorMsg(ccPairResponse);
return (
<div className="mx-auto container">
<BackButton />
<ErrorCallout errorTitle={errorMsg} />
</div>
);
}
const ccPair = (await ccPairResponse.json()) as CCPairFullInfo;
const lastIndexAttempt = ccPair.index_attempts[0];
const isDeleting =
ccPair?.latest_deletion_attempt?.status === "PENDING" ||
ccPair?.latest_deletion_attempt?.status === "STARTED";
return (
<>
<SSRAutoRefresh />
<div className="mx-auto container dark">
<div className="mb-4">
<HealthCheckBanner />
</div>
<BackButton />
<div className="pb-1 flex mt-1">
<h1 className="text-3xl font-bold">{ccPair.name}</h1>
<div className="ml-auto">
<ModifyStatusButtonCluster ccPair={ccPair} />
</div>
</div>
<CCPairStatus
status={lastIndexAttempt?.status || "not_started"}
disabled={ccPair.connector.disabled}
isDeleting={isDeleting}
/>
<div className="text-gray-400 text-sm mt-1">
Total Documents Indexed:{" "}
<b className="text-gray-300">{ccPair.num_docs_indexed}</b>
</div>
<Divider />
<ConfigDisplay
connectorSpecificConfig={ccPair.connector.connector_specific_config}
sourceType={ccPair.connector.source}
/>
{/* NOTE: no divider / title here for `ConfigDisplay` since it is optional and we need
to render these conditionally.*/}
<div className="mt-6">
<div className="flex">
<Title>Indexing Attempts</Title>
<ReIndexButton
connectorId={ccPair.connector.id}
credentialId={ccPair.credential.id}
/>
</div>
<IndexingAttemptsTable ccPair={ccPair} />
</div>
<Divider />
<div className="mt-4">
<Title>Delete Connector</Title>
<Text>
Deleting the connector will also delete all associated documents.
</Text>
<div className="flex mt-16">
<div className="mx-auto">
<DeletionButton ccPair={ccPair} />
</div>
</div>
</div>
{/* TODO: add document search*/}
</div>
</>
);
}

View File

@@ -0,0 +1,16 @@
import {
Connector,
Credential,
DeletionAttemptSnapshot,
IndexAttemptSnapshot,
} from "@/lib/types";
export interface CCPairFullInfo {
id: number;
name: string;
num_docs_indexed: number;
connector: Connector<any>;
credential: Credential<any>;
index_attempts: IndexAttemptSnapshot[];
latest_deletion_attempt: DeletionAttemptSnapshot | null;
}

View File

@@ -17,11 +17,7 @@ import { LoadingAnimation } from "@/components/Loading";
import { Form, Formik } from "formik";
import { TextFormField } from "@/components/admin/connectors/Field";
import { FileUpload } from "@/components/admin/connectors/FileUpload";
const getNameFromPath = (path: string) => {
const pathParts = path.split("/");
return pathParts[pathParts.length - 1];
};
import { getNameFromPath } from "@/lib/fileUtils";
const Main = () => {
const [selectedFiles, setSelectedFiles] = useState<File[]>([]);

View File

@@ -145,6 +145,7 @@ export const DocumentSetCreationForm = ({
<div className="my-auto">
<ConnectorTitle
connector={ccPair.connector}
ccPairId={ccPair.cc_pair_id}
ccPairName={ccPair.name}
isLink={false}
showMetadata={false}

View File

@@ -159,6 +159,7 @@ const DocumentSetTable = ({
<ConnectorTitle
connector={ccPairDescriptor.connector}
ccPairName={ccPairDescriptor.name}
ccPairId={ccPairDescriptor.id}
showMetadata={false}
/>
</div>

View File

@@ -151,6 +151,7 @@ function Main() {
connector: (
<ConnectorTitle
ccPairName={connectorIndexingStatus.name}
ccPairId={connectorIndexingStatus.cc_pair_id}
connector={connectorIndexingStatus.connector}
isPublic={connectorIndexingStatus.public_doc}
owner={connectorIndexingStatus.owner}

View File

@@ -0,0 +1,29 @@
"use client";
import { useRouter } from "next/navigation";
import { FiChevronLeft } from "react-icons/fi";
export function BackButton() {
const router = useRouter();
return (
<div
className={`
my-auto
flex
mb-1
hover:bg-gray-800
w-fit
p-1
pr-2
cursor-pointer
rounded-lg
text-sm`}
onClick={() => router.back()}
>
<FiChevronLeft className="mr-1 my-auto" />
Back
</div>
);
}

View File

@@ -0,0 +1,24 @@
import { Callout } from "@tremor/react";
import { FiAlertOctagon } from "react-icons/fi";
export function ErrorCallout({
errorTitle,
errorMsg,
}: {
errorTitle?: string;
errorMsg?: string;
}) {
console.log(errorMsg);
return (
<div>
<Callout
className="mt-4"
title={errorTitle || "Page not found"}
icon={FiAlertOctagon}
color="rose"
>
{errorMsg}
</Callout>
</div>
);
}

View File

@@ -0,0 +1,19 @@
"use client";
import { useRouter } from "next/navigation";
import { useEffect } from "react";
export function SSRAutoRefresh({ refreshFreq = 5 }: { refreshFreq?: number }) {
// Helper which automatically refreshes a SSR page X seconds
const router = useRouter();
useEffect(() => {
const interval = setInterval(() => {
router.refresh();
}, refreshFreq * 1000);
return () => clearInterval(interval);
});
return <></>;
}

View File

@@ -0,0 +1,92 @@
"use client";
import { ValidStatuses } from "@/lib/types";
import { Badge } from "@tremor/react";
import {
FiAlertTriangle,
FiCheckCircle,
FiClock,
FiPauseCircle,
} from "react-icons/fi";
export function IndexAttemptStatus({
status,
size = "md",
}: {
status: ValidStatuses;
size?: "xs" | "sm" | "md" | "lg";
}) {
let badge;
if (status === "failed") {
badge = (
<Badge size={size} color="red" icon={FiAlertTriangle}>
Failed
</Badge>
);
} else if (status === "success") {
badge = (
<Badge size={size} color="green" icon={FiCheckCircle}>
Succeeded
</Badge>
);
} else if (status === "in_progress" || status === "not_started") {
badge = (
<Badge size={size} color="fuchsia" icon={FiClock}>
In Progress
</Badge>
);
} else {
badge = (
<Badge size={size} color="yellow" icon={FiClock}>
Initializing
</Badge>
);
}
// TODO: remove wrapping `dark` once we have light/dark mode
return <div className="dark">{badge}</div>;
}
export function CCPairStatus({
status,
disabled,
isDeleting,
size = "md",
}: {
status: ValidStatuses;
disabled: boolean;
isDeleting: boolean;
size?: "xs" | "sm" | "md" | "lg";
}) {
let badge;
if (isDeleting) {
badge = (
<Badge size={size} color="red" icon={FiAlertTriangle}>
Deleting
</Badge>
);
} else if (disabled) {
badge = (
<Badge size={size} color="yellow" icon={FiPauseCircle}>
Disabled
</Badge>
);
} else if (status === "failed") {
badge = (
<Badge size={size} color="red" icon={FiAlertTriangle}>
Error
</Badge>
);
} else {
badge = (
<Badge size={size} color="green" icon={FiCheckCircle}>
Running
</Badge>
);
}
// TODO: remove wrapping `dark` once we have light/dark mode
return <div className="dark">{badge}</div>;
}

View File

@@ -10,9 +10,11 @@ import {
WebConfig,
ZulipConfig,
} from "@/lib/types";
import Link from "next/link";
interface ConnectorTitleProps {
connector: Connector<any>;
ccPairId: number;
ccPairName: string | null | undefined;
isPublic?: boolean;
owner?: string;
@@ -22,6 +24,7 @@ interface ConnectorTitleProps {
export const ConnectorTitle = ({
connector,
ccPairId,
ccPairName,
owner,
isPublic = true,
@@ -82,17 +85,28 @@ export const ConnectorTitle = ({
typedConnector.connector_specific_config.realm_name
);
}
const mainSectionClassName = "text-blue-500 flex w-fit";
const mainDisplay = (
<>
{sourceMetadata.icon({ size: 20 })}
<div className="ml-1 my-auto">
{ccPairName || sourceMetadata.displayName}
</div>
</>
);
return (
<div>
<a
className="text-blue-500 flex w-fit"
href={isLink ? sourceMetadata.adminPageLink : undefined}
>
{sourceMetadata.icon({ size: 20 })}
<div className="ml-1 my-auto">
{ccPairName || sourceMetadata.displayName}
</div>
</a>
{isLink ? (
<Link
className={mainSectionClassName}
href={`/admin/connector/${ccPairId}`}
>
{mainDisplay}
</Link>
) : (
<div className={mainSectionClassName}>{mainDisplay}</div>
)}
{showMetadata && (
<div className="text-xs text-gray-300 mt-1">
{Array.from(additionalMetadata.entries()).map(([key, value]) => {

View File

@@ -1,9 +1,9 @@
import { Connector, ConnectorIndexingStatus, Credential } from "@/lib/types";
import { ConnectorIndexingStatus, Credential } from "@/lib/types";
import { BasicTable } from "@/components/admin/connectors/BasicTable";
import { Popup, PopupSpec } from "@/components/admin/connectors/Popup";
import { PopupSpec, usePopup } from "@/components/admin/connectors/Popup";
import { useState } from "react";
import { LinkBreakIcon, LinkIcon, TrashIcon } from "@/components/icons/icons";
import { updateConnector } from "@/lib/connector";
import { LinkBreakIcon, LinkIcon } from "@/components/icons/icons";
import { disableConnector } from "@/lib/connector";
import { AttachCredentialButtonForTable } from "@/components/admin/connectors/buttons/AttachCredentialButtonForTable";
import { DeleteColumn } from "./DeleteColumn";
@@ -53,23 +53,7 @@ export function StatusRow<ConnectorConfigType, ConnectorCredentialType>({
className="cursor-pointer ml-1 my-auto relative"
onMouseEnter={() => setStatusHovered(true)}
onMouseLeave={() => setStatusHovered(false)}
onClick={() => {
updateConnector({
...connector,
disabled: !connector.disabled,
}).then(() => {
setPopup({
message: connector.disabled
? "Enabled connector!"
: "Disabled connector!",
type: "success",
});
setTimeout(() => {
setPopup(null);
}, 4000);
onUpdate();
});
}}
onClick={() => disableConnector(connector, setPopup, onUpdate)}
>
{statusHovered && (
<div className="flex flex-nowrap absolute top-0 left-0 ml-8 bg-gray-700 px-3 py-2 rounded shadow-lg">
@@ -137,10 +121,7 @@ export function ConnectorsTable<ConnectorConfigType, ConnectorCredentialType>({
onCredentialLink,
includeName = false,
}: ConnectorsTableProps<ConnectorConfigType, ConnectorCredentialType>) {
const [popup, setPopup] = useState<{
message: string;
type: "success" | "error";
} | null>(null);
const { popup, setPopup } = usePopup();
const connectorIncludesCredential =
getCredential !== undefined && onCredentialLink !== undefined;
@@ -166,7 +147,7 @@ export function ConnectorsTable<ConnectorConfigType, ConnectorCredentialType>({
return (
<>
{popup && <Popup message={popup.message} type={popup.type} />}
{popup}
<BasicTable
columns={columns}
data={connectorIndexingStatuses.map((connectorIndexingStatus) => {

View File

@@ -1,5 +1,8 @@
import { InfoIcon, TrashIcon } from "@/components/icons/icons";
import { scheduleDeletionJobForConnector } from "@/lib/documentDeletion";
import {
deleteCCPair,
scheduleDeletionJobForConnector,
} from "@/lib/documentDeletion";
import { ConnectorIndexingStatus } from "@/lib/types";
import { PopupSpec } from "../Popup";
import { useState } from "react";
@@ -32,29 +35,9 @@ export function DeleteColumn<ConnectorConfigType, ConnectorCredentialType>({
{connectorIndexingStatus.is_deletable ? (
<div
className="cursor-pointer mx-auto flex"
onClick={async () => {
const deletionScheduleError = await scheduleDeletionJobForConnector(
connector.id,
credential.id
);
if (deletionScheduleError) {
setPopup({
message:
"Failed to schedule deletion of connector - " +
deletionScheduleError,
type: "error",
});
} else {
setPopup({
message: "Scheduled deletion of connector!",
type: "success",
});
}
setTimeout(() => {
setPopup(null);
}, 4000);
onUpdate();
}}
onClick={() =>
deleteCCPair(connector.id, credential.id, setPopup, onUpdate)
}
>
<TrashIcon />
</div>

View File

@@ -1,3 +1,4 @@
import { PopupSpec } from "@/components/admin/connectors/Popup";
import { Connector, ConnectorBase, ValidSources } from "./types";
async function handleResponse(
@@ -36,6 +37,28 @@ export async function updateConnector<T>(
return await response.json();
}
export async function disableConnector(
connector: Connector<any>,
setPopup: (popupSpec: PopupSpec | null) => void,
onUpdate: () => void
) {
updateConnector({
...connector,
disabled: !connector.disabled,
}).then(() => {
setPopup({
message: connector.disabled
? "Enabled connector!"
: "Disabled connector!",
type: "success",
});
setTimeout(() => {
setPopup(null);
}, 4000);
onUpdate && onUpdate();
});
}
export async function deleteConnector(
connectorId: number
): Promise<string | null> {

View File

@@ -1,7 +1,9 @@
export const scheduleDeletionJobForConnector = async (
import { PopupSpec } from "@/components/admin/connectors/Popup";
export async function scheduleDeletionJobForConnector(
connectorId: number,
credentialId: number
) => {
) {
// Will schedule a background job which will:
// 1. Remove all documents indexed by the connector / credential pair
// 2. Remove the connector (if this is the only pair using the connector)
@@ -19,4 +21,29 @@ export const scheduleDeletionJobForConnector = async (
return null;
}
return (await response.json()).detail;
};
}
export async function deleteCCPair(
connectorId: number,
credentialId: number,
setPopup: (popupSpec: PopupSpec | null) => void,
onCompletion: () => void
) {
const deletionScheduleError = await scheduleDeletionJobForConnector(
connectorId,
credentialId
);
if (deletionScheduleError) {
setPopup({
message:
"Failed to schedule deletion of connector - " + deletionScheduleError,
type: "error",
});
} else {
setPopup({
message: "Scheduled deletion of connector!",
type: "success",
});
}
onCompletion();
}

4
web/src/lib/fileUtils.ts Normal file
View File

@@ -0,0 +1,4 @@
export function getNameFromPath(path: string) {
const pathParts = path.split("/");
return pathParts[pathParts.length - 1];
}

5
web/src/lib/ss/ccPair.ts Normal file
View File

@@ -0,0 +1,5 @@
import { fetchSS } from "../utilsSS";
export async function getCCPairSS(ccPairId: number) {
return fetchSS(`/manage/admin/cc-pair/${ccPairId}`);
}

View File

@@ -54,3 +54,8 @@ export const timeAgo = (
const yearsDiff = Math.floor(monthsDiff / 12);
return `${yearsDiff} ${conditionallyAddPlural("year", yearsDiff)} ago`;
};
export function localizeAndPrettify(dateString: string) {
const date = new Date(dateString);
return date.toLocaleString();
}

View File

@@ -129,6 +129,7 @@ export interface GoogleSitesConfig {
}
export interface IndexAttemptSnapshot {
id: number;
status: ValidStatuses | null;
num_docs_indexed: number;
error_msg: string | null;

View File

@@ -1,8 +1,24 @@
import { cookies } from "next/headers";
import { INTERNAL_URL } from "./constants";
export const buildUrl = (path: string) => {
export function buildUrl(path: string) {
if (path.startsWith("/")) {
return `${INTERNAL_URL}${path}`;
}
return `${INTERNAL_URL}/${path}`;
};
}
export function fetchSS(url: string, options?: RequestInit) {
const init = options || {
credentials: "include",
next: { revalidate: 0 },
headers: {
cookie: cookies()
.getAll()
.map((cookie) => `${cookie.name}=${cookie.value}`)
.join("; "),
},
};
return fetch(buildUrl(url), init);
}