Paginate connector page (#2328)

* Added pagination to individual connector pages

* I cooked

* Gordon Ramsay in this b

* meepe

* properly calculated max chunk and switch dict to array

* chunks -> batches

* increased max page size

* renmaed var

web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx

@@ -1,5 +1,6 @@
 "use client";
 
+import { useEffect, useRef } from "react";
 import {
   Table,
   TableHead,
@@ -8,31 +9,172 @@ import {
   TableBody,
   TableCell,
   Text,
-  Button,
-  Divider,
 } from "@tremor/react";
-import { IndexAttemptStatus } from "@/components/Status";
 import { CCPairFullInfo } from "./types";
+import { IndexAttemptStatus } from "@/components/Status";
 import { useState } from "react";
 import { PageSelector } from "@/components/PageSelector";
+import { ThreeDotsLoader } from "@/components/Loading";
+import { buildCCPairInfoUrl } from "./lib";
 import { localizeAndPrettify } from "@/lib/time";
 import { getDocsProcessedPerMinute } from "@/lib/indexAttempt";
-import { Modal } from "@/components/Modal";
-import { CheckmarkIcon, CopyIcon, SearchIcon } from "@/components/icons/icons";
+import { ErrorCallout } from "@/components/ErrorCallout";
+import { SearchIcon } from "@/components/icons/icons";
 import Link from "next/link";
 import ExceptionTraceModal from "@/components/modals/ExceptionTraceModal";
+import { PaginatedIndexAttempts } from "./types";
+import { useRouter } from "next/navigation";
 
+// This is the number of index attempts to display per page
 const NUM_IN_PAGE = 8;
+// This is the number of pages to fetch at a time
+const BATCH_SIZE = 8;
 
 export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
-  const [page, setPage] = useState(1);
   const [indexAttemptTracePopupId, setIndexAttemptTracePopupId] = useState<
     number | null
   >(null);
-  const indexAttemptToDisplayTraceFor = ccPair.index_attempts.find(
+
+  const totalPages = Math.ceil(ccPair.number_of_index_attempts / NUM_IN_PAGE);
+
+  const router = useRouter();
+  const [page, setPage] = useState(() => {
+    if (typeof window !== "undefined") {
+      const urlParams = new URLSearchParams(window.location.search);
+      return parseInt(urlParams.get("page") || "1", 10);
+    }
+    return 1;
+  });
+
+  const [currentPageData, setCurrentPageData] =
+    useState<PaginatedIndexAttempts | null>(null);
+  const [currentPageError, setCurrentPageError] = useState<Error | null>(null);
+  const [isCurrentPageLoading, setIsCurrentPageLoading] = useState(false);
+
+  // This is a cache of the data for each "batch" which is a set of pages
+  const [cachedBatches, setCachedBatches] = useState<{
+    [key: number]: PaginatedIndexAttempts[];
+  }>({});
+
+  // This is a set of the batches that are currently being fetched
+  // we use it to avoid duplicate requests
+  const ongoingRequestsRef = useRef<Set<number>>(new Set());
+
+  const batchRetrievalUrlBuilder = (batchNum: number) =>
+    `${buildCCPairInfoUrl(ccPair.id)}/index-attempts?page=${batchNum}&page_size=${BATCH_SIZE * NUM_IN_PAGE}`;
+
+  // This fetches and caches the data for a given batch number
+  const fetchBatchData = async (batchNum: number) => {
+    if (ongoingRequestsRef.current.has(batchNum)) return;
+    ongoingRequestsRef.current.add(batchNum);
+
+    try {
+      const response = await fetch(batchRetrievalUrlBuilder(batchNum + 1));
+      if (!response.ok) {
+        throw new Error("Failed to fetch data");
+      }
+      const data = await response.json();
+
+      const newBatchData: PaginatedIndexAttempts[] = [];
+      for (let i = 0; i < BATCH_SIZE; i++) {
+        const startIndex = i * NUM_IN_PAGE;
+        const endIndex = startIndex + NUM_IN_PAGE;
+        const pageIndexAttempts = data.index_attempts.slice(
+          startIndex,
+          endIndex
+        );
+        newBatchData.push({
+          ...data,
+          index_attempts: pageIndexAttempts,
+        });
+      }
+
+      setCachedBatches((prev) => ({
+        ...prev,
+        [batchNum]: newBatchData,
+      }));
+    } catch (error) {
+      setCurrentPageError(
+        error instanceof Error ? error : new Error("An error occurred")
+      );
+    } finally {
+      ongoingRequestsRef.current.delete(batchNum);
+    }
+  };
+
+  // This fetches and caches the data for the current batch and the next and previous batches
+  useEffect(() => {
+    const batchNum = Math.floor((page - 1) / BATCH_SIZE);
+
+    if (!cachedBatches[batchNum]) {
+      setIsCurrentPageLoading(true);
+      fetchBatchData(batchNum);
+    } else {
+      setIsCurrentPageLoading(false);
+    }
+
+    const nextBatchNum = Math.min(
+      batchNum + 1,
+      Math.ceil(totalPages / BATCH_SIZE) - 1
+    );
+    if (!cachedBatches[nextBatchNum]) {
+      fetchBatchData(nextBatchNum);
+    }
+
+    const prevBatchNum = Math.max(batchNum - 1, 0);
+    if (!cachedBatches[prevBatchNum]) {
+      fetchBatchData(prevBatchNum);
+    }
+
+    // Always fetch the first batch if it's not cached
+    if (!cachedBatches[0]) {
+      fetchBatchData(0);
+    }
+  }, [ccPair.id, page, cachedBatches, totalPages]);
+
+  // This updates the data on the current page
+  useEffect(() => {
+    const batchNum = Math.floor((page - 1) / BATCH_SIZE);
+    const batchPageNum = (page - 1) % BATCH_SIZE;
+
+    if (cachedBatches[batchNum] && cachedBatches[batchNum][batchPageNum]) {
+      setCurrentPageData(cachedBatches[batchNum][batchPageNum]);
+      setIsCurrentPageLoading(false);
+    } else {
+      setIsCurrentPageLoading(true);
+    }
+  }, [page, cachedBatches]);
+
+  // This updates the page number and manages the URL
+  const updatePage = (newPage: number) => {
+    setPage(newPage);
+    router.push(`/admin/connector/${ccPair.id}?page=${newPage}`, {
+      scroll: false,
+    });
+    window.scrollTo({
+      top: 0,
+      left: 0,
+      behavior: "smooth",
+    });
+  };
+
+  if (isCurrentPageLoading || !currentPageData) {
+    return <ThreeDotsLoader />;
+  }
+
+  if (currentPageError) {
+    return (
+      <ErrorCallout
+        errorTitle={`Failed to fetch info on Connector with ID ${ccPair.id}`}
+        errorMsg={currentPageError?.toString() || "Unknown error"}
+      />
+    );
+  }
+
+  // This is the index attempt that the user wants to view the trace for
+  const indexAttemptToDisplayTraceFor = currentPageData?.index_attempts?.find(
     (indexAttempt) => indexAttempt.id === indexAttemptTracePopupId
   );
-  const [copyClicked, setCopyClicked] = useState(false);
 
   return (
     <>
@@ -55,101 +197,92 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
           </TableRow>
         </TableHead>
         <TableBody>
-          {ccPair.index_attempts
-            .slice(NUM_IN_PAGE * (page - 1), NUM_IN_PAGE * page)
-            .map((indexAttempt) => {
-              const docsPerMinute =
-                getDocsProcessedPerMinute(indexAttempt)?.toFixed(2);
-              return (
-                <TableRow key={indexAttempt.id}>
-                  <TableCell>
-                    {indexAttempt.time_started
-                      ? localizeAndPrettify(indexAttempt.time_started)
-                      : "-"}
-                  </TableCell>
-                  <TableCell>
-                    <IndexAttemptStatus
-                      status={indexAttempt.status || "not_started"}
-                      size="xs"
-                    />
-                    {docsPerMinute && (
-                      <div className="text-xs mt-1">
-                        {docsPerMinute} docs / min
-                      </div>
-                    )}
-                  </TableCell>
-                  <TableCell>
-                    <div className="flex">
-                      <div className="text-right">
-                        <div>{indexAttempt.new_docs_indexed}</div>
-                        {indexAttempt.docs_removed_from_index > 0 && (
-                          <div className="text-xs w-52 text-wrap flex italic overflow-hidden whitespace-normal px-1">
-                            (also removed {indexAttempt.docs_removed_from_index}{" "}
-                            docs that were detected as deleted in the source)
-                          </div>
-                        )}
-                      </div>
+          {currentPageData.index_attempts.map((indexAttempt) => {
+            const docsPerMinute =
+              getDocsProcessedPerMinute(indexAttempt)?.toFixed(2);
+            return (
+              <TableRow key={indexAttempt.id}>
+                <TableCell>
+                  {indexAttempt.time_started
+                    ? localizeAndPrettify(indexAttempt.time_started)
+                    : "-"}
+                </TableCell>
+                <TableCell>
+                  <IndexAttemptStatus
+                    status={indexAttempt.status || "not_started"}
+                    size="xs"
+                  />
+                  {docsPerMinute && (
+                    <div className="text-xs mt-1">
+                      {docsPerMinute} docs / min
                     </div>
-                  </TableCell>
-                  <TableCell>{indexAttempt.total_docs_indexed}</TableCell>
-                  <TableCell>
-                    <div>
-                      {indexAttempt.error_count > 0 && (
-                        <Link
-                          className="cursor-pointer my-auto"
-                          href={`/admin/indexing/${indexAttempt.id}`}
-                        >
-                          <Text className="flex flex-wrap text-link whitespace-normal">
-                            <SearchIcon />
-                            &nbsp;View Errors
-                          </Text>
-                        </Link>
-                      )}
-
-                      {indexAttempt.status === "success" && (
-                        <Text className="flex flex-wrap whitespace-normal">
-                          {"-"}
-                        </Text>
-                      )}
-
-                      {indexAttempt.status === "failed" &&
-                        indexAttempt.error_msg && (
-                          <Text className="flex flex-wrap whitespace-normal">
-                            {indexAttempt.error_msg}
-                          </Text>
-                        )}
-
-                      {indexAttempt.full_exception_trace && (
-                        <div
-                          onClick={() => {
-                            setIndexAttemptTracePopupId(indexAttempt.id);
-                          }}
-                          className="mt-2 text-link cursor-pointer select-none"
-                        >
-                          View Full Trace
+                  )}
+                </TableCell>
+                <TableCell>
+                  <div className="flex">
+                    <div className="text-right">
+                      <div>{indexAttempt.new_docs_indexed}</div>
+                      {indexAttempt.docs_removed_from_index > 0 && (
+                        <div className="text-xs w-52 text-wrap flex italic overflow-hidden whitespace-normal px-1">
+                          (also removed {indexAttempt.docs_removed_from_index}{" "}
+                          docs that were detected as deleted in the source)
                         </div>
                       )}
                     </div>
-                  </TableCell>
-                </TableRow>
-              );
-            })}
+                  </div>
+                </TableCell>
+                <TableCell>{indexAttempt.total_docs_indexed}</TableCell>
+                <TableCell>
+                  <div>
+                    {indexAttempt.error_count > 0 && (
+                      <Link
+                        className="cursor-pointer my-auto"
+                        href={`/admin/indexing/${indexAttempt.id}`}
+                      >
+                        <Text className="flex flex-wrap text-link whitespace-normal">
+                          <SearchIcon />
+                          &nbsp;View Errors
+                        </Text>
+                      </Link>
+                    )}
+
+                    {indexAttempt.status === "success" && (
+                      <Text className="flex flex-wrap whitespace-normal">
+                        {"-"}
+                      </Text>
+                    )}
+
+                    {indexAttempt.status === "failed" &&
+                      indexAttempt.error_msg && (
+                        <Text className="flex flex-wrap whitespace-normal">
+                          {indexAttempt.error_msg}
+                        </Text>
+                      )}
+
+                    {indexAttempt.full_exception_trace && (
+                      <div
+                        onClick={() => {
+                          setIndexAttemptTracePopupId(indexAttempt.id);
+                        }}
+                        className="mt-2 text-link cursor-pointer select-none"
+                      >
+                        View Full Trace
+                      </div>
+                    )}
+                  </div>
+                </TableCell>
+              </TableRow>
+            );
+          })}
         </TableBody>
       </Table>
-      {ccPair.index_attempts.length > NUM_IN_PAGE && (
+      {totalPages > 1 && (
         <div className="mt-3 flex">
           <div className="mx-auto">
             <PageSelector
-              totalPages={Math.ceil(ccPair.index_attempts.length / NUM_IN_PAGE)}
+              totalPages={totalPages}
               currentPage={page}
-              onPageChange={(newPage) => {
-                setPage(newPage);
-                window.scrollTo({
-                  top: 0,
-                  left: 0,
-                  behavior: "smooth",
-                });
-              }}
+              onPageChange={updatePage}
             />
           </div>
         </div>

web/src/app/admin/connector/[ccPairId]/page.tsx

@@ -1,7 +1,6 @@
 "use client";
 
 import { CCPairFullInfo, ConnectorCredentialPairStatus } from "./types";
-import { HealthCheckBanner } from "@/components/health/healthcheck";
 import { CCPairStatus } from "@/components/Status";
 import { BackButton } from "@/components/BackButton";
 import { Button, Divider, Title } from "@tremor/react";
@@ -11,7 +10,6 @@ import { ModifyStatusButtonCluster } from "./ModifyStatusButtonCluster";
 import { DeletionButton } from "./DeletionButton";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { ReIndexButton } from "./ReIndexButton";
-import { isCurrentlyDeleting } from "@/lib/documentDeletion";
 import { ValidSources } from "@/lib/types";
 import useSWR, { mutate } from "swr";
 import { errorHandlingFetcher } from "@/lib/fetcher";
@@ -86,24 +84,13 @@ function Main({ ccPairId }: { ccPairId: number }) {
     return (
       <ErrorCallout
         errorTitle={`Failed to fetch info on Connector with ID ${ccPairId}`}
-        errorMsg={error?.info?.detail || error.toString()}
+        errorMsg={error?.info?.detail || error?.toString() || "Unknown error"}
       />
     );
   }
 
-  const lastIndexAttempt = ccPair.index_attempts[0];
   const isDeleting = ccPair.status === ConnectorCredentialPairStatus.DELETING;
 
-  // figure out if we need to artificially deflate the number of docs indexed.
-  // This is required since the total number of docs indexed by a CC Pair is
-  // updated before the new docs for an indexing attempt. If we don't do this,
-  // there is a mismatch between these two numbers which may confuse users.
-  const totalDocsIndexed =
-    lastIndexAttempt?.status === "in_progress" &&
-    ccPair.index_attempts.length === 1
-      ? lastIndexAttempt.total_docs_indexed
-      : ccPair.num_docs_indexed;
-
   const refresh = () => {
     mutate(buildCCPairInfoUrl(ccPairId));
   };
@@ -182,13 +169,13 @@ function Main({ ccPairId }: { ccPairId: number }) {
         )}
       </div>
       <CCPairStatus
-        status={lastIndexAttempt?.status || "not_started"}
+        status={ccPair.last_index_attempt_status || "not_started"}
         disabled={ccPair.status === ConnectorCredentialPairStatus.PAUSED}
         isDeleting={isDeleting}
       />
       <div className="text-sm mt-1">
         Total Documents Indexed:{" "}
-        <b className="text-emphasis">{totalDocsIndexed}</b>
+        <b className="text-emphasis">{ccPair.num_docs_indexed}</b>
       </div>
       {!ccPair.is_editable_for_current_user && (
         <div className="text-sm mt-2 text-neutral-500 italic">

web/src/app/admin/connector/[ccPairId]/types.ts

@@ -1,6 +1,10 @@
 import { Connector } from "@/lib/connectors/connectors";
 import { Credential } from "@/lib/connectors/credentials";
-import { DeletionAttemptSnapshot, IndexAttemptSnapshot } from "@/lib/types";
+import {
+  DeletionAttemptSnapshot,
+  IndexAttemptSnapshot,
+  ValidStatuses,
+} from "@/lib/types";
 
 export enum ConnectorCredentialPairStatus {
   ACTIVE = "ACTIVE",
@@ -15,8 +19,15 @@ export interface CCPairFullInfo {
   num_docs_indexed: number;
   connector: Connector<any>;
   credential: Credential<any>;
-  index_attempts: IndexAttemptSnapshot[];
+  number_of_index_attempts: number;
+  last_index_attempt_status: ValidStatuses | null;
   latest_deletion_attempt: DeletionAttemptSnapshot | null;
   is_public: boolean;
   is_editable_for_current_user: boolean;
 }
+
+export interface PaginatedIndexAttempts {
+  index_attempts: IndexAttemptSnapshot[];
+  page: number;
+  total_pages: number;
+}

web/src/lib/ss/ccPair.ts

@@ -1,5 +0,0 @@
-import { fetchSS } from "../utilsSS";
-
-export async function getCCPairSS(ccPairId: number) {
-  return fetchSS(`/manage/admin/cc-pair/${ccPairId}`);
-}