Add Github admin page + adjust way index APIs work

backend/danswer/background/update.py

@@ -2,6 +2,8 @@ import time
 from typing import cast
 
 from danswer.configs.constants import DocumentSource
+from danswer.connectors.factory import build_connector
+from danswer.connectors.models import InputType
 from danswer.connectors.slack.config import get_pull_frequency
 from danswer.connectors.slack.pull import PeriodicSlackLoader
 from danswer.connectors.web.pull import WebLoader
@@ -35,6 +37,8 @@ def run_update() -> None:
     current_time = int(time.time())
 
     # Slack
+    # TODO (chris): make Slack use the same approach as other connectors /
+    # make other connectors periodic
     try:
         pull_frequency = get_pull_frequency()
     except ConfigNotFoundError:
@@ -56,17 +60,15 @@ def run_update() -> None:
                 indexing_pipeline(doc_batch)
             dynamic_config_store.store(last_slack_pull_key, current_time)
 
-    # Web
     # TODO (chris): make this more efficient / in a single transaction to
     # prevent race conditions across multiple background jobs. For now,
     # this assumes we only ever run a single background job at a time
-    # TODO (chris): make this generic for all pull connectors (not just web)
     not_started_index_attempts = fetch_index_attempts(
-        sources=[DocumentSource.WEB], statuses=[IndexingStatus.NOT_STARTED]
+        input_types=[InputType.PULL], statuses=[IndexingStatus.NOT_STARTED]
     )
     for not_started_index_attempt in not_started_index_attempts:
         logger.info(
-            "Attempting to index website with IndexAttempt id: "
+            "Attempting to index with IndexAttempt id: "
             f"{not_started_index_attempt.id}, source: "
             f"{not_started_index_attempt.source}, input_type: "
             f"{not_started_index_attempt.input_type}, and connector_specific_config: "
@@ -78,17 +80,25 @@ def run_update() -> None:
         )
 
         error_msg = None
-        base_url = not_started_index_attempt.connector_specific_config["url"]
         try:
             # TODO (chris): spawn processes to parallelize / take advantage of
             # multiple cores + implement retries
+            connector = build_connector(
+                source=not_started_index_attempt.source,
+                input_type=InputType.PULL,
+                connector_specific_config=not_started_index_attempt.connector_specific_config,
+            )
+
             document_ids: list[str] = []
-            for doc_batch in WebLoader(base_url=base_url).load():
+            for doc_batch in connector.load():
                 chunks = indexing_pipeline(doc_batch)
                 document_ids.extend([chunk.source_document.id for chunk in chunks])
         except Exception as e:
             logger.exception(
-                "Failed to index website with url %s due to: %s", base_url, e
+                "Failed to index for source %s with config %s due to: %s",
+                not_started_index_attempt.source,
+                not_started_index_attempt.connector_specific_config,
+                e,
             )
             error_msg = str(e)
 

backend/danswer/connectors/factory.py

@@ -0,0 +1,40 @@
+from typing import Any
+
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.github.batch import BatchGithubLoader
+from danswer.connectors.google_drive.batch import BatchGoogleDriveLoader
+from danswer.connectors.interfaces import PullLoader
+from danswer.connectors.interfaces import RangePullLoader
+from danswer.connectors.models import InputType
+from danswer.connectors.slack.batch import BatchSlackLoader
+from danswer.connectors.slack.pull import PeriodicSlackLoader
+from danswer.connectors.web.pull import WebLoader
+
+
+class ConnectorMissingException(Exception):
+    pass
+
+
+def build_connector(
+    source: DocumentSource,
+    input_type: InputType,
+    connector_specific_config: dict[str, Any],
+) -> PullLoader | RangePullLoader:
+    if source == DocumentSource.SLACK:
+        if input_type == InputType.PULL:
+            return PeriodicSlackLoader(**connector_specific_config)
+        if input_type == InputType.LOAD_STATE:
+            return BatchSlackLoader(**connector_specific_config)
+    elif source == DocumentSource.GOOGLE_DRIVE:
+        if input_type == InputType.PULL:
+            return BatchGoogleDriveLoader(**connector_specific_config)
+    elif source == DocumentSource.GITHUB:
+        if input_type == InputType.PULL:
+            return BatchGithubLoader(**connector_specific_config)
+    elif source == DocumentSource.WEB:
+        if input_type == InputType.PULL:
+            return WebLoader(**connector_specific_config)
+
+    raise ConnectorMissingException(
+        f"Connector not found for source={source}, input_type={input_type}"
+    )

backend/danswer/connectors/interfaces.py

@@ -9,6 +9,7 @@ from danswer.connectors.models import Document
 SecondsSinceUnixEpoch = float
 
 
+# TODO (chris): rename from Loader -> Connector
 class PullLoader:
     @abc.abstractmethod
     def load(self) -> Generator[List[Document], None, None]:

backend/danswer/db/index_attempt.py

@@ -1,4 +1,5 @@
 from danswer.configs.constants import DocumentSource
+from danswer.connectors.models import InputType
 from danswer.db.engine import build_engine
 from danswer.db.models import IndexAttempt
 from danswer.db.models import IndexingStatus
@@ -20,6 +21,7 @@ def fetch_index_attempts(
     *,
     sources: list[DocumentSource] | None = None,
     statuses: list[IndexingStatus] | None = None,
+    input_types: list[InputType] | None = None,
 ) -> list[IndexAttempt]:
     with Session(build_engine(), future=True, expire_on_commit=False) as session:
         stmt = select(IndexAttempt)
@@ -27,6 +29,8 @@ def fetch_index_attempts(
             stmt = stmt.where(IndexAttempt.source.in_(sources))
         if statuses:
             stmt = stmt.where(IndexAttempt.status.in_(statuses))
+        if input_types:
+            stmt = stmt.where(IndexAttempt.input_type.in_(input_types))
         results = session.scalars(stmt)
         return list(results.all())
 

backend/danswer/server/admin.py

@@ -1,6 +1,9 @@
+from typing import Any
+
 from danswer.auth.users import current_admin_user
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import NO_AUTH_USER
+from danswer.connectors.factory import build_connector
 from danswer.connectors.google_drive.connector_auth import get_auth_url
 from danswer.connectors.google_drive.connector_auth import get_drive_tokens
 from danswer.connectors.google_drive.connector_auth import save_access_tokens
@@ -19,11 +22,10 @@ from danswer.server.models import AuthStatus
 from danswer.server.models import AuthUrl
 from danswer.server.models import GDriveCallback
 from danswer.server.models import IndexAttemptSnapshot
-from danswer.server.models import ListWebsiteIndexAttemptsResponse
-from danswer.server.models import WebIndexAttemptRequest
 from danswer.utils.logging import setup_logger
 from fastapi import APIRouter
 from fastapi import Depends
+from pydantic import BaseModel
 
 router = APIRouter(prefix="/admin")
 
@@ -67,29 +69,51 @@ def modify_slack_config(
     update_slack_config(slack_config)
 
 
-@router.post("/connectors/web/index-attempt", status_code=201)
+class IndexAttemptRequest(BaseModel):
-def index_website(
+    input_type: InputType = InputType.PULL
-    web_index_attempt_request: WebIndexAttemptRequest,
+    connector_specific_config: dict[str, Any]
+
+
+@router.post("/connectors/{source}/index-attempt", status_code=201)
+def index(
+    source: DocumentSource,
+    index_attempt_request: IndexAttemptRequest,
     _: User = Depends(current_admin_user),
 ) -> None:
-    index_request = IndexAttempt(
+    # validate that the connector specified by the source / input_type combination
-        source=DocumentSource.WEB,
+    # exists AND that the connector_specific_config is valid for that connector type
-        input_type=InputType.PULL,
+    build_connector(
-        connector_specific_config={"url": web_index_attempt_request.url},
+        source=source,
-        status=IndexingStatus.NOT_STARTED,
+        input_type=index_attempt_request.input_type,
+        connector_specific_config=index_attempt_request.connector_specific_config,
+    )
+
+    # once validated, insert the index attempt into the database where it will
+    # get picked up by a background job
+    insert_index_attempt(
+        index_attempt=IndexAttempt(
+            source=source,
+            input_type=index_attempt_request.input_type,
+            connector_specific_config=index_attempt_request.connector_specific_config,
+            status=IndexingStatus.NOT_STARTED,
+        )
     )
-    insert_index_attempt(index_request)
 
 
-@router.get("/connectors/web/index-attempt")
+class ListIndexAttemptsResponse(BaseModel):
-def list_website_index_attempts(
+    index_attempts: list[IndexAttemptSnapshot]
+
+
+@router.get("/connectors/{source}/index-attempt")
+def list_index_attempts(
+    source: DocumentSource,
     _: User = Depends(current_admin_user),
-) -> ListWebsiteIndexAttemptsResponse:
+) -> ListIndexAttemptsResponse:
-    index_attempts = fetch_index_attempts(sources=[DocumentSource.WEB])
+    index_attempts = fetch_index_attempts(sources=[source])
-    return ListWebsiteIndexAttemptsResponse(
+    return ListIndexAttemptsResponse(
         index_attempts=[
             IndexAttemptSnapshot(
-                url=index_attempt.connector_specific_config["url"],
+                connector_specific_config=index_attempt.connector_specific_config,
                 status=index_attempt.status,
                 time_created=index_attempt.time_created,
                 time_updated=index_attempt.time_updated,

backend/danswer/server/models.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from typing import Any
 
 from danswer.datastores.interfaces import DatastoreFilter
 from danswer.db.models import IndexingStatus
@@ -49,12 +50,8 @@ class UserByEmail(BaseModel):
     user_email: str
 
 
-class WebIndexAttemptRequest(BaseModel):
-    url: str
-
-
 class IndexAttemptSnapshot(BaseModel):
-    url: str
+    connector_specific_config: dict[str, Any]
     status: IndexingStatus
     time_created: datetime
     time_updated: datetime

web/src/app/admin/connectors/github/page.tsx

@@ -0,0 +1,58 @@
+"use client";
+
+import * as Yup from "yup";
+import { IndexForm } from "@/components/admin/connectors/Form";
+import {
+  ConnectorStatus,
+  ReccuringConnectorStatus,
+} from "@/components/admin/connectors/RecurringConnectorStatus";
+import { GithubIcon } from "@/components/icons/icons";
+import { TextFormField } from "@/components/admin/connectors/Field";
+
+export default function Page() {
+  return (
+    <div className="mx-auto">
+      <div className="border-solid border-gray-600 border-b mb-4 pb-2 flex">
+        <GithubIcon size="32" />
+        <h1 className="text-3xl font-bold pl-2">Github PRs</h1>
+      </div>
+
+      <h2 className="text-xl font-bold pl-2 mb-2 mt-6 ml-auto mr-auto">
+        Status
+      </h2>
+      <ReccuringConnectorStatus
+        status={ConnectorStatus.Running}
+        source="github"
+      />
+
+      {/* TODO: make this periodic */}
+      <h2 className="text-xl font-bold pl-2 mb-2 mt-6 ml-auto mr-auto">
+        Request Indexing
+      </h2>
+      <div className="border-solid border-gray-600 border rounded-md p-6">
+        <IndexForm
+          source="github"
+          formBody={
+            <>
+              <TextFormField name="repo_owner" label="Owner of repo:" />
+              <TextFormField name="repo_name" label="Name of repo:" />
+            </>
+          }
+          validationSchema={Yup.object().shape({
+            repo_owner: Yup.string().required(
+              "Please enter the owner of the repo scrape e.g. danswer-ai"
+            ),
+            repo_name: Yup.string().required(
+              "Please enter the name of the repo scrape e.g. danswer "
+            ),
+          })}
+          initialValues={{
+            repo_owner: "",
+            repo_name: "",
+          }}
+          onSubmit={(isSuccess) => console.log(isSuccess)}
+        />
+      </div>
+    </div>
+  );
+}

web/src/app/admin/connectors/interfaces.ts

@@ -0,0 +1,17 @@
+export interface SlackConfig {
+  slack_bot_token: string;
+  workspace_id: string;
+  pull_frequency: number;
+}
+
+export interface IndexAttempt {
+  connector_specific_config: { [key: string]: any };
+  status: "success" | "failure" | "in_progress" | "not_started";
+  time_created: string;
+  time_updated: string;
+  docs_indexed: number;
+}
+
+export interface ListIndexingResponse {
+  index_attempts: IndexAttempt[];
+}

web/src/app/admin/connectors/slack/SlackForm.tsx

@@ -3,7 +3,7 @@ import { Formik, Form, FormikHelpers } from "formik";
 import * as Yup from "yup";
 import { Popup } from "../../../../components/admin/connectors/Popup";
 import { TextFormField } from "../../../../components/admin/connectors/Field";
-import { SlackConfig } from "./interfaces";
+import { SlackConfig } from "../interfaces";
 
 const validationSchema = Yup.object().shape({
   slack_bot_token: Yup.string().required("Please enter your Slack Bot Token"),

web/src/app/admin/connectors/slack/interfaces.ts

@@ -1,11 +0,0 @@
-export interface SlackConfig {
-  slack_bot_token: string;
-  workspace_id: string;
-  pull_frequency: number;
-}
-
-// interface SlackIndexAttempt {}
-
-// interface ListSlackIndexingResponse {
-//   index_attempts: SlackIndexAttempt[];
-// }

web/src/app/admin/connectors/slack/page.tsx

@@ -8,7 +8,7 @@ import { SlackForm } from "@/app/admin/connectors/slack/SlackForm";
 import { SlackIcon } from "@/components/icons/icons";
 import { fetcher } from "@/lib/fetcher";
 import useSWR, { useSWRConfig } from "swr";
-import { SlackConfig } from "./interfaces";
+import { SlackConfig } from "../interfaces";
 import { ThinkingAnimation } from "@/components/Thinking";
 
 const MainSection = () => {
@@ -47,6 +47,7 @@ const MainSection = () => {
               ? ConnectorStatus.Running
               : ConnectorStatus.NotSetup
           }
+          source="slack"
         />
       }
 

web/src/app/admin/connectors/web/WebIndexForm.tsx

@@ -1,94 +0,0 @@
-import React, { useState } from "react";
-import { Formik, Form, FormikHelpers } from "formik";
-import * as Yup from "yup";
-import { Popup } from "../../../../components/admin/connectors/Popup";
-import { TextFormField } from "../../../../components/admin/connectors/Field";
-
-interface FormData {
-  url: string;
-}
-
-const validationSchema = Yup.object().shape({
-  url: Yup.string().required(
-    "Please enter the website URL to scrape e.g. https://docs.github.com/en/actions"
-  ),
-});
-
-const handleSubmit = async (
-  values: FormData,
-  { setSubmitting }: FormikHelpers<FormData>,
-  setPopup: (
-    popup: { message: string; type: "success" | "error" } | null
-  ) => void
-): Promise<boolean> => {
-  setSubmitting(true);
-  let isSuccess = false;
-  try {
-    const response = await fetch("/api/admin/connectors/web/index-attempt", {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify(values),
-    });
-
-    if (response.ok) {
-      isSuccess = true;
-      setPopup({ message: "Success!", type: "success" });
-    } else {
-      const errorData = await response.json();
-      setPopup({ message: `Error: ${errorData.detail}`, type: "error" });
-    }
-  } catch (error) {
-    setPopup({ message: `Error: ${error}`, type: "error" });
-  } finally {
-    setSubmitting(false);
-    setTimeout(() => {
-      setPopup(null);
-    }, 3000);
-    return isSuccess;
-  }
-};
-
-interface SlackFormProps {
-  onSubmit: (isSuccess: boolean) => void;
-}
-
-export const WebIndexForm: React.FC<SlackFormProps> = ({ onSubmit }) => {
-  const [popup, setPopup] = useState<{
-    message: string;
-    type: "success" | "error";
-  } | null>(null);
-
-  return (
-    <>
-      {popup && <Popup message={popup.message} type={popup.type} />}
-      <Formik
-        initialValues={{ url: "" }}
-        validationSchema={validationSchema}
-        onSubmit={(values, formikHelpers) =>
-          handleSubmit(values, formikHelpers, setPopup).then((isSuccess) =>
-            onSubmit(isSuccess)
-          )
-        }
-      >
-        {({ isSubmitting }) => (
-          <Form>
-            <TextFormField name="url" label="URL to Index:" />
-            <button
-              type="submit"
-              disabled={isSubmitting}
-              className={
-                "bg-slate-500 hover:bg-slate-700 text-white " +
-                "font-bold py-2 px-4 rounded focus:outline-none " +
-                "focus:shadow-outline w-full max-w-xs"
-              }
-            >
-              Index
-            </button>
-          </Form>
-        )}
-      </Formik>
-    </>
-  );
-};

web/src/app/admin/connectors/web/page.tsx

@@ -1,25 +1,16 @@
 "use client";
 
 import useSWR, { useSWRConfig } from "swr";
+import * as Yup from "yup";
 
 import { BasicTable } from "@/components/admin/connectors/BasicTable";
-import { WebIndexForm } from "@/app/admin/connectors/web/WebIndexForm";
 import { ThinkingAnimation } from "@/components/Thinking";
 import { timeAgo } from "@/lib/time";
 import { GlobeIcon } from "@/components/icons/icons";
 import { fetcher } from "@/lib/fetcher";
-
+import { IndexAttempt, ListIndexingResponse } from "../interfaces";
-interface WebsiteIndexAttempt {
+import { IndexForm } from "@/components/admin/connectors/Form";
-  url: string;
+import { TextFormField } from "@/components/admin/connectors/Field";
-  status: "success" | "failure" | "in_progress" | "not_started";
-  time_created: string;
-  time_updated: string;
-  docs_indexed: number;
-}
-
-interface ListWebIndexingResponse {
-  index_attempts: WebsiteIndexAttempt[];
-}
 
 const COLUMNS = [
   { header: "Base URL", key: "url" },
@@ -30,28 +21,29 @@ const COLUMNS = [
 
 export default function Web() {
   const { mutate } = useSWRConfig();
-  const { data, isLoading, error } = useSWR<ListWebIndexingResponse>(
+  const { data, isLoading, error } = useSWR<ListIndexingResponse>(
     "/api/admin/connectors/web/index-attempt",
     fetcher
   );
 
-  const urlToLatestIndexAttempt = new Map<string, WebsiteIndexAttempt>();
+  const urlToLatestIndexAttempt = new Map<string, IndexAttempt>();
   const urlToLatestIndexSuccess = new Map<string, string>();
   data?.index_attempts?.forEach((indexAttempt) => {
-    const latestIndexAttempt = urlToLatestIndexAttempt.get(indexAttempt.url);
+    const url = indexAttempt.connector_specific_config.base_url;
+    const latestIndexAttempt = urlToLatestIndexAttempt.get(url);
     if (
       !latestIndexAttempt ||
       indexAttempt.time_created > latestIndexAttempt.time_created
     ) {
-      urlToLatestIndexAttempt.set(indexAttempt.url, indexAttempt);
+      urlToLatestIndexAttempt.set(url, indexAttempt);
     }
 
-    const latestIndexSuccess = urlToLatestIndexSuccess.get(indexAttempt.url);
+    const latestIndexSuccess = urlToLatestIndexSuccess.get(url);
     if (
       indexAttempt.status === "success" &&
       (!latestIndexSuccess || indexAttempt.time_updated > latestIndexSuccess)
     ) {
-      urlToLatestIndexSuccess.set(indexAttempt.url, indexAttempt.time_updated);
+      urlToLatestIndexSuccess.set(url, indexAttempt.time_updated);
     }
   });
 
@@ -65,7 +57,15 @@ export default function Web() {
         Request Indexing
       </h2>
       <div className="border-solid border-gray-600 border rounded-md p-6">
-        <WebIndexForm
+        <IndexForm
+          source="web"
+          formBody={<TextFormField name="base_url" label="URL to Index:" />}
+          validationSchema={Yup.object().shape({
+            base_url: Yup.string().required(
+              "Please enter the website URL to scrape e.g. https://docs.github.com/en/actions"
+            ),
+          })}
+          initialValues={{ base_url: "" }}
           onSubmit={(success) => {
             if (success) {
               mutate("/api/admin/connectors/web/index-attempt");
@@ -87,22 +87,21 @@ export default function Web() {
           data={
             urlToLatestIndexAttempt.size > 0
               ? Array.from(urlToLatestIndexAttempt.values()).map(
-                  (indexAttempt) => ({
+                  (indexAttempt) => {
-                    ...indexAttempt,
+                    const url = indexAttempt.connector_specific_config
-                    indexed_at:
+                      .base_url as string;
-                      timeAgo(urlToLatestIndexSuccess.get(indexAttempt.url)) ||
+                    return {
-                      "-",
+                      indexed_at:
-                    docs_indexed: indexAttempt.docs_indexed || "-",
+                        timeAgo(urlToLatestIndexSuccess.get(url)) || "-",
-                    url: (
+                      docs_indexed: indexAttempt.docs_indexed || "-",
-                      <a
+                      url: (
-                        className="text-blue-500"
+                        <a className="text-blue-500" target="_blank" href={url}>
-                        target="_blank"
+                          {url}
-                        href={indexAttempt.url}
+                        </a>
-                      >
+                      ),
-                        {indexAttempt.url}
+                      status: indexAttempt.status,
-                      </a>
+                    };
-                    ),
+                  }
-                  })
                 )
               : []
           }

web/src/app/admin/layout.tsx

@@ -1,6 +1,6 @@
 import { Header } from "@/components/Header";
 import { Sidebar } from "@/components/admin/connectors/Sidebar";
-import { GlobeIcon, SlackIcon } from "@/components/icons/icons";
+import { GithubIcon, GlobeIcon, SlackIcon } from "@/components/icons/icons";
 import { getCurrentUserSS } from "@/lib/userSS";
 import { redirect } from "next/navigation";
 
@@ -45,6 +45,15 @@ export default async function AdminLayout({
                   ),
                   link: "/admin/connectors/web",
                 },
+                {
+                  name: (
+                    <div className="flex">
+                      <GithubIcon size="16" />
+                      <div className="ml-1">Github</div>
+                    </div>
+                  ),
+                  link: "/admin/connectors/github",
+                },
               ],
             },
           ]}

web/src/components/admin/connectors/Form.tsx

@@ -0,0 +1,106 @@
+import React, { useState } from "react";
+import { Formik, Form, FormikHelpers } from "formik";
+import * as Yup from "yup";
+import { Popup } from "./Popup";
+
+type ValidSources = "web" | "github";
+
+const handleSubmit = async (
+  source: ValidSources,
+  values: Yup.AnyObject,
+  { setSubmitting }: FormikHelpers<Yup.AnyObject>,
+  setPopup: (
+    popup: { message: string; type: "success" | "error" } | null
+  ) => void
+): Promise<boolean> => {
+  setSubmitting(true);
+  let isSuccess = false;
+  try {
+    const response = await fetch(
+      `/api/admin/connectors/${source}/index-attempt`,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({ connector_specific_config: values }),
+      }
+    );
+
+    if (response.ok) {
+      isSuccess = true;
+      setPopup({ message: "Success!", type: "success" });
+    } else {
+      const errorData = await response.json();
+      setPopup({ message: `Error: ${errorData.detail}`, type: "error" });
+    }
+  } catch (error) {
+    setPopup({ message: `Error: ${error}`, type: "error" });
+  } finally {
+    setSubmitting(false);
+    setTimeout(() => {
+      setPopup(null);
+    }, 3000);
+    return isSuccess;
+  }
+};
+
+interface IndexFormProps<YupObjectType extends Yup.AnyObject> {
+  source: ValidSources;
+  formBody: JSX.Element | null;
+  validationSchema: Yup.ObjectSchema<YupObjectType>;
+  initialValues: YupObjectType;
+  onSubmit: (isSuccess: boolean) => void;
+  additionalNonFormValues?: Yup.AnyObject;
+}
+
+export function IndexForm<YupObjectType extends Yup.AnyObject>({
+  source,
+  formBody,
+  validationSchema,
+  initialValues,
+  onSubmit,
+  additionalNonFormValues = {},
+}: IndexFormProps<YupObjectType>): JSX.Element {
+  const [popup, setPopup] = useState<{
+    message: string;
+    type: "success" | "error";
+  } | null>(null);
+
+  return (
+    <>
+      {popup && <Popup message={popup.message} type={popup.type} />}
+      <Formik
+        initialValues={initialValues}
+        validationSchema={validationSchema}
+        onSubmit={(values, formikHelpers) => {
+          handleSubmit(
+            source,
+            { ...values, ...additionalNonFormValues },
+            formikHelpers as FormikHelpers<Yup.AnyObject>,
+            setPopup
+          ).then((isSuccess) => onSubmit(isSuccess));
+        }}
+      >
+        {({ isSubmitting }) => (
+          <Form>
+            {formBody}
+            <div className="flex">
+              <button
+                type="submit"
+                disabled={isSubmitting}
+                className={
+                  "bg-slate-500 hover:bg-slate-700 text-white " +
+                  "font-bold py-2 px-4 rounded focus:outline-none " +
+                  "focus:shadow-outline w-full max-w-sm mx-auto"
+                }
+              >
+                Index
+              </button>
+            </div>
+          </Form>
+        )}
+      </Formik>
+    </>
+  );
+}

web/src/components/admin/connectors/RecurringConnectorStatus.tsx

@@ -1,4 +1,10 @@
+"use client";
+
+import { ListIndexingResponse } from "@/app/admin/connectors/interfaces";
+import { fetcher } from "@/lib/fetcher";
+import { timeAgo } from "@/lib/time";
 import { CheckCircle, MinusCircle } from "@phosphor-icons/react";
+import useSWR from "swr";
 
 export enum ConnectorStatus {
   Running = "Running",
@@ -7,16 +13,39 @@ export enum ConnectorStatus {
 
 interface ReccuringConnectorStatusProps {
   status: ConnectorStatus;
+  source: string;
 }
 
 export const ReccuringConnectorStatus = ({
   status,
+  source,
 }: ReccuringConnectorStatusProps) => {
+  const { data } = useSWR<ListIndexingResponse>(
+    `/api/admin/connectors/${source}/index-attempt`,
+    fetcher
+  );
+
+  const lastSuccessfulAttempt = data?.index_attempts
+    .filter((attempt) => attempt.status === "success")
+    .sort((a, b) => {
+      if (a.time_updated === b.time_updated) {
+        return 0;
+      }
+      return a.time_updated > b.time_updated ? -1 : 1;
+    })[0];
+
   if (status === ConnectorStatus.Running) {
     return (
-      <div className="text-emerald-600 flex align-middle text-center">
+      <div>
-        <CheckCircle size={20} className="my-auto" />
+        <div className="text-emerald-600 flex align-middle text-center">
-        <p className="my-auto ml-1">{status}</p>
+          <CheckCircle size={20} className="my-auto" />
+          <p className="my-auto ml-1">{status}</p>
+        </div>
+        {lastSuccessfulAttempt && (
+          <p className="text-xs my-auto ml-1">
+            Last updated {timeAgo(lastSuccessfulAttempt.time_updated)}
+          </p>
+        )}
       </div>
     );
   }

web/src/components/icons/icons.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { Globe, SlackLogo } from "@phosphor-icons/react";
+import { Globe, SlackLogo, GithubLogo } from "@phosphor-icons/react";
 
 interface IconProps {
   size?: string;
@@ -22,3 +22,10 @@ export const SlackIcon = ({
 }: IconProps) => {
   return <SlackLogo size={size} className={className} />;
 };
+
+export const GithubIcon = ({
+  size = "16",
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return <GithubLogo size={size} className={className} />;
+};

web/src/components/search/SearchResultsDisplay.tsx

@@ -3,6 +3,7 @@ import { Globe, SlackLogo, GoogleDriveLogo } from "@phosphor-icons/react";
 import "tailwindcss/tailwind.css";
 import { Quote, Document } from "./types";
 import { ThinkingAnimation } from "../Thinking";
+import { GithubIcon } from "../icons/icons";
 
 interface SearchResultsDisplayProps {
   answer: string | null;
@@ -22,6 +23,8 @@ const getSourceIcon = (sourceType: string) => {
       return <SlackLogo size={ICON_SIZE} className={ICON_STYLE} />;
     case "google_drive":
       return <GoogleDriveLogo size={ICON_SIZE} className={ICON_STYLE} />;
+    case "github":
+      return <GithubIcon size={ICON_SIZE} className={ICON_STYLE} />;
     default:
       return null;
   }
@@ -102,7 +105,7 @@ export const SearchResultsDisplay: React.FC<SearchResultsDisplayProps> = ({
               >
                 {getSourceIcon(doc.source_type)}
                 <p className="truncate break-all">
-                  {doc.semantic_name || doc.document_id}
+                  {doc.semantic_identifier || doc.document_id}
                 </p>
               </a>
               <p className="pl-1 py-3 text-gray-200">{doc.blurb}</p>

web/src/components/search/types.tsx

@@ -11,7 +11,7 @@ export interface Document {
   link: string;
   source_type: string;
   blurb: string;
-  semantic_name: string | null;
+  semantic_identifier: string | null;
 }
 
 export interface SearchResponse {