Add Github admin page + adjust way index APIs work

web/src/app/admin/connectors/github/page.tsx

@@ -0,0 +1,58 @@
+"use client";
+
+import * as Yup from "yup";
+import { IndexForm } from "@/components/admin/connectors/Form";
+import {
+  ConnectorStatus,
+  ReccuringConnectorStatus,
+} from "@/components/admin/connectors/RecurringConnectorStatus";
+import { GithubIcon } from "@/components/icons/icons";
+import { TextFormField } from "@/components/admin/connectors/Field";
+
+export default function Page() {
+  return (
+    <div className="mx-auto">
+      <div className="border-solid border-gray-600 border-b mb-4 pb-2 flex">
+        <GithubIcon size="32" />
+        <h1 className="text-3xl font-bold pl-2">Github PRs</h1>
+      </div>
+
+      <h2 className="text-xl font-bold pl-2 mb-2 mt-6 ml-auto mr-auto">
+        Status
+      </h2>
+      <ReccuringConnectorStatus
+        status={ConnectorStatus.Running}
+        source="github"
+      />
+
+      {/* TODO: make this periodic */}
+      <h2 className="text-xl font-bold pl-2 mb-2 mt-6 ml-auto mr-auto">
+        Request Indexing
+      </h2>
+      <div className="border-solid border-gray-600 border rounded-md p-6">
+        <IndexForm
+          source="github"
+          formBody={
+            <>
+              <TextFormField name="repo_owner" label="Owner of repo:" />
+              <TextFormField name="repo_name" label="Name of repo:" />
+            </>
+          }
+          validationSchema={Yup.object().shape({
+            repo_owner: Yup.string().required(
+              "Please enter the owner of the repo scrape e.g. danswer-ai"
+            ),
+            repo_name: Yup.string().required(
+              "Please enter the name of the repo scrape e.g. danswer "
+            ),
+          })}
+          initialValues={{
+            repo_owner: "",
+            repo_name: "",
+          }}
+          onSubmit={(isSuccess) => console.log(isSuccess)}
+        />
+      </div>
+    </div>
+  );
+}

web/src/app/admin/connectors/interfaces.ts

@@ -0,0 +1,17 @@
+export interface SlackConfig {
+  slack_bot_token: string;
+  workspace_id: string;
+  pull_frequency: number;
+}
+
+export interface IndexAttempt {
+  connector_specific_config: { [key: string]: any };
+  status: "success" | "failure" | "in_progress" | "not_started";
+  time_created: string;
+  time_updated: string;
+  docs_indexed: number;
+}
+
+export interface ListIndexingResponse {
+  index_attempts: IndexAttempt[];
+}

web/src/app/admin/connectors/slack/SlackForm.tsx

@@ -3,7 +3,7 @@ import { Formik, Form, FormikHelpers } from "formik";
 import * as Yup from "yup";
 import { Popup } from "../../../../components/admin/connectors/Popup";
 import { TextFormField } from "../../../../components/admin/connectors/Field";
-import { SlackConfig } from "./interfaces";
+import { SlackConfig } from "../interfaces";
 
 const validationSchema = Yup.object().shape({
   slack_bot_token: Yup.string().required("Please enter your Slack Bot Token"),

web/src/app/admin/connectors/slack/interfaces.ts

@@ -1,11 +0,0 @@
-export interface SlackConfig {
-  slack_bot_token: string;
-  workspace_id: string;
-  pull_frequency: number;
-}
-
-// interface SlackIndexAttempt {}
-
-// interface ListSlackIndexingResponse {
-//   index_attempts: SlackIndexAttempt[];
-// }

web/src/app/admin/connectors/slack/page.tsx

@@ -8,7 +8,7 @@ import { SlackForm } from "@/app/admin/connectors/slack/SlackForm";
 import { SlackIcon } from "@/components/icons/icons";
 import { fetcher } from "@/lib/fetcher";
 import useSWR, { useSWRConfig } from "swr";
-import { SlackConfig } from "./interfaces";
+import { SlackConfig } from "../interfaces";
 import { ThinkingAnimation } from "@/components/Thinking";
 
 const MainSection = () => {
@@ -47,6 +47,7 @@ const MainSection = () => {
               ? ConnectorStatus.Running
               : ConnectorStatus.NotSetup
           }
+          source="slack"
         />
       }
 

web/src/app/admin/connectors/web/WebIndexForm.tsx

@@ -1,94 +0,0 @@
-import React, { useState } from "react";
-import { Formik, Form, FormikHelpers } from "formik";
-import * as Yup from "yup";
-import { Popup } from "../../../../components/admin/connectors/Popup";
-import { TextFormField } from "../../../../components/admin/connectors/Field";
-
-interface FormData {
-  url: string;
-}
-
-const validationSchema = Yup.object().shape({
-  url: Yup.string().required(
-    "Please enter the website URL to scrape e.g. https://docs.github.com/en/actions"
-  ),
-});
-
-const handleSubmit = async (
-  values: FormData,
-  { setSubmitting }: FormikHelpers<FormData>,
-  setPopup: (
-    popup: { message: string; type: "success" | "error" } | null
-  ) => void
-): Promise<boolean> => {
-  setSubmitting(true);
-  let isSuccess = false;
-  try {
-    const response = await fetch("/api/admin/connectors/web/index-attempt", {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify(values),
-    });
-
-    if (response.ok) {
-      isSuccess = true;
-      setPopup({ message: "Success!", type: "success" });
-    } else {
-      const errorData = await response.json();
-      setPopup({ message: `Error: ${errorData.detail}`, type: "error" });
-    }
-  } catch (error) {
-    setPopup({ message: `Error: ${error}`, type: "error" });
-  } finally {
-    setSubmitting(false);
-    setTimeout(() => {
-      setPopup(null);
-    }, 3000);
-    return isSuccess;
-  }
-};
-
-interface SlackFormProps {
-  onSubmit: (isSuccess: boolean) => void;
-}
-
-export const WebIndexForm: React.FC<SlackFormProps> = ({ onSubmit }) => {
-  const [popup, setPopup] = useState<{
-    message: string;
-    type: "success" | "error";
-  } | null>(null);
-
-  return (
-    <>
-      {popup && <Popup message={popup.message} type={popup.type} />}
-      <Formik
-        initialValues={{ url: "" }}
-        validationSchema={validationSchema}
-        onSubmit={(values, formikHelpers) =>
-          handleSubmit(values, formikHelpers, setPopup).then((isSuccess) =>
-            onSubmit(isSuccess)
-          )
-        }
-      >
-        {({ isSubmitting }) => (
-          <Form>
-            <TextFormField name="url" label="URL to Index:" />
-            <button
-              type="submit"
-              disabled={isSubmitting}
-              className={
-                "bg-slate-500 hover:bg-slate-700 text-white " +
-                "font-bold py-2 px-4 rounded focus:outline-none " +
-                "focus:shadow-outline w-full max-w-xs"
-              }
-            >
-              Index
-            </button>
-          </Form>
-        )}
-      </Formik>
-    </>
-  );
-};

web/src/app/admin/connectors/web/page.tsx

@@ -1,25 +1,16 @@
 "use client";
 
 import useSWR, { useSWRConfig } from "swr";
+import * as Yup from "yup";
 
 import { BasicTable } from "@/components/admin/connectors/BasicTable";
-import { WebIndexForm } from "@/app/admin/connectors/web/WebIndexForm";
 import { ThinkingAnimation } from "@/components/Thinking";
 import { timeAgo } from "@/lib/time";
 import { GlobeIcon } from "@/components/icons/icons";
 import { fetcher } from "@/lib/fetcher";
-
-interface WebsiteIndexAttempt {
-  url: string;
-  status: "success" | "failure" | "in_progress" | "not_started";
-  time_created: string;
-  time_updated: string;
-  docs_indexed: number;
-}
-
-interface ListWebIndexingResponse {
-  index_attempts: WebsiteIndexAttempt[];
-}
+import { IndexAttempt, ListIndexingResponse } from "../interfaces";
+import { IndexForm } from "@/components/admin/connectors/Form";
+import { TextFormField } from "@/components/admin/connectors/Field";
 
 const COLUMNS = [
   { header: "Base URL", key: "url" },
@@ -30,28 +21,29 @@ const COLUMNS = [
 
 export default function Web() {
   const { mutate } = useSWRConfig();
-  const { data, isLoading, error } = useSWR<ListWebIndexingResponse>(
+  const { data, isLoading, error } = useSWR<ListIndexingResponse>(
     "/api/admin/connectors/web/index-attempt",
     fetcher
   );
 
-  const urlToLatestIndexAttempt = new Map<string, WebsiteIndexAttempt>();
+  const urlToLatestIndexAttempt = new Map<string, IndexAttempt>();
   const urlToLatestIndexSuccess = new Map<string, string>();
   data?.index_attempts?.forEach((indexAttempt) => {
-    const latestIndexAttempt = urlToLatestIndexAttempt.get(indexAttempt.url);
+    const url = indexAttempt.connector_specific_config.base_url;
+    const latestIndexAttempt = urlToLatestIndexAttempt.get(url);
     if (
       !latestIndexAttempt ||
       indexAttempt.time_created > latestIndexAttempt.time_created
     ) {
-      urlToLatestIndexAttempt.set(indexAttempt.url, indexAttempt);
+      urlToLatestIndexAttempt.set(url, indexAttempt);
     }
 
-    const latestIndexSuccess = urlToLatestIndexSuccess.get(indexAttempt.url);
+    const latestIndexSuccess = urlToLatestIndexSuccess.get(url);
     if (
       indexAttempt.status === "success" &&
       (!latestIndexSuccess || indexAttempt.time_updated > latestIndexSuccess)
     ) {
-      urlToLatestIndexSuccess.set(indexAttempt.url, indexAttempt.time_updated);
+      urlToLatestIndexSuccess.set(url, indexAttempt.time_updated);
     }
   });
 
@@ -65,7 +57,15 @@ export default function Web() {
         Request Indexing
       </h2>
       <div className="border-solid border-gray-600 border rounded-md p-6">
-        <WebIndexForm
+        <IndexForm
+          source="web"
+          formBody={<TextFormField name="base_url" label="URL to Index:" />}
+          validationSchema={Yup.object().shape({
+            base_url: Yup.string().required(
+              "Please enter the website URL to scrape e.g. https://docs.github.com/en/actions"
+            ),
+          })}
+          initialValues={{ base_url: "" }}
           onSubmit={(success) => {
             if (success) {
               mutate("/api/admin/connectors/web/index-attempt");
@@ -87,22 +87,21 @@ export default function Web() {
           data={
             urlToLatestIndexAttempt.size > 0
               ? Array.from(urlToLatestIndexAttempt.values()).map(
-                  (indexAttempt) => ({
-                    ...indexAttempt,
-                    indexed_at:
-                      timeAgo(urlToLatestIndexSuccess.get(indexAttempt.url)) ||
-                      "-",
-                    docs_indexed: indexAttempt.docs_indexed || "-",
-                    url: (
-                      <a
-                        className="text-blue-500"
-                        target="_blank"
-                        href={indexAttempt.url}
-                      >
-                        {indexAttempt.url}
-                      </a>
-                    ),
-                  })
+                  (indexAttempt) => {
+                    const url = indexAttempt.connector_specific_config
+                      .base_url as string;
+                    return {
+                      indexed_at:
+                        timeAgo(urlToLatestIndexSuccess.get(url)) || "-",
+                      docs_indexed: indexAttempt.docs_indexed || "-",
+                      url: (
+                        <a className="text-blue-500" target="_blank" href={url}>
+                          {url}
+                        </a>
+                      ),
+                      status: indexAttempt.status,
+                    };
+                  }
                 )
               : []
           }

web/src/app/admin/layout.tsx

@@ -1,6 +1,6 @@
 import { Header } from "@/components/Header";
 import { Sidebar } from "@/components/admin/connectors/Sidebar";
-import { GlobeIcon, SlackIcon } from "@/components/icons/icons";
+import { GithubIcon, GlobeIcon, SlackIcon } from "@/components/icons/icons";
 import { getCurrentUserSS } from "@/lib/userSS";
 import { redirect } from "next/navigation";
 
@@ -45,6 +45,15 @@ export default async function AdminLayout({
                   ),
                   link: "/admin/connectors/web",
                 },
+                {
+                  name: (
+                    <div className="flex">
+                      <GithubIcon size="16" />
+                      <div className="ml-1">Github</div>
+                    </div>
+                  ),
+                  link: "/admin/connectors/github",
+                },
               ],
             },
           ]}

web/src/components/admin/connectors/Form.tsx

@@ -0,0 +1,106 @@
+import React, { useState } from "react";
+import { Formik, Form, FormikHelpers } from "formik";
+import * as Yup from "yup";
+import { Popup } from "./Popup";
+
+type ValidSources = "web" | "github";
+
+const handleSubmit = async (
+  source: ValidSources,
+  values: Yup.AnyObject,
+  { setSubmitting }: FormikHelpers<Yup.AnyObject>,
+  setPopup: (
+    popup: { message: string; type: "success" | "error" } | null
+  ) => void
+): Promise<boolean> => {
+  setSubmitting(true);
+  let isSuccess = false;
+  try {
+    const response = await fetch(
+      `/api/admin/connectors/${source}/index-attempt`,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({ connector_specific_config: values }),
+      }
+    );
+
+    if (response.ok) {
+      isSuccess = true;
+      setPopup({ message: "Success!", type: "success" });
+    } else {
+      const errorData = await response.json();
+      setPopup({ message: `Error: ${errorData.detail}`, type: "error" });
+    }
+  } catch (error) {
+    setPopup({ message: `Error: ${error}`, type: "error" });
+  } finally {
+    setSubmitting(false);
+    setTimeout(() => {
+      setPopup(null);
+    }, 3000);
+    return isSuccess;
+  }
+};
+
+interface IndexFormProps<YupObjectType extends Yup.AnyObject> {
+  source: ValidSources;
+  formBody: JSX.Element | null;
+  validationSchema: Yup.ObjectSchema<YupObjectType>;
+  initialValues: YupObjectType;
+  onSubmit: (isSuccess: boolean) => void;
+  additionalNonFormValues?: Yup.AnyObject;
+}
+
+export function IndexForm<YupObjectType extends Yup.AnyObject>({
+  source,
+  formBody,
+  validationSchema,
+  initialValues,
+  onSubmit,
+  additionalNonFormValues = {},
+}: IndexFormProps<YupObjectType>): JSX.Element {
+  const [popup, setPopup] = useState<{
+    message: string;
+    type: "success" | "error";
+  } | null>(null);
+
+  return (
+    <>
+      {popup && <Popup message={popup.message} type={popup.type} />}
+      <Formik
+        initialValues={initialValues}
+        validationSchema={validationSchema}
+        onSubmit={(values, formikHelpers) => {
+          handleSubmit(
+            source,
+            { ...values, ...additionalNonFormValues },
+            formikHelpers as FormikHelpers<Yup.AnyObject>,
+            setPopup
+          ).then((isSuccess) => onSubmit(isSuccess));
+        }}
+      >
+        {({ isSubmitting }) => (
+          <Form>
+            {formBody}
+            <div className="flex">
+              <button
+                type="submit"
+                disabled={isSubmitting}
+                className={
+                  "bg-slate-500 hover:bg-slate-700 text-white " +
+                  "font-bold py-2 px-4 rounded focus:outline-none " +
+                  "focus:shadow-outline w-full max-w-sm mx-auto"
+                }
+              >
+                Index
+              </button>
+            </div>
+          </Form>
+        )}
+      </Formik>
+    </>
+  );
+}

web/src/components/admin/connectors/RecurringConnectorStatus.tsx

@@ -1,4 +1,10 @@
+"use client";
+
+import { ListIndexingResponse } from "@/app/admin/connectors/interfaces";
+import { fetcher } from "@/lib/fetcher";
+import { timeAgo } from "@/lib/time";
 import { CheckCircle, MinusCircle } from "@phosphor-icons/react";
+import useSWR from "swr";
 
 export enum ConnectorStatus {
   Running = "Running",
@@ -7,16 +13,39 @@ export enum ConnectorStatus {
 
 interface ReccuringConnectorStatusProps {
   status: ConnectorStatus;
+  source: string;
 }
 
 export const ReccuringConnectorStatus = ({
   status,
+  source,
 }: ReccuringConnectorStatusProps) => {
+  const { data } = useSWR<ListIndexingResponse>(
+    `/api/admin/connectors/${source}/index-attempt`,
+    fetcher
+  );
+
+  const lastSuccessfulAttempt = data?.index_attempts
+    .filter((attempt) => attempt.status === "success")
+    .sort((a, b) => {
+      if (a.time_updated === b.time_updated) {
+        return 0;
+      }
+      return a.time_updated > b.time_updated ? -1 : 1;
+    })[0];
+
   if (status === ConnectorStatus.Running) {
     return (
-      <div className="text-emerald-600 flex align-middle text-center">
-        <CheckCircle size={20} className="my-auto" />
-        <p className="my-auto ml-1">{status}</p>
+      <div>
+        <div className="text-emerald-600 flex align-middle text-center">
+          <CheckCircle size={20} className="my-auto" />
+          <p className="my-auto ml-1">{status}</p>
+        </div>
+        {lastSuccessfulAttempt && (
+          <p className="text-xs my-auto ml-1">
+            Last updated {timeAgo(lastSuccessfulAttempt.time_updated)}
+          </p>
+        )}
       </div>
     );
   }

web/src/components/icons/icons.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { Globe, SlackLogo } from "@phosphor-icons/react";
+import { Globe, SlackLogo, GithubLogo } from "@phosphor-icons/react";
 
 interface IconProps {
   size?: string;
@@ -22,3 +22,10 @@ export const SlackIcon = ({
 }: IconProps) => {
   return <SlackLogo size={size} className={className} />;
 };
+
+export const GithubIcon = ({
+  size = "16",
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return <GithubLogo size={size} className={className} />;
+};

web/src/components/search/SearchResultsDisplay.tsx

@@ -3,6 +3,7 @@ import { Globe, SlackLogo, GoogleDriveLogo } from "@phosphor-icons/react";
 import "tailwindcss/tailwind.css";
 import { Quote, Document } from "./types";
 import { ThinkingAnimation } from "../Thinking";
+import { GithubIcon } from "../icons/icons";
 
 interface SearchResultsDisplayProps {
   answer: string | null;
@@ -22,6 +23,8 @@ const getSourceIcon = (sourceType: string) => {
       return <SlackLogo size={ICON_SIZE} className={ICON_STYLE} />;
     case "google_drive":
       return <GoogleDriveLogo size={ICON_SIZE} className={ICON_STYLE} />;
+    case "github":
+      return <GithubIcon size={ICON_SIZE} className={ICON_STYLE} />;
     default:
       return null;
   }
@@ -102,7 +105,7 @@ export const SearchResultsDisplay: React.FC<SearchResultsDisplayProps> = ({
               >
                 {getSourceIcon(doc.source_type)}
                 <p className="truncate break-all">
-                  {doc.semantic_name || doc.document_id}
+                  {doc.semantic_identifier || doc.document_id}
                 </p>
               </a>
               <p className="pl-1 py-3 text-gray-200">{doc.blurb}</p>

web/src/components/search/types.tsx

@@ -11,7 +11,7 @@ export interface Document {
   link: string;
   source_type: string;
   blurb: string;
-  semantic_name: string | null;
+  semantic_identifier: string | null;
 }
 
 export interface SearchResponse {