Add ingestion as a "Source" for the FE + improve typing (#2312)

web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx

@@ -22,7 +22,7 @@ import AdvancedFormPage from "./pages/Advanced";
 import DynamicConnectionForm from "./pages/DynamicConnectorCreationForm";
 import CreateCredential from "@/components/credentials/actions/CreateCredential";
 import ModifyCredential from "@/components/credentials/actions/ModifyCredential";
-import { ValidSources } from "@/lib/types";
+import { ConfigurableSources, ValidSources } from "@/lib/types";
 import { Credential, credentialTemplates } from "@/lib/connectors/credentials";
 import {
   ConnectionConfiguration,
@@ -54,7 +54,7 @@ export type AdvancedConfigFinal = {
 export default function AddConnector({
   connector,
 }: {
-  connector: ValidSources;
+  connector: ConfigurableSources;
 }) {
   const [currentCredential, setCurrentCredential] =
     useState<Credential<any> | null>(null);
@@ -195,7 +195,7 @@ export default function AddConnector({
     };
 
     // google sites-specific handling
-    if (connector == "google_site") {
+    if (connector == "google_sites") {
       const response = await submitGoogleSite(
         selectedFiles,
         formValues?.base_url,
@@ -443,26 +443,29 @@ export default function AddConnector({
                 </button>
               )}
 
-              {!(connector == "google_drive") && createConnectorToggle && (
-                <Modal
-                  className="max-w-3xl rounded-lg"
-                  onOutsideClick={() => setCreateConnectorToggle(false)}
-                >
-                  <>
-                    <Title className="mb-2 text-lg">
-                      Create a {getSourceDisplayName(connector)} credential
-                    </Title>
-                    <CreateCredential
-                      close
-                      refresh={refresh}
-                      sourceType={connector}
-                      setPopup={setPopup}
-                      onSwitch={onSwap}
-                      onClose={() => setCreateConnectorToggle(false)}
-                    />
-                  </>
-                </Modal>
-              )}
+              {/* NOTE: connector will never be google_drive, since the ternary above will 
+              prevent that, but still keeping this here for safety in case the above changes. */}
+              {(connector as ValidSources) !== "google_drive" &&
+                createConnectorToggle && (
+                  <Modal
+                    className="max-w-3xl rounded-lg"
+                    onOutsideClick={() => setCreateConnectorToggle(false)}
+                  >
+                    <>
+                      <Title className="mb-2 text-lg">
+                        Create a {getSourceDisplayName(connector)} credential
+                      </Title>
+                      <CreateCredential
+                        close
+                        refresh={refresh}
+                        sourceType={connector}
+                        setPopup={setPopup}
+                        onSwitch={onSwap}
+                        onClose={() => setCreateConnectorToggle(false)}
+                      />
+                    </>
+                  </Modal>
+                )}
             </Card>
             <div className="mt-4 flex w-full justify-end">
               <button

web/src/app/admin/connectors/[connector]/ConnectorWrapper.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { ValidSources } from "@/lib/types";
+import { ConfigurableSources, ValidSources } from "@/lib/types";
 import AddConnector from "./AddConnectorPage";
 import { FormProvider } from "@/components/context/FormContext";
 import Sidebar from "./Sidebar";
@@ -8,7 +8,11 @@ import { HeaderTitle } from "@/components/header/HeaderTitle";
 import { Button } from "@tremor/react";
 import { isValidSource } from "@/lib/sources";
 
-export default function ConnectorWrapper({ connector }: { connector: string }) {
+export default function ConnectorWrapper({
+  connector,
+}: {
+  connector: ConfigurableSources;
+}) {
   return (
     <FormProvider connector={connector}>
       <div className="flex justify-center w-full h-full">
@@ -28,7 +32,7 @@ export default function ConnectorWrapper({ connector }: { connector: string }) {
               </Button>
             </div>
           ) : (
-            <AddConnector connector={connector as ValidSources} />
+            <AddConnector connector={connector} />
           )}
         </div>
       </div>

web/src/app/admin/connectors/[connector]/page.tsx

@@ -1,3 +1,4 @@
+import { ConfigurableSources } from "@/lib/types";
 import ConnectorWrapper from "./ConnectorWrapper";
 
 export default async function Page({
@@ -5,5 +6,9 @@ export default async function Page({
 }: {
   params: { connector: string };
 }) {
-  return <ConnectorWrapper connector={params.connector.replace("-", "_")} />;
+  return (
+    <ConnectorWrapper
+      connector={params.connector.replace("-", "_") as ConfigurableSources}
+    />
+  );
 }

web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx

@@ -465,7 +465,10 @@ export function CCPairIndexingStatusTable({
             </Button>
           </div>
           {sortedSources
-            .filter((source) => source != "not_applicable")
+            .filter(
+              (source) =>
+                source != "not_applicable" && source != "ingestion_api"
+            )
             .map((source, ind) => {
               const sourceMatches = source
                 .toLowerCase()

web/src/app/chat/ChatIntro.tsx

@@ -1,30 +1,9 @@
-import { getSourceMetadataForSources, listSourceMetadata } from "@/lib/sources";
+import { getSourceMetadataForSources } from "@/lib/sources";
 import { ValidSources } from "@/lib/types";
-import Image from "next/image";
 import { Persona } from "../admin/assistants/interfaces";
 import { Divider } from "@tremor/react";
-import { FiBookmark, FiCpu, FiInfo, FiX, FiZoomIn } from "react-icons/fi";
+import { FiBookmark, FiInfo } from "react-icons/fi";
 import { HoverPopup } from "@/components/HoverPopup";
-import { Modal } from "@/components/Modal";
-import { useState } from "react";
-import { Logo } from "@/components/Logo";
-
-const MAX_PERSONAS_TO_DISPLAY = 4;
-
-function HelperItemDisplay({
-  title,
-  description,
-}: {
-  title: string;
-  description: string;
-}) {
-  return (
-    <div className="cursor-pointer hover:bg-hover-light border border-border rounded py-2 px-4">
-      <div className="text-emphasis font-bold text-lg flex">{title}</div>
-      <div className="text-sm">{description}</div>
-    </div>
-  );
-}
 
 export function ChatIntro({
   availableSources,

web/src/lib/connectors/connectors.ts

@@ -1,4 +1,4 @@
-import { ValidInputTypes, ValidSources } from "../types";
+import { ConfigurableSources, ValidInputTypes, ValidSources } from "../types";
 
 export type InputType =
   | "list"
@@ -76,7 +76,10 @@ export interface ConnectionConfiguration {
   overrideDefaultFreq?: number;
 }
 
-export const connectorConfigs: Record<ValidSources, ConnectionConfiguration> = {
+export const connectorConfigs: Record<
+  ConfigurableSources,
+  ConnectionConfiguration
+> = {
   web: {
     description: "Configure Web connector",
     values: [

web/src/lib/connectors/credentials.ts

@@ -288,6 +288,7 @@ export const credentialTemplates: Record<ValidSources, any> = {
   mediawiki: null,
   web: null,
   not_applicable: null,
+  ingestion_api: null,
 
   // NOTE: These are Special Cases
   google_drive: { google_drive_tokens: "" } as GoogleDriveCredentialJson,

web/src/lib/sources.ts

@@ -272,6 +272,11 @@ const SOURCE_METADATA_MAP: SourceMap = {
     category: SourceCategory.Storage,
     docs: "https://docs.danswer.dev/connectors/google_storage",
   },
+  ingestion_api: {
+    icon: GlobeIcon,
+    displayName: "Ingestion",
+    category: SourceCategory.Other,
+  },
   // currently used for the Internet Search tool docs, which is why
   // a globe is used
   not_applicable: {
@@ -302,8 +307,12 @@ export function getSourceMetadata(sourceType: ValidSources): SourceMetadata {
 }
 
 export function listSourceMetadata(): SourceMetadata[] {
+  /* This gives back all the viewable / common sources, primarily for 
+  display in the Add Connector page */
   const entries = Object.entries(SOURCE_METADATA_MAP)
-    .filter(([source, _]) => source !== "not_applicable")
+    .filter(
+      ([source, _]) => source !== "not_applicable" && source != "ingestion_api"
+    )
     .map(([source, metadata]) => {
       return fillSourceMetadata(metadata, source as ValidSources);
     });

web/src/lib/types.ts

@@ -237,6 +237,12 @@ const validSources = [
   "google_cloud_storage",
   "oci_storage",
   "not_applicable",
-];
+  "ingestion_api",
+] as const;
 
 export type ValidSources = (typeof validSources)[number];
+// The valid sources that are actually valid to select in the UI
+export type ConfigurableSources = Exclude<
+  ValidSources,
+  "not_applicable" | "ingestion_api"
+>;