Discourse Connector (#1420)

web/src/app/admin/connectors/discourse/page.tsx

@@ -0,0 +1,274 @@
+"use client";
+
+import * as Yup from "yup";
+import { DiscourseIcon, TrashIcon } from "@/components/icons/icons";
+import {
+  TextFormField,
+  TextArrayFieldBuilder,
+} from "@/components/admin/connectors/Field";
+import { HealthCheckBanner } from "@/components/health/healthcheck";
+import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
+import {
+  Credential,
+  ConnectorIndexingStatus,
+  DiscourseConfig,
+  DiscourseCredentialJson,
+} from "@/lib/types";
+import useSWR, { useSWRConfig } from "swr";
+import { fetcher } from "@/lib/fetcher";
+import { LoadingAnimation } from "@/components/Loading";
+import { adminDeleteCredential, linkCredential } from "@/lib/credential";
+import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
+import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
+import { usePopup } from "@/components/admin/connectors/Popup";
+import { usePublicCredentials } from "@/lib/hooks";
+import { Card, Divider, Text, Title } from "@tremor/react";
+import { AdminPageTitle } from "@/components/admin/Title";
+
+const Main = () => {
+  const { popup, setPopup } = usePopup();
+
+  const { mutate } = useSWRConfig();
+  const {
+    data: connectorIndexingStatuses,
+    isLoading: isConnectorIndexingStatusesLoading,
+    error: isConnectorIndexingStatusesError,
+  } = useSWR<ConnectorIndexingStatus<any, any>[]>(
+    "/api/manage/admin/connector/indexing-status",
+    fetcher
+  );
+
+  const {
+    data: credentialsData,
+    isLoading: isCredentialsLoading,
+    error: isCredentialsError,
+    refreshCredentials,
+  } = usePublicCredentials();
+
+  if (
+    (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) ||
+    (!credentialsData && isCredentialsLoading)
+  ) {
+    return <LoadingAnimation text="Loading" />;
+  }
+
+  if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) {
+    return <div>Failed to load connectors</div>;
+  }
+
+  if (isCredentialsError || !credentialsData) {
+    return <div>Failed to load credentials</div>;
+  }
+
+  const discourseConnectorIndexingStatuses: ConnectorIndexingStatus<
+    DiscourseConfig,
+    DiscourseCredentialJson
+  >[] = connectorIndexingStatuses.filter(
+    (connectorIndexingStatus) =>
+      connectorIndexingStatus.connector.source === "discourse"
+  );
+  const discourseCredential: Credential<DiscourseCredentialJson> | undefined =
+    credentialsData.find(
+      (credential) => credential.credential_json?.discourse_api_username
+    );
+
+  return (
+    <>
+      {popup}
+      <Text>
+        This connector allows you to sync all your Discourse Topics into
+        Danswer. More details on how to setup the Discourse connector can be
+        found in{" "}
+        <a
+          className="text-link"
+          href="https://docs.danswer.dev/connectors/discourse"
+          target="_blank"
+        >
+          this guide.
+        </a>
+      </Text>
+
+      <Title className="mb-2 mt-6 ml-auto mr-auto">
+        Step 1: Provide your API Access info
+      </Title>
+
+      {discourseCredential ? (
+        <>
+          <div className="flex mb-1 text-sm">
+            <p className="my-auto">Existing API Key: </p>
+            <p className="ml-1 italic my-auto max-w-md truncate">
+              {discourseCredential.credential_json?.discourse_api_key}
+            </p>
+            <button
+              className="ml-1 hover:bg-hover rounded p-1"
+              onClick={async () => {
+                if (discourseConnectorIndexingStatuses.length > 0) {
+                  setPopup({
+                    type: "error",
+                    message:
+                      "Must delete all connectors before deleting credentials",
+                  });
+                  return;
+                }
+                await adminDeleteCredential(discourseCredential.id);
+                refreshCredentials();
+              }}
+            >
+              <TrashIcon />
+            </button>
+          </div>
+        </>
+      ) : (
+        <>
+          <Card className="mt-4">
+            <CredentialForm<DiscourseCredentialJson>
+              formBody={
+                <>
+                  <TextFormField
+                    name="discourse_api_username"
+                    label="API Key Username:"
+                  />
+                  <TextFormField
+                    name="discourse_api_key"
+                    label="API Key:"
+                    type="password"
+                  />
+                </>
+              }
+              validationSchema={Yup.object().shape({
+                discourse_api_username: Yup.string().required(
+                  "Please enter the Username associated with the API key"
+                ),
+                discourse_api_key: Yup.string().required(
+                  "Please enter the API key"
+                ),
+              })}
+              initialValues={{
+                discourse_api_username: "",
+                discourse_api_key: "",
+              }}
+              onSubmit={(isSuccess) => {
+                if (isSuccess) {
+                  refreshCredentials();
+                }
+              }}
+            />
+          </Card>
+        </>
+      )}
+
+      <Title className="mb-2 mt-6 ml-auto mr-auto">
+        Step 2: Which Categories do you want to make searchable?
+      </Title>
+
+      {discourseConnectorIndexingStatuses.length > 0 && (
+        <>
+          <Text className="mb-2">
+            We pull Topics with new Posts every <b>10</b> minutes.
+          </Text>
+          <div className="mb-2">
+            <ConnectorsTable<DiscourseConfig, DiscourseCredentialJson>
+              connectorIndexingStatuses={discourseConnectorIndexingStatuses}
+              liveCredential={discourseCredential}
+              getCredential={(credential) =>
+                credential.credential_json.discourse_api_username
+              }
+              specialColumns={[
+                {
+                  header: "Categories",
+                  key: "categories",
+                  getValue: (ccPairStatus) =>
+                    ccPairStatus.connector.connector_specific_config
+                      .categories &&
+                    ccPairStatus.connector.connector_specific_config.categories
+                      .length > 0
+                      ? ccPairStatus.connector.connector_specific_config.categories.join(
+                          ", "
+                        )
+                      : "",
+                },
+              ]}
+              includeName={true}
+              onUpdate={() =>
+                mutate("/api/manage/admin/connector/indexing-status")
+              }
+              onCredentialLink={async (connectorId) => {
+                if (discourseCredential) {
+                  await linkCredential(connectorId, discourseCredential.id);
+                  mutate("/api/manage/admin/connector/indexing-status");
+                }
+              }}
+            />
+          </div>
+          <Divider />
+        </>
+      )}
+
+      {discourseCredential ? (
+        <>
+          <Card className="mt-4">
+            <h2 className="font-bold mb-3">Create a new Discourse Connector</h2>
+            <ConnectorForm<DiscourseConfig>
+              nameBuilder={(values) =>
+                values.categories
+                  ? `${values.base_url}-${values.categories.join("_")}`
+                  : `${values.base_url}-All`
+              }
+              source="discourse"
+              inputType="poll"
+              formBody={
+                <>
+                  <TextFormField
+                    name="base_url"
+                    label="Base URL:"
+                    subtext="This might be something like https://danswer.discourse.group/ or https://community.yourcompany.com/"
+                  />
+                </>
+              }
+              formBodyBuilder={TextArrayFieldBuilder({
+                name: "categories",
+                label: "Categories:",
+                subtext:
+                  "Specify 0 or more Categories to index. If no Categories are specified, Topics from " +
+                  "all categories will be indexed.",
+              })}
+              validationSchema={Yup.object().shape({
+                base_url: Yup.string().required(
+                  "Please the base URL of your Discourse site."
+                ),
+                categories: Yup.array().of(
+                  Yup.string().required("Category names must be strings")
+                ),
+              })}
+              initialValues={{
+                categories: [],
+                base_url: "",
+              }}
+              refreshFreq={10 * 60} // 10 minutes
+              credentialId={discourseCredential.id}
+            />
+          </Card>
+        </>
+      ) : (
+        <Text>
+          Please provide your API Key Info in Step 1 first! Once done with that,
+          you can then start indexing all your Discourse Topics.
+        </Text>
+      )}
+    </>
+  );
+};
+
+export default function Page() {
+  return (
+    <div className="mx-auto container">
+      <div className="mb-4">
+        <HealthCheckBanner />
+      </div>
+
+      <AdminPageTitle icon={<DiscourseIcon size={32} />} title="Discourse" />
+
+      <Main />
+    </div>
+  );
+}

web/src/components/icons/icons.tsx

@@ -52,6 +52,7 @@ import document360Icon from "../../../public/Document360.png";
 import googleSitesIcon from "../../../public/GoogleSites.png";
 import zendeskIcon from "../../../public/Zendesk.svg";
 import sharepointIcon from "../../../public/Sharepoint.png";
+import discourseIcon from "../../../public/Discourse.png";
 import { FaRobot } from "react-icons/fa";
 
 interface IconProps {
@@ -601,6 +602,18 @@ export const ZendeskIcon = ({
   </div>
 );
 
+export const DiscourseIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => (
+  <div
+    style={{ width: `${size}px`, height: `${size}px` }}
+    className={`w-[${size}px] h-[${size}px] ` + className}
+  >
+    <Image src={discourseIcon} alt="Logo" width="96" height="96" />
+  </div>
+);
+
 export const AxeroIcon = ({
   size = 16,
   className = defaultTailwindCSS,

web/src/lib/sources.ts

@@ -2,6 +2,7 @@ import {
   AxeroIcon,
   BookstackIcon,
   ConfluenceIcon,
+  DiscourseIcon,
   Document360Icon,
   FileIcon,
   GithubIcon,
@@ -155,6 +156,11 @@ const SOURCE_METADATA_MAP: SourceMap = {
     displayName: "Sharepoint",
     category: SourceCategory.AppConnection,
   },
+  discourse: {
+    icon: DiscourseIcon,
+    displayName: "Discourse",
+    category: SourceCategory.AppConnection,
+  },
   axero: {
     icon: AxeroIcon,
     displayName: "Axero",

web/src/lib/types.ts

@@ -39,6 +39,7 @@ export type ValidSources =
   | "loopio"
   | "sharepoint"
   | "zendesk"
+  | "discourse"
   | "axero";
 
 export type ValidInputTypes = "load_state" | "poll" | "event";
@@ -118,6 +119,11 @@ export interface SharepointConfig {
   sites?: string[];
 }
 
+export interface DiscourseConfig {
+  base_url: string;
+  categories?: string[];
+}
+
 export interface AxeroConfig {
   spaces?: string[];
 }
@@ -337,6 +343,11 @@ export interface SharepointCredentialJson {
   aad_directory_id: string;
 }
 
+export interface DiscourseCredentialJson {
+  discourse_api_key: string;
+  discourse_api_username: string;
+}
+
 export interface AxeroCredentialJson {
   base_url: string;
   axero_api_token: string;