support for zendesk help center (#661)

.gitignore

@@ -1,2 +1,3 @@
 .env
 .DS_store
+.venv

backend/danswer/configs/constants.py

@@ -70,6 +70,7 @@ class DocumentSource(str, Enum):
     DOCUMENT360 = "document360"
     GONG = "gong"
     GOOGLE_SITES = "google_sites"
+    ZENDESK = "zendesk"
 
 
 class DocumentIndexType(str, Enum):

backend/danswer/connectors/factory.py

@@ -26,6 +26,7 @@ from danswer.connectors.slack.connector import SlackLoadConnector
 from danswer.connectors.slack.connector import SlackPollConnector
 from danswer.connectors.web.connector import WebConnector
 from danswer.connectors.zulip.connector import ZulipConnector
+from danswer.connectors.zendesk.connector import ZendeskConnector
 
 
 class ConnectorMissingException(Exception):
@@ -58,6 +59,7 @@ def identify_connector_class(
         DocumentSource.DOCUMENT360: Document360Connector,
         DocumentSource.GONG: GongConnector,
         DocumentSource.GOOGLE_SITES: GoogleSitesConnector,
+        DocumentSource.ZENDESK: ZendeskConnector,
     }
     connector_by_source = connector_map.get(source, {})
 

backend/danswer/connectors/zendesk/connector.py

@@ -0,0 +1,63 @@
+from typing import Any
+from zenpy import Zenpy
+from zenpy.lib.api_objects.help_centre_objects import Article
+
+from danswer.configs.app_configs import INDEX_BATCH_SIZE
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.models import Document, Section
+from danswer.connectors.interfaces import GenerateDocumentsOutput, LoadConnector, PollConnector, SecondsSinceUnixEpoch
+
+class ZendeskClientNotSetUpError(PermissionError):
+    def __init__(self) -> None:
+        super().__init__(
+            "Zendesk Client is not set up, was load_credentials called?"
+        )
+
+
+class ZendeskConnector(LoadConnector, PollConnector):
+    def __init__(
+            self,
+            batch_size: int = INDEX_BATCH_SIZE
+    ) -> None:
+        self.batch_size = batch_size
+        self.zendesk_client: Zenpy | None = None
+    
+    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
+        self.zendesk_client = Zenpy(
+            subdomain=credentials["zendesk_subdomain"],
+            email=credentials["zendesk_email"],
+            token=credentials["zendesk_token"],
+        )
+        return None
+    
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        return self.poll_source(None, None)
+    
+    def _article_to_document(self, article: Article) -> Document:
+        return Document(
+            id=f"article:{article.id}",
+            sections=[Section(link=article.html_url, text=article.body)],
+            source=DocumentSource.ZENDESK,
+            semantic_identifier="Article: " + article.title,
+            metadata={
+                "type": "article",
+                "updated_at": article.updated_at,
+            }
+        )
+
+
+    def poll_source(self, start: SecondsSinceUnixEpoch | None, end: SecondsSinceUnixEpoch | None) -> GenerateDocumentsOutput:
+        if self.zendesk_client is None:
+            raise ZendeskClientNotSetUpError()
+        
+        articles = self.zendesk_client.help_center.articles(cursor_pagination=True) if start is None else self.zendesk_client.help_center.articles.incremental(start_time=int(start))
+        doc_batch = []
+        for article in articles:
+            if article.body is None:
+                continue
+
+            doc_batch.append(self._article_to_document(article))
+            if len(doc_batch) >= self.batch_size:
+                yield doc_batch
+                doc_batch.clear()
+

backend/requirements/default.txt

@@ -56,3 +56,4 @@ transformers==4.30.1
 uvicorn==0.21.1
 zulip==0.8.2
 hubspot-api-client==8.1.0
+zenpy==2.0.41

backend/scripts/reset_indexes.py

@@ -1,5 +1,11 @@
 # This file is purely for development use, not included in any builds
 import requests
+import os
+import sys
+
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
 
 from danswer.configs.app_configs import DOCUMENT_INDEX_NAME
 from danswer.document_index.vespa.index import DOCUMENT_ID_ENDPOINT

backend/scripts/reset_postgres.py

@@ -1,5 +1,11 @@
 import psycopg2
 
+import os
+import sys
+
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+
 from danswer.configs.app_configs import POSTGRES_DB
 from danswer.configs.app_configs import POSTGRES_HOST
 from danswer.configs.app_configs import POSTGRES_PASSWORD

web/public/Zendesk.svg

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="0 -30.5 256 256" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid">
+    <g>
+        <path d="M118.249172,51.2326115 L118.249172,194.005605 L0,194.005605 L118.249172,51.2326115 Z M118.249172,2.84217094e-14 C118.249172,32.6440764 91.7686624,59.124586 59.124586,59.124586 C26.4805096,59.124586 0,32.6440764 0,2.84217094e-14 L118.249172,2.84217094e-14 Z M137.750828,194.005605 C137.750828,161.328917 164.198726,134.881019 196.875414,134.881019 C229.552102,134.881019 256,161.361529 256,194.005605 L137.750828,194.005605 Z M137.750828,142.740382 L137.750828,0 L256,0 L137.750828,142.740382 Z" fill="#03363D">
+
</path>
+    </g>
+</svg>

web/src/app/admin/connectors/zendesk/page.tsx

@@ -0,0 +1,242 @@
+"use client";
+
+import * as Yup from "yup";
+import { TrashIcon, ZendeskIcon } from "@/components/icons/icons";
+import { TextFormField } from "@/components/admin/connectors/Field";
+import { HealthCheckBanner } from "@/components/health/healthcheck";
+import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
+import {
+  ZendeskCredentialJson,
+  ZendeskConfig,
+  ConnectorIndexingStatus,
+  Credential,
+} from "@/lib/types";
+import useSWR, { useSWRConfig } from "swr";
+import { fetcher } from "@/lib/fetcher";
+import { LoadingAnimation } from "@/components/Loading";
+import { adminDeleteCredential, linkCredential } from "@/lib/credential";
+import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
+import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
+import { usePopup } from "@/components/admin/connectors/Popup";
+import { usePublicCredentials } from "@/lib/hooks";
+
+const Main = () => {
+  const { popup, setPopup } = usePopup();
+
+  const { mutate } = useSWRConfig();
+  const {
+    data: connectorIndexingStatuses,
+    isLoading: isConnectorIndexingStatusesLoading,
+    error: isConnectorIndexingStatusesError,
+  } = useSWR<ConnectorIndexingStatus<any, any>[]>(
+    "/api/manage/admin/connector/indexing-status",
+    fetcher
+  );
+  const {
+    data: credentialsData,
+    isLoading: isCredentialsLoading,
+    error: isCredentialsError,
+    refreshCredentials,
+  } = usePublicCredentials();
+
+  if (
+    (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) ||
+    (!credentialsData && isCredentialsLoading)
+  ) {
+    return <LoadingAnimation text="Loading" />;
+  }
+
+  if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) {
+    return <div>Failed to load connectors</div>;
+  }
+
+  if (isCredentialsError || !credentialsData) {
+    return <div>Failed to load credentials</div>;
+  }
+
+  const zendeskConnectorIndexingStatuses: ConnectorIndexingStatus<
+    ZendeskConfig,
+    ZendeskCredentialJson
+  >[] = connectorIndexingStatuses.filter(
+    (connectorIndexingStatus) =>
+      connectorIndexingStatus.connector.source === "zendesk"
+  );
+  const zendeskCredential: Credential<ZendeskCredentialJson> | undefined =
+    credentialsData.find(
+      (credential) => credential.credential_json?.zendesk_email
+    );
+
+  return (
+    <>
+      {popup}
+      <h2 className="font-bold mb-2 mt-6 ml-auto mr-auto">
+        Step 1: Provide your API details
+      </h2>
+
+      {zendeskCredential ? (
+        <>
+          <div className="flex mb-1 text-sm">
+            <p className="my-auto">Existing API Token: </p>
+            <p className="ml-1 italic my-auto max-w-md">
+              {zendeskCredential.credential_json?.zendesk_email}
+            </p>
+            <button
+              className="ml-1 hover:bg-gray-700 rounded-full p-1"
+              onClick={async () => {
+                if (zendeskConnectorIndexingStatuses.length > 0) {
+                  setPopup({
+                    type: "error",
+                    message:
+                      "Must delete all connectors before deleting credentials",
+                  });
+                  return;
+                }
+                await adminDeleteCredential(zendeskCredential.id);
+                refreshCredentials();
+              }}
+            >
+              <TrashIcon />
+            </button>
+          </div>
+        </>
+      ) : (
+        <>
+          <p className="text-sm">
+            To get started you&apos;ll need API token details for your Zendesk
+            instance. You can generate this by access the Admin Center of your instance
+            (e.g. https://&lt;subdomain&gt;.zendesk.com/admin/). Proceed to the "Apps and
+            Integrations" section and "Zendesk API" page. Add a new API token and provide
+            it with a name. You will also need to provide the e-mail address of a user that
+            the system will impersonate. This is of little consequence as we are only performing
+            read actions.
+          </p>
+          <div className="border-solid border-gray-600 border rounded-md p-6 mt-2 mb-4">
+            <CredentialForm<ZendeskCredentialJson>
+              formBody={
+                <>
+                  <TextFormField
+                    name="zendesk_subdomain"
+                    label="Zendesk Subdomain (<subdomain>.zendesk.com):"
+                  />
+                  <TextFormField
+                    name="zendesk_email"
+                    label="Zendesk User Email:"
+                  />
+                  <TextFormField
+                    name="zendesk_token"
+                    label="Zendesk API Token:"
+                    type="password"
+                  />
+                </>
+              }
+              validationSchema={Yup.object().shape({
+                zendesk_subdomain: Yup.string().required(
+                  "Please enter the subdomain for your Zendesk instance"
+                ),
+                zendesk_email: Yup.string().required(
+                  "Please enter your user email to user with the token"
+                ),
+                zendesk_token: Yup.string().required(
+                  "Please enter your Zendesk API token"
+                ),
+              })}
+              initialValues={{
+                zendesk_subdomain: "",
+                zendesk_email: "",
+                zendesk_token: "",
+              }}
+              onSubmit={(isSuccess) => {
+                if (isSuccess) {
+                  refreshCredentials();
+                  mutate("/api/manage/admin/connector/indexing-status");
+                }
+              }}
+            />
+          </div>
+        </>
+      )}
+
+      {zendeskConnectorIndexingStatuses.length > 0 && (
+        <>
+          <h2 className="font-bold mb-2 mt-6 ml-auto mr-auto">
+            Zendesk indexing status
+          </h2>
+          <p className="text-sm mb-2">
+            The latest article changes are fetched every 10 minutes.
+          </p>
+          <div className="mb-2">
+            <ConnectorsTable<ZendeskConfig, ZendeskCredentialJson>
+              connectorIndexingStatuses={zendeskConnectorIndexingStatuses}
+              liveCredential={zendeskCredential}
+              getCredential={(credential) => {
+                return (
+                  <div>
+                    <p>{credential.credential_json.zendesk_email}</p>
+                  </div>
+                );
+              }}
+              onCredentialLink={async (connectorId) => {
+                if (zendeskCredential) {
+                  await linkCredential(connectorId, zendeskCredential.id);
+                  mutate("/api/manage/admin/connector/indexing-status");
+                }
+              }}
+              onUpdate={() =>
+                mutate("/api/manage/admin/connector/indexing-status")
+              }
+            />
+          </div>
+        </>
+      )}
+
+      {zendeskCredential &&
+        zendeskConnectorIndexingStatuses.length === 0 && (
+          <>
+            <div className="border-solid border-gray-600 border rounded-md p-6 mt-4">
+              <h2 className="font-bold mb-3">Create Connection</h2>
+              <p className="text-sm mb-4">
+                Press connect below to start the connection to your Zendesk
+                instance.
+              </p>
+              <ConnectorForm<ZendeskConfig>
+                nameBuilder={(values) => `ZendeskConnector`}
+                ccPairNameBuilder={(values) => `ZendeskConnector`}
+                source="zendesk"
+                inputType="poll"
+                formBody={<></>}
+                validationSchema={Yup.object().shape({})}
+                initialValues={{}}
+                refreshFreq={10 * 60} // 10 minutes
+                credentialId={zendeskCredential.id}
+              />
+            </div>
+          </>
+        )}
+
+      {!zendeskCredential && (
+        <>
+          <p className="text-sm mb-4">
+            Please provide your API details in Step 1 first! Once done with
+            that, you&apos;ll be able to start the connection then see indexing
+            status.
+          </p>
+        </>
+      )}
+    </>
+  );
+};
+
+export default function Page() {
+  return (
+    <div className="mx-auto container">
+      <div className="mb-4">
+        <HealthCheckBanner />
+      </div>
+      <div className="border-solid border-gray-600 border-b mb-4 pb-2 flex">
+        <ZendeskIcon size={32} />
+        <h1 className="text-3xl font-bold pl-2">Zendesk</h1>
+      </div>
+      <Main />
+    </div>
+  );
+}

web/src/components/admin/Layout.tsx

@@ -26,6 +26,7 @@ import {
   GoogleSitesIcon,
   GongIcon,
   ZoomInIcon,
+  ZendeskIcon
 } from "@/components/icons/icons";
 import { getAuthDisabledSS, getCurrentUserSS } from "@/lib/userSS";
 import { redirect } from "next/navigation";
@@ -231,6 +232,15 @@ export async function Layout({ children }: { children: React.ReactNode }) {
                   ),
                   link: "/admin/connectors/document360",
                 },
+                {
+                  name: (
+                    <div className="flex">
+                      <ZendeskIcon size={16} />
+                      <div className="ml-1">Zendesk</div>
+                    </div>
+                  ),
+                  link: "/admin/connectors/zendesk",
+                }
               ],
             },
             {

web/src/components/icons/icons.tsx

@@ -45,6 +45,7 @@ import linearIcon from "../../../public/Linear.png";
 import hubSpotIcon from "../../../public/HubSpot.png";
 import document360Icon from "../../../public/Document360.png";
 import googleSitesIcon from "../../../public/GoogleSites.png";
+import zendeskIcon from "../../../public/Zendesk.svg";
 
 interface IconProps {
   size?: number;
@@ -480,3 +481,15 @@ export const GoogleSitesIcon = ({
     </div>
   );
 };
+
+export const ZendeskIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => (
+  <div
+    style={{ width: `${size}px`, height: `${size}px` }}
+    className={`w-[${size}px] h-[${size}px] ` + className}
+  >
+    <Image src={zendeskIcon} alt="Logo" width="96" height="96" />
+  </div>
+);

web/src/components/source.tsx

@@ -18,6 +18,7 @@ import {
   HubSpotIcon,
   Document360Icon,
   GoogleSitesIcon,
+  ZendeskIcon,
 } from "./icons/icons";
 
 interface SourceMetadata {
@@ -136,6 +137,12 @@ export const getSourceMetadata = (sourceType: ValidSources): SourceMetadata => {
         displayName: "Google Sites",
         adminPageLink: "/admin/connectors/google-sites",
       };
+    case "zendesk":
+      return {
+        icon: ZendeskIcon,
+        displayName: "Zendesk",
+        adminPageLink: "/admin/connectors/zendesk",
+      }
     default:
       throw new Error("Invalid source type");
   }

web/src/lib/types.ts

@@ -25,7 +25,8 @@ export type ValidSources =
   | "hubspot"
   | "document360"
   | "file"
-  | "google_sites";
+  | "google_sites"
+  | "zendesk";
 
 export type ValidInputTypes = "load_state" | "poll" | "event";
 export type ValidStatuses =
@@ -128,6 +129,8 @@ export interface GoogleSitesConfig {
   base_url: string;
 }
 
+export interface ZendeskConfig{}
+
 export interface IndexAttemptSnapshot {
   id: number;
   status: ValidStatuses | null;
@@ -242,6 +245,12 @@ export interface Document360CredentialJson {
   document360_api_token: string;
 }
 
+export interface ZendeskCredentialJson {
+  zendesk_subdomain: string;
+  zendesk_email: string;
+  zendesk_token: string;
+}
+
 // DELETION
 
 export interface DeletionAttemptSnapshot {