From 44e3dcb19f06fe57ab58f51b7e8927025b904dec Mon Sep 17 00:00:00 2001 From: Bryan Peterson Date: Thu, 2 Nov 2023 05:11:56 +0100 Subject: [PATCH] support for zendesk help center (#661) --- .gitignore | 1 + backend/danswer/configs/constants.py | 1 + backend/danswer/connectors/factory.py | 2 + .../danswer/connectors/zendesk/__init__.py | 0 .../danswer/connectors/zendesk/connector.py | 63 +++++ backend/requirements/default.txt | 1 + backend/scripts/reset_indexes.py | 6 + backend/scripts/reset_postgres.py | 6 + web/public/Zendesk.svg | 8 + web/src/app/admin/connectors/zendesk/page.tsx | 242 ++++++++++++++++++ web/src/components/admin/Layout.tsx | 10 + web/src/components/icons/icons.tsx | 13 + web/src/components/source.tsx | 7 + web/src/lib/types.ts | 11 +- 14 files changed, 370 insertions(+), 1 deletion(-) create mode 100644 backend/danswer/connectors/zendesk/__init__.py create mode 100644 backend/danswer/connectors/zendesk/connector.py create mode 100644 web/public/Zendesk.svg create mode 100644 web/src/app/admin/connectors/zendesk/page.tsx diff --git a/.gitignore b/.gitignore index 6069bf98f..b582ec01c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .env .DS_store +.venv \ No newline at end of file diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py index 24886c1e7..98d199e17 100644 --- a/backend/danswer/configs/constants.py +++ b/backend/danswer/configs/constants.py @@ -70,6 +70,7 @@ class DocumentSource(str, Enum): DOCUMENT360 = "document360" GONG = "gong" GOOGLE_SITES = "google_sites" + ZENDESK = "zendesk" class DocumentIndexType(str, Enum): diff --git a/backend/danswer/connectors/factory.py b/backend/danswer/connectors/factory.py index 3acc0f376..bc477e69a 100644 --- a/backend/danswer/connectors/factory.py +++ b/backend/danswer/connectors/factory.py @@ -26,6 +26,7 @@ from danswer.connectors.slack.connector import SlackLoadConnector from danswer.connectors.slack.connector import SlackPollConnector from danswer.connectors.web.connector import WebConnector from danswer.connectors.zulip.connector import ZulipConnector +from danswer.connectors.zendesk.connector import ZendeskConnector class ConnectorMissingException(Exception): @@ -58,6 +59,7 @@ def identify_connector_class( DocumentSource.DOCUMENT360: Document360Connector, DocumentSource.GONG: GongConnector, DocumentSource.GOOGLE_SITES: GoogleSitesConnector, + DocumentSource.ZENDESK: ZendeskConnector, } connector_by_source = connector_map.get(source, {}) diff --git a/backend/danswer/connectors/zendesk/__init__.py b/backend/danswer/connectors/zendesk/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/danswer/connectors/zendesk/connector.py b/backend/danswer/connectors/zendesk/connector.py new file mode 100644 index 000000000..aa967b6dd --- /dev/null +++ b/backend/danswer/connectors/zendesk/connector.py @@ -0,0 +1,63 @@ +from typing import Any +from zenpy import Zenpy +from zenpy.lib.api_objects.help_centre_objects import Article + +from danswer.configs.app_configs import INDEX_BATCH_SIZE +from danswer.configs.constants import DocumentSource +from danswer.connectors.models import Document, Section +from danswer.connectors.interfaces import GenerateDocumentsOutput, LoadConnector, PollConnector, SecondsSinceUnixEpoch + +class ZendeskClientNotSetUpError(PermissionError): + def __init__(self) -> None: + super().__init__( + "Zendesk Client is not set up, was load_credentials called?" + ) + + +class ZendeskConnector(LoadConnector, PollConnector): + def __init__( + self, + batch_size: int = INDEX_BATCH_SIZE + ) -> None: + self.batch_size = batch_size + self.zendesk_client: Zenpy | None = None + + def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None: + self.zendesk_client = Zenpy( + subdomain=credentials["zendesk_subdomain"], + email=credentials["zendesk_email"], + token=credentials["zendesk_token"], + ) + return None + + def load_from_state(self) -> GenerateDocumentsOutput: + return self.poll_source(None, None) + + def _article_to_document(self, article: Article) -> Document: + return Document( + id=f"article:{article.id}", + sections=[Section(link=article.html_url, text=article.body)], + source=DocumentSource.ZENDESK, + semantic_identifier="Article: " + article.title, + metadata={ + "type": "article", + "updated_at": article.updated_at, + } + ) + + + def poll_source(self, start: SecondsSinceUnixEpoch | None, end: SecondsSinceUnixEpoch | None) -> GenerateDocumentsOutput: + if self.zendesk_client is None: + raise ZendeskClientNotSetUpError() + + articles = self.zendesk_client.help_center.articles(cursor_pagination=True) if start is None else self.zendesk_client.help_center.articles.incremental(start_time=int(start)) + doc_batch = [] + for article in articles: + if article.body is None: + continue + + doc_batch.append(self._article_to_document(article)) + if len(doc_batch) >= self.batch_size: + yield doc_batch + doc_batch.clear() + diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt index 5d276647a..4b6b31e75 100644 --- a/backend/requirements/default.txt +++ b/backend/requirements/default.txt @@ -56,3 +56,4 @@ transformers==4.30.1 uvicorn==0.21.1 zulip==0.8.2 hubspot-api-client==8.1.0 +zenpy==2.0.41 diff --git a/backend/scripts/reset_indexes.py b/backend/scripts/reset_indexes.py index c8ffe5620..d606336bc 100644 --- a/backend/scripts/reset_indexes.py +++ b/backend/scripts/reset_indexes.py @@ -1,5 +1,11 @@ # This file is purely for development use, not included in any builds import requests +import os +import sys + +parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(parent_dir) + from danswer.configs.app_configs import DOCUMENT_INDEX_NAME from danswer.document_index.vespa.index import DOCUMENT_ID_ENDPOINT diff --git a/backend/scripts/reset_postgres.py b/backend/scripts/reset_postgres.py index 59a3fd055..e33b57b00 100644 --- a/backend/scripts/reset_postgres.py +++ b/backend/scripts/reset_postgres.py @@ -1,5 +1,11 @@ import psycopg2 +import os +import sys + +parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(parent_dir) + from danswer.configs.app_configs import POSTGRES_DB from danswer.configs.app_configs import POSTGRES_HOST from danswer.configs.app_configs import POSTGRES_PASSWORD diff --git a/web/public/Zendesk.svg b/web/public/Zendesk.svg new file mode 100644 index 000000000..cc7edc68c --- /dev/null +++ b/web/public/Zendesk.svg @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/web/src/app/admin/connectors/zendesk/page.tsx b/web/src/app/admin/connectors/zendesk/page.tsx new file mode 100644 index 000000000..9ebaa7bff --- /dev/null +++ b/web/src/app/admin/connectors/zendesk/page.tsx @@ -0,0 +1,242 @@ +"use client"; + +import * as Yup from "yup"; +import { TrashIcon, ZendeskIcon } from "@/components/icons/icons"; +import { TextFormField } from "@/components/admin/connectors/Field"; +import { HealthCheckBanner } from "@/components/health/healthcheck"; +import { CredentialForm } from "@/components/admin/connectors/CredentialForm"; +import { + ZendeskCredentialJson, + ZendeskConfig, + ConnectorIndexingStatus, + Credential, +} from "@/lib/types"; +import useSWR, { useSWRConfig } from "swr"; +import { fetcher } from "@/lib/fetcher"; +import { LoadingAnimation } from "@/components/Loading"; +import { adminDeleteCredential, linkCredential } from "@/lib/credential"; +import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; +import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable"; +import { usePopup } from "@/components/admin/connectors/Popup"; +import { usePublicCredentials } from "@/lib/hooks"; + +const Main = () => { + const { popup, setPopup } = usePopup(); + + const { mutate } = useSWRConfig(); + const { + data: connectorIndexingStatuses, + isLoading: isConnectorIndexingStatusesLoading, + error: isConnectorIndexingStatusesError, + } = useSWR[]>( + "/api/manage/admin/connector/indexing-status", + fetcher + ); + const { + data: credentialsData, + isLoading: isCredentialsLoading, + error: isCredentialsError, + refreshCredentials, + } = usePublicCredentials(); + + if ( + (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) || + (!credentialsData && isCredentialsLoading) + ) { + return ; + } + + if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) { + return
Failed to load connectors
; + } + + if (isCredentialsError || !credentialsData) { + return
Failed to load credentials
; + } + + const zendeskConnectorIndexingStatuses: ConnectorIndexingStatus< + ZendeskConfig, + ZendeskCredentialJson + >[] = connectorIndexingStatuses.filter( + (connectorIndexingStatus) => + connectorIndexingStatus.connector.source === "zendesk" + ); + const zendeskCredential: Credential | undefined = + credentialsData.find( + (credential) => credential.credential_json?.zendesk_email + ); + + return ( + <> + {popup} +

+ Step 1: Provide your API details +

+ + {zendeskCredential ? ( + <> +
+

Existing API Token:

+

+ {zendeskCredential.credential_json?.zendesk_email} +

+ +
+ + ) : ( + <> +

+ To get started you'll need API token details for your Zendesk + instance. You can generate this by access the Admin Center of your instance + (e.g. https://<subdomain>.zendesk.com/admin/). Proceed to the "Apps and + Integrations" section and "Zendesk API" page. Add a new API token and provide + it with a name. You will also need to provide the e-mail address of a user that + the system will impersonate. This is of little consequence as we are only performing + read actions. +

+
+ + formBody={ + <> + + + + + } + validationSchema={Yup.object().shape({ + zendesk_subdomain: Yup.string().required( + "Please enter the subdomain for your Zendesk instance" + ), + zendesk_email: Yup.string().required( + "Please enter your user email to user with the token" + ), + zendesk_token: Yup.string().required( + "Please enter your Zendesk API token" + ), + })} + initialValues={{ + zendesk_subdomain: "", + zendesk_email: "", + zendesk_token: "", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + refreshCredentials(); + mutate("/api/manage/admin/connector/indexing-status"); + } + }} + /> +
+ + )} + + {zendeskConnectorIndexingStatuses.length > 0 && ( + <> +

+ Zendesk indexing status +

+

+ The latest article changes are fetched every 10 minutes. +

+
+ + connectorIndexingStatuses={zendeskConnectorIndexingStatuses} + liveCredential={zendeskCredential} + getCredential={(credential) => { + return ( +
+

{credential.credential_json.zendesk_email}

+
+ ); + }} + onCredentialLink={async (connectorId) => { + if (zendeskCredential) { + await linkCredential(connectorId, zendeskCredential.id); + mutate("/api/manage/admin/connector/indexing-status"); + } + }} + onUpdate={() => + mutate("/api/manage/admin/connector/indexing-status") + } + /> +
+ + )} + + {zendeskCredential && + zendeskConnectorIndexingStatuses.length === 0 && ( + <> +
+

Create Connection

+

+ Press connect below to start the connection to your Zendesk + instance. +

+ + nameBuilder={(values) => `ZendeskConnector`} + ccPairNameBuilder={(values) => `ZendeskConnector`} + source="zendesk" + inputType="poll" + formBody={<>} + validationSchema={Yup.object().shape({})} + initialValues={{}} + refreshFreq={10 * 60} // 10 minutes + credentialId={zendeskCredential.id} + /> +
+ + )} + + {!zendeskCredential && ( + <> +

+ Please provide your API details in Step 1 first! Once done with + that, you'll be able to start the connection then see indexing + status. +

+ + )} + + ); +}; + +export default function Page() { + return ( +
+
+ +
+
+ +

Zendesk

+
+
+
+ ); +} diff --git a/web/src/components/admin/Layout.tsx b/web/src/components/admin/Layout.tsx index 171ece11a..8d426c068 100644 --- a/web/src/components/admin/Layout.tsx +++ b/web/src/components/admin/Layout.tsx @@ -26,6 +26,7 @@ import { GoogleSitesIcon, GongIcon, ZoomInIcon, + ZendeskIcon } from "@/components/icons/icons"; import { getAuthDisabledSS, getCurrentUserSS } from "@/lib/userSS"; import { redirect } from "next/navigation"; @@ -231,6 +232,15 @@ export async function Layout({ children }: { children: React.ReactNode }) { ), link: "/admin/connectors/document360", }, + { + name: ( +
+ +
Zendesk
+
+ ), + link: "/admin/connectors/zendesk", + } ], }, { diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx index f8217df56..1695ec590 100644 --- a/web/src/components/icons/icons.tsx +++ b/web/src/components/icons/icons.tsx @@ -45,6 +45,7 @@ import linearIcon from "../../../public/Linear.png"; import hubSpotIcon from "../../../public/HubSpot.png"; import document360Icon from "../../../public/Document360.png"; import googleSitesIcon from "../../../public/GoogleSites.png"; +import zendeskIcon from "../../../public/Zendesk.svg"; interface IconProps { size?: number; @@ -480,3 +481,15 @@ export const GoogleSitesIcon = ({ ); }; + +export const ZendeskIcon = ({ + size = 16, + className = defaultTailwindCSS, +}: IconProps) => ( +
+ Logo +
+); \ No newline at end of file diff --git a/web/src/components/source.tsx b/web/src/components/source.tsx index 145aa630a..fa6ee3f66 100644 --- a/web/src/components/source.tsx +++ b/web/src/components/source.tsx @@ -18,6 +18,7 @@ import { HubSpotIcon, Document360Icon, GoogleSitesIcon, + ZendeskIcon, } from "./icons/icons"; interface SourceMetadata { @@ -136,6 +137,12 @@ export const getSourceMetadata = (sourceType: ValidSources): SourceMetadata => { displayName: "Google Sites", adminPageLink: "/admin/connectors/google-sites", }; + case "zendesk": + return { + icon: ZendeskIcon, + displayName: "Zendesk", + adminPageLink: "/admin/connectors/zendesk", + } default: throw new Error("Invalid source type"); } diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 381b3a147..61420d96f 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -25,7 +25,8 @@ export type ValidSources = | "hubspot" | "document360" | "file" - | "google_sites"; + | "google_sites" + | "zendesk"; export type ValidInputTypes = "load_state" | "poll" | "event"; export type ValidStatuses = @@ -128,6 +129,8 @@ export interface GoogleSitesConfig { base_url: string; } +export interface ZendeskConfig{} + export interface IndexAttemptSnapshot { id: number; status: ValidStatuses | null; @@ -242,6 +245,12 @@ export interface Document360CredentialJson { document360_api_token: string; } +export interface ZendeskCredentialJson { + zendesk_subdomain: string; + zendesk_email: string; + zendesk_token: string; +} + // DELETION export interface DeletionAttemptSnapshot {