create a hubspot connector (#482)

backend/.gitignore

@@ -8,3 +8,4 @@ qdrant-data/
 typesense-data/
 .env
 vespa-app.zip
+dynamic_config_storage/

backend/danswer/configs/constants.py

@@ -59,6 +59,7 @@ class DocumentSource(str, Enum):
     NOTION = "notion"
     ZULIP = "zulip"
     LINEAR = "linear"
+    HUBSPOT = "hubspot"
 
 
 class DocumentIndexType(str, Enum):

backend/danswer/connectors/factory.py

@@ -22,6 +22,7 @@ from danswer.connectors.slack.connector import SlackLoadConnector
 from danswer.connectors.slack.connector import SlackPollConnector
 from danswer.connectors.web.connector import WebConnector
 from danswer.connectors.zulip.connector import ZulipConnector
+from danswer.connectors.hubspot.connector import HubSpotConnector
 
 
 class ConnectorMissingException(Exception):
@@ -50,6 +51,7 @@ def identify_connector_class(
         DocumentSource.ZULIP: ZulipConnector,
         DocumentSource.GURU: GuruConnector,
         DocumentSource.LINEAR: LinearConnector,
+        DocumentSource.HUBSPOT: HubSpotConnector,
     }
     connector_by_source = connector_map.get(source, {})
 

backend/danswer/connectors/hubspot/connector.py

@@ -0,0 +1,138 @@
+import requests
+import json
+from typing import Any
+from datetime import datetime
+from hubspot import HubSpot
+from danswer.configs.app_configs import INDEX_BATCH_SIZE
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.interfaces import GenerateDocumentsOutput
+from danswer.connectors.interfaces import LoadConnector
+from danswer.connectors.interfaces import PollConnector
+from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+from danswer.connectors.models import ConnectorMissingCredentialError
+from danswer.connectors.models import Document
+from danswer.connectors.models import Section
+from danswer.utils.logger import setup_logger
+
+HUBSPOT_BASE_URL = "https://app.hubspot.com/contacts/"
+HUBSPOT_API_URL = "https://api.hubapi.com/integrations/v1/me"
+
+logger = setup_logger()
+
+class HubSpotConnector(LoadConnector, PollConnector):
+    def __init__(self, batch_size: int = INDEX_BATCH_SIZE, access_token: str | None = None) -> None:
+        self.batch_size = batch_size
+        self.access_token = access_token
+        self.portal_id = None
+        self.ticket_base_url = HUBSPOT_BASE_URL
+        
+    def get_portal_id(self) -> str:
+        headers = {
+            'Authorization': f'Bearer {self.access_token}',
+            'Content-Type': 'application/json'
+        }
+        
+        response = requests.get(HUBSPOT_API_URL, headers=headers)
+        if response.status_code != 200:
+            raise Exception("Error fetching portal ID")
+
+        data = response.json()
+        return data["portalId"]
+
+    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
+        self.access_token = credentials["hubspot_access_token"]
+        
+        if self.access_token:
+            self.portal_id = self.get_portal_id()
+            self.ticket_base_url = f"{HUBSPOT_BASE_URL}{self.portal_id}/ticket/"
+
+        return None
+
+    def _process_tickets(
+        self, start: datetime | None = None, end: datetime | None = None
+    ) -> GenerateDocumentsOutput:
+        if self.access_token is None:
+            raise ConnectorMissingCredentialError("HubSpot")
+        
+        api_client = HubSpot(access_token=self.access_token)
+        all_tickets = api_client.crm.tickets.get_all(associations=['contacts', 'notes'])
+
+        doc_batch: list[Document] = []
+        
+        for ticket in all_tickets:
+            updated_at = ticket.updated_at.replace(tzinfo=None)
+            if start is not None and updated_at < start:
+                continue
+            if end is not None and updated_at > end:
+                continue
+                
+            title = ticket.properties["subject"]
+            link = self.ticket_base_url + ticket.id
+            content_text = title + "\n" + ticket.properties["content"]
+
+            associated_emails = []
+            associated_notes = []
+
+            if ticket.associations:
+                contacts = ticket.associations.get("contacts")
+                notes = ticket.associations.get("notes")
+
+                if contacts:
+                    for contact in contacts.results:
+                        contact = api_client.crm.contacts.basic_api.get_by_id(contact_id=contact.id)
+                        associated_emails.append(contact.properties["email"])
+
+                if notes:                   
+                    for note in notes.results:
+                        note = api_client.crm.objects.notes.basic_api.get_by_id(note_id=note.id, properties=["content", "hs_body_preview"])
+                        associated_notes.append(note.properties["hs_body_preview"])
+            
+            associated_emails = " ,".join(associated_emails)
+            associated_notes = " ".join(associated_notes)
+
+            content_text = f"{content_text}\n emails: {associated_emails} \n notes: {associated_notes}"
+
+            doc_batch.append(
+                Document(
+                    id=ticket.id,
+                    sections=[Section(link=link, text=content_text)],
+                    source=DocumentSource.HUBSPOT,
+                    semantic_identifier=title,
+                    metadata={},
+                )
+            )
+            
+            if len(doc_batch) >= self.batch_size:
+                yield doc_batch
+                doc_batch = []
+
+        if doc_batch:
+            yield doc_batch
+
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        return self._process_tickets()
+
+    def poll_source(
+        self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
+    ) -> GenerateDocumentsOutput:
+        start_datetime = datetime.fromtimestamp(start)
+        end_datetime = datetime.fromtimestamp(end)
+        return self._process_tickets(start_datetime, end_datetime)
+
+
+
+if __name__ == "__main__":
+    import os
+    import time
+    test_connector = HubSpotConnector()
+    test_connector.load_credentials({
+        "hubspot_access_token": os.environ["HUBSPOT_ACCESS_TOKEN"]
+    })
+    all_docs = test_connector.load_from_state()
+    
+    current = time.time()
+    one_day_ago = current - 24 * 60 * 60  # 1 day
+    latest_docs = test_connector.poll_source(one_day_ago, current)
+    print(latest_docs)
+
+

backend/requirements/default.txt

@@ -53,3 +53,4 @@ transformers==4.30.1
 typesense==0.15.1
 uvicorn==0.21.1
 zulip==0.8.2
+hubspot-api-client==8.1.0

web/src/app/admin/connectors/hubspot/page.tsx

@@ -0,0 +1,223 @@
+"use client";
+
+import * as Yup from "yup";
+import { HubSpotIcon, TrashIcon } from "@/components/icons/icons";
+import { TextFormField } from "@/components/admin/connectors/Field";
+import { HealthCheckBanner } from "@/components/health/healthcheck";
+import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
+import {
+  Credential,
+  ConnectorIndexingStatus,
+  HubSpotConfig,
+  HubSpotCredentialJson,
+} from "@/lib/types";
+import useSWR, { useSWRConfig } from "swr";
+import { fetcher } from "@/lib/fetcher";
+import { LoadingAnimation } from "@/components/Loading";
+import { adminDeleteCredential, linkCredential } from "@/lib/credential";
+import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
+import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
+import { usePopup } from "@/components/admin/connectors/Popup";
+import { usePublicCredentials } from "@/lib/hooks";
+
+const Main = () => {
+  const { popup, setPopup } = usePopup();
+
+  const { mutate } = useSWRConfig();
+  const {
+    data: connectorIndexingStatuses,
+    isLoading: isConnectorIndexingStatusesLoading,
+    error: isConnectorIndexingStatusesError,
+  } = useSWR<ConnectorIndexingStatus<any, any>[]>(
+    "/api/manage/admin/connector/indexing-status",
+    fetcher
+  );
+
+  const {
+    data: credentialsData,
+    isLoading: isCredentialsLoading,
+    isValidating: isCredentialsValidating,
+    error: isCredentialsError,
+    refreshCredentials,
+  } = usePublicCredentials();
+
+  if (
+    isConnectorIndexingStatusesLoading ||
+    isCredentialsLoading ||
+    isCredentialsValidating
+  ) {
+    return <LoadingAnimation text="Loading" />;
+  }
+
+  if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) {
+    return <div>Failed to load connectors</div>;
+  }
+
+  if (isCredentialsError || !credentialsData) {
+    return <div>Failed to load credentials</div>;
+  }
+
+  const hubSpotConnectorIndexingStatuses: ConnectorIndexingStatus<
+    HubSpotConfig,
+    HubSpotCredentialJson
+  >[] = connectorIndexingStatuses.filter(
+    (connectorIndexingStatus) =>
+      connectorIndexingStatus.connector.source === "hubspot"
+  );
+  const hubSpotCredential: Credential<HubSpotCredentialJson> = credentialsData.filter(
+    (credential) => credential.credential_json?.hubspot_access_token
+  )[0];
+
+  return (
+    <>
+      {popup}
+      <p className="text-sm">
+        This connector allows you to sync all your HubSpot Tickets into Danswer.
+      </p>
+
+      <h2 className="font-bold mb-2 mt-6 ml-auto mr-auto">
+        Step 1: Provide your Credentials
+      </h2>
+
+      {hubSpotCredential ? (
+        <>
+          <div className="flex mb-1 text-sm">
+            <p className="my-auto">Existing Access Token: </p>
+            <p className="ml-1 italic my-auto max-w-md truncate">
+              {hubSpotCredential.credential_json?.hubspot_access_token}
+            </p>
+            <button
+              className="ml-1 hover:bg-gray-700 rounded-full p-1"
+              onClick={async () => {
+                if (hubSpotConnectorIndexingStatuses.length > 0) {
+                  setPopup({
+                    type: "error",
+                    message:
+                      "Must delete all connectors before deleting credentials",
+                  });
+                  return;
+                }
+                await adminDeleteCredential(hubSpotCredential.id);
+                refreshCredentials();
+              }}
+            >
+              <TrashIcon />
+            </button>
+          </div>
+        </>
+      ) : (
+        <>
+          <p className="text-sm">
+            To use the HubSpot connector, provide the HubSpot Access Token.
+          </p>
+          <div className="border-solid border-gray-600 border rounded-md p-6 mt-2">
+            <CredentialForm<HubSpotCredentialJson>
+              formBody={
+                <>
+                  <TextFormField
+                    name="hubspot_access_token"
+                    label="HubSpot Access Token:"
+                    type="password"
+                  />
+                </>
+              }
+              validationSchema={Yup.object().shape({
+                hubspot_access_token: Yup.string().required(
+                  "Please enter your HubSpot Access Token"
+                )
+              })}
+              initialValues={{
+                hubspot_access_token: "",
+              }}
+              onSubmit={(isSuccess) => {
+                if (isSuccess) {
+                  refreshCredentials();
+                }
+              }}
+            />
+          </div>
+        </>
+      )}
+
+      <h2 className="font-bold mb-2 mt-6 ml-auto mr-auto">
+        Step 2: Start indexing!
+      </h2>
+      {hubSpotCredential ? (
+        !hubSpotConnectorIndexingStatuses.length ? (
+          <>
+            <p className="text-sm mb-2">
+              Click the button below to start indexing! We will pull the latest
+              tickets from HubSpot every <b>10</b> minutes.
+            </p>
+            <div className="flex">
+              <ConnectorForm<HubSpotConfig>
+                nameBuilder={() => "HubSpotConnector"}
+                source="hubspot"
+                inputType="poll"
+                formBody={null}
+                validationSchema={Yup.object().shape({})}
+                initialValues={{}}
+                refreshFreq={10 * 60} // 10 minutes
+                onSubmit={async (isSuccess, responseJson) => {
+                  if (isSuccess && responseJson) {
+                    await linkCredential(responseJson.id, hubSpotCredential.id);
+                    mutate("/api/manage/admin/connector/indexing-status");
+                  }
+                }}
+              />
+            </div>
+          </>
+        ) : (
+          <>
+            <p className="text-sm mb-2">
+              HubSpot connector is setup! We are pulling the latest tickets from HubSpot
+              every <b>10</b> minutes.
+            </p>
+            <ConnectorsTable<HubSpotConfig, HubSpotCredentialJson>
+              connectorIndexingStatuses={hubSpotConnectorIndexingStatuses}
+              liveCredential={hubSpotCredential}
+              getCredential={(credential) => {
+                return (
+                  <div>
+                    <p>{credential.credential_json.hubspot_access_token}</p>
+                  </div>
+                );
+              }}
+              onCredentialLink={async (connectorId) => {
+                if (hubSpotCredential) {
+                  await linkCredential(connectorId, hubSpotCredential.id);
+                  mutate("/api/manage/admin/connector/indexing-status");
+                }
+              }}
+              onUpdate={() =>
+                mutate("/api/manage/admin/connector/indexing-status")
+              }
+            />
+          </>
+        )
+      ) : (
+        <>
+          <p className="text-sm">
+            Please provide your access token in Step 1 first! Once done with
+            that, you can then start indexing all your HubSpot tickets.
+          </p>
+        </>
+      )}
+    </>
+  );
+};
+
+export default function Page() {
+  return (
+    <div className="mx-auto container">
+      <div className="mb-4">
+        <HealthCheckBanner />
+      </div>
+      <div className="border-solid border-gray-600 border-b mb-4 pb-2 flex">
+        <HubSpotIcon size={32} />
+        <h1 className="text-3xl font-bold pl-2">HubSpot</h1>
+      </div>
+      <Main />
+    </div>
+  );
+}

web/src/app/admin/layout.tsx

@@ -19,6 +19,7 @@ import {
   LinearIcon,
   UsersIcon,
   ThumbsUpIcon,
+  HubSpotIcon,
   BookmarkIcon,
   CPUIcon,
 } from "@/components/icons/icons";
@@ -192,6 +193,15 @@ export default async function AdminLayout({
                   ),
                   link: "/admin/connectors/file",
                 },
+                {
+                  name: (
+                    <div className="flex">
+                      <HubSpotIcon size={16} />
+                      <div className="ml-1">HubSpot</div>
+                    </div>
+                  ),
+                  link: "/admin/connectors/hubspot",
+                },
               ],
             },
             {

web/src/components/icons/icons.tsx

@@ -41,6 +41,7 @@ import confluenceSVG from "../../../public/Confluence.svg";
 import guruIcon from "../../../public/Guru.svg";
 import zulipIcon from "../../../public/Zulip.png";
 import linearIcon from "../../../public/Linear.png";
+import hubSpotIcon from "../../../public/HubSpot.png";
 
 interface IconProps {
   size?: number;
@@ -421,3 +422,18 @@ export const GuruIcon = ({
     <Image src={guruIcon} alt="Logo" width="96" height="96" />
   </div>
 );
+
+export const HubSpotIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return (
+    <div
+      // HubSpot Icon has a bit more surrounding whitespace than other icons, which is why we need to adjust it here
+      style={{ width: `${size + 4}px`, height: `${size + 4}px` }}
+      className={`w-[${size + 4}px] h-[${size + 4}px] -m-0.5 ` + className}
+    >
+      <Image src={hubSpotIcon} alt="Logo" width="96" height="96" />
+    </div>
+  );
+};

web/src/components/search/Filters.tsx

@@ -27,6 +27,7 @@ const sources: Source[] = [
   { displayName: "Notion", internalName: "notion" },
   { displayName: "Zulip", internalName: "zulip" },
   { displayName: "Linear", internalName: "linear" },
+  { displayName: "HubSpot", internalName: "hubspot" },
 ];
 
 interface SourceSelectorProps {

web/src/components/source.tsx

@@ -14,6 +14,7 @@ import {
   SlabIcon,
   SlackIcon,
   ZulipIcon,
+  HubSpotIcon,
 } from "./icons/icons";
 
 interface SourceMetadata {
@@ -108,6 +109,12 @@ export const getSourceMetadata = (sourceType: ValidSources): SourceMetadata => {
         displayName: "Linear",
         adminPageLink: "/admin/connectors/linear",
       };
+    case "hubspot":
+      return {
+        icon: HubSpotIcon,
+        displayName: "HubSpot",
+        adminPageLink: "/admin/connectors/hubspot",
+      };
     default:
       throw new Error("Invalid source type");
   }

web/src/lib/types.ts

@@ -21,6 +21,7 @@ export type ValidSources =
   | "guru"
   | "zulip"
   | "linear"
+  | "hubspot"
   | "file";
 export type ValidInputTypes = "load_state" | "poll" | "event";
 export type ValidStatuses =
@@ -106,6 +107,8 @@ export interface ZulipConfig {
 
 export interface NotionConfig {}
 
+export interface HubSpotConfig {}
+
 export interface IndexAttemptSnapshot {
   status: ValidStatuses | null;
   num_docs_indexed: number;
@@ -204,6 +207,10 @@ export interface LinearCredentialJson {
   linear_api_key: string;
 }
 
+export interface HubSpotCredentialJson {
+  hubspot_access_token: string;
+}
+
 // DELETION
 
 export interface DeletionAttemptSnapshot {