Added ClickUp Connector (#1521)

* Added connector for clickup

* Fixed mypy issues

* Fallback to description if markdown is not available

* Added extra information in metadata, and support to index comments

* Fixes for fields parsing

* updated fetcher to errorHandlingFetcher

---------

Co-authored-by: hagen-danswer <hagen@danswer.ai>

backend/danswer/configs/constants.py

@@ -98,6 +98,7 @@ class DocumentSource(str, Enum):
     TEAMS = "teams"
     DISCOURSE = "discourse"
     AXERO = "axero"
+    CLICKUP = "clickup"
     MEDIAWIKI = "mediawiki"
     WIKIPEDIA = "wikipedia"
 

backend/danswer/connectors/clickup/connector.py

@@ -0,0 +1,216 @@
+from datetime import datetime
+from datetime import timezone
+from typing import Any
+from typing import Optional
+
+import requests
+
+from danswer.configs.app_configs import INDEX_BATCH_SIZE
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.cross_connector_utils.rate_limit_wrapper import (
+    rate_limit_builder,
+)
+from danswer.connectors.cross_connector_utils.retry_wrapper import retry_builder
+from danswer.connectors.interfaces import GenerateDocumentsOutput
+from danswer.connectors.interfaces import LoadConnector
+from danswer.connectors.interfaces import PollConnector
+from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+from danswer.connectors.models import BasicExpertInfo
+from danswer.connectors.models import ConnectorMissingCredentialError
+from danswer.connectors.models import Document
+from danswer.connectors.models import Section
+
+
+CLICKUP_API_BASE_URL = "https://api.clickup.com/api/v2"
+
+
+class ClickupConnector(LoadConnector, PollConnector):
+    def __init__(
+        self,
+        batch_size: int = INDEX_BATCH_SIZE,
+        api_token: str | None = None,
+        team_id: str | None = None,
+        connector_type: str | None = None,
+        connector_ids: list[str] | None = None,
+        retrieve_task_comments: bool = True,
+    ) -> None:
+        self.batch_size = batch_size
+        self.api_token = api_token
+        self.team_id = team_id
+        self.connector_type = connector_type if connector_type else "workspace"
+        self.connector_ids = connector_ids
+        self.retrieve_task_comments = retrieve_task_comments
+
+    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
+        self.api_token = credentials["clickup_api_token"]
+        self.team_id = credentials["clickup_team_id"]
+        return None
+
+    @retry_builder()
+    @rate_limit_builder(max_calls=100, period=60)
+    def _make_request(self, endpoint: str, params: Optional[dict] = None) -> Any:
+        if not self.api_token:
+            raise ConnectorMissingCredentialError("Clickup")
+
+        headers = {"Authorization": self.api_token}
+
+        response = requests.get(
+            f"{CLICKUP_API_BASE_URL}/{endpoint}", headers=headers, params=params
+        )
+
+        response.raise_for_status()
+
+        return response.json()
+
+    def _get_task_comments(self, task_id: str) -> list[Section]:
+        url_endpoint = f"/task/{task_id}/comment"
+        response = self._make_request(url_endpoint)
+        comments = [
+            Section(
+                link=f'https://app.clickup.com/t/{task_id}?comment={comment_dict["id"]}',
+                text=comment_dict["comment_text"],
+            )
+            for comment_dict in response["comments"]
+        ]
+
+        return comments
+
+    def _get_all_tasks_filtered(
+        self,
+        start: int | None = None,
+        end: int | None = None,
+    ) -> GenerateDocumentsOutput:
+        doc_batch: list[Document] = []
+        page: int = 0
+        params = {
+            "include_markdown_description": "true",
+            "include_closed": "true",
+            "page": page,
+        }
+
+        if start is not None:
+            params["date_updated_gt"] = start
+        if end is not None:
+            params["date_updated_lt"] = end
+
+        if self.connector_type == "list":
+            params["list_ids[]"] = self.connector_ids
+        elif self.connector_type == "folder":
+            params["project_ids[]"] = self.connector_ids
+        elif self.connector_type == "space":
+            params["space_ids[]"] = self.connector_ids
+
+        url_endpoint = f"/team/{self.team_id}/task"
+
+        while True:
+            response = self._make_request(url_endpoint, params)
+
+            page += 1
+            params["page"] = page
+
+            for task in response["tasks"]:
+                document = Document(
+                    id=task["id"],
+                    source=DocumentSource.CLICKUP,
+                    semantic_identifier=task["name"],
+                    doc_updated_at=(
+                        datetime.fromtimestamp(
+                            round(float(task["date_updated"]) / 1000, 3)
+                        ).replace(tzinfo=timezone.utc)
+                    ),
+                    primary_owners=[
+                        BasicExpertInfo(
+                            display_name=task["creator"]["username"],
+                            email=task["creator"]["email"],
+                        )
+                    ],
+                    secondary_owners=[
+                        BasicExpertInfo(
+                            display_name=assignee["username"],
+                            email=assignee["email"],
+                        )
+                        for assignee in task["assignees"]
+                    ],
+                    title=task["name"],
+                    sections=[
+                        Section(
+                            link=task["url"],
+                            text=(
+                                task["markdown_description"]
+                                if "markdown_description" in task
+                                else task["description"]
+                            ),
+                        )
+                    ],
+                    metadata={
+                        "id": task["id"],
+                        "status": task["status"]["status"],
+                        "list": task["list"]["name"],
+                        "project": task["project"]["name"],
+                        "folder": task["folder"]["name"],
+                        "space_id": task["space"]["id"],
+                        "tags": [tag["name"] for tag in task["tags"]],
+                        "priority": (
+                            task["priority"]["priority"]
+                            if "priority" in task and task["priority"] is not None
+                            else ""
+                        ),
+                    },
+                )
+
+                extra_fields = [
+                    "date_created",
+                    "date_updated",
+                    "date_closed",
+                    "date_done",
+                    "due_date",
+                ]
+                for extra_field in extra_fields:
+                    if extra_field in task and task[extra_field] is not None:
+                        document.metadata[extra_field] = task[extra_field]
+
+                if self.retrieve_task_comments:
+                    document.sections.extend(self._get_task_comments(task["id"]))
+
+                doc_batch.append(document)
+
+                if len(doc_batch) >= self.batch_size:
+                    yield doc_batch
+                    doc_batch = []
+
+            if response.get("last_page") is True or len(response["tasks"]) < 100:
+                break
+
+        if doc_batch:
+            yield doc_batch
+
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        if self.api_token is None:
+            raise ConnectorMissingCredentialError("Clickup")
+
+        return self._get_all_tasks_filtered(None, None)
+
+    def poll_source(
+        self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
+    ) -> GenerateDocumentsOutput:
+        if self.api_token is None:
+            raise ConnectorMissingCredentialError("Clickup")
+
+        return self._get_all_tasks_filtered(int(start * 1000), int(end * 1000))
+
+
+if __name__ == "__main__":
+    import os
+
+    clickup_connector = ClickupConnector()
+
+    clickup_connector.load_credentials(
+        {
+            "clickup_api_token": os.environ["clickup_api_token"],
+            "clickup_team_id": os.environ["clickup_team_id"],
+        }
+    )
+    latest_docs = clickup_connector.load_from_state()
+
+    for doc in latest_docs:
+        print(doc)

backend/danswer/connectors/factory.py

@@ -4,6 +4,7 @@ from typing import Type
 from danswer.configs.constants import DocumentSource
 from danswer.connectors.axero.connector import AxeroConnector
 from danswer.connectors.bookstack.connector import BookstackConnector
+from danswer.connectors.clickup.connector import ClickupConnector
 from danswer.connectors.confluence.connector import ConfluenceConnector
 from danswer.connectors.danswer_jira.connector import JiraConnector
 from danswer.connectors.discourse.connector import DiscourseConnector
@@ -80,6 +81,7 @@ def identify_connector_class(
         DocumentSource.TEAMS: TeamsConnector,
         DocumentSource.DISCOURSE: DiscourseConnector,
         DocumentSource.AXERO: AxeroConnector,
+        DocumentSource.CLICKUP: ClickupConnector,
         DocumentSource.MEDIAWIKI: MediaWikiConnector,
         DocumentSource.WIKIPEDIA: WikipediaConnector,
     }

web/public/Clickup.svg

@@ -0,0 +1 @@
+<svg width="130" height="155" xmlns="http://www.w3.org/2000/svg"><defs><linearGradient x1="0%" y1="68.01%" y2="68.01%" id="a"><stop stop-color="#8930FD" offset="0%"/><stop stop-color="#49CCF9" offset="100%"/></linearGradient><linearGradient x1="0%" y1="68.01%" y2="68.01%" id="b"><stop stop-color="#FF02F0" offset="0%"/><stop stop-color="#FFC800" offset="100%"/></linearGradient></defs><g fill-rule="nonzero" fill="none"><path d="M.4 119.12l23.81-18.24C36.86 117.39 50.3 125 65.26 125c14.88 0 27.94-7.52 40.02-23.9l24.15 17.8C112 142.52 90.34 155 65.26 155c-25 0-46.87-12.4-64.86-35.88z" fill="url(#a)"/><path fill="url(#b)" d="M65.18 39.84L22.8 76.36 3.21 53.64 65.27.16l61.57 53.52-19.68 22.64z"/></g></svg>

web/src/app/admin/connectors/clickup/page.tsx

@@ -0,0 +1,343 @@
+"use client";
+
+import * as Yup from "yup";
+import { TrashIcon, ClickupIcon } from "@/components/icons/icons";
+import { errorHandlingFetcher } from "@/lib/fetcher";
+import useSWR, { useSWRConfig } from "swr";
+import { LoadingAnimation } from "@/components/Loading";
+import { HealthCheckBanner } from "@/components/health/healthcheck";
+import {
+  ClickupConfig,
+  ClickupCredentialJson,
+  ConnectorIndexingStatus,
+  Credential,
+} from "@/lib/types";
+import { adminDeleteCredential, linkCredential } from "@/lib/credential";
+import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
+import {
+  BooleanFormField,
+  SelectorFormField,
+  TextFormField,
+  TextArrayFieldBuilder,
+} from "@/components/admin/connectors/Field";
+import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
+import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
+import { usePopup } from "@/components/admin/connectors/Popup";
+import { usePublicCredentials } from "@/lib/hooks";
+import { Title, Text, Card, Divider } from "@tremor/react";
+import { AdminPageTitle } from "@/components/admin/Title";
+
+const MainSection = () => {
+  const { mutate } = useSWRConfig();
+  const { popup, setPopup } = usePopup();
+  const {
+    data: connectorIndexingStatuses,
+    isLoading: isConnectorIndexingStatusesLoading,
+    error: isConnectorIndexingStatusesError,
+  } = useSWR<ConnectorIndexingStatus<any, any>[]>(
+    "/api/manage/admin/connector/indexing-status",
+    errorHandlingFetcher
+  );
+
+  const {
+    data: credentialsData,
+    isLoading: isCredentialsLoading,
+    error: isCredentialsError,
+    refreshCredentials,
+  } = usePublicCredentials();
+
+  if (
+    (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) ||
+    (!credentialsData && isCredentialsLoading)
+  ) {
+    return <LoadingAnimation text="Loading" />;
+  }
+
+  if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) {
+    return <div>Failed to load connectors</div>;
+  }
+
+  if (isCredentialsError || !credentialsData) {
+    return <div>Failed to load credentials</div>;
+  }
+
+  const clickupConnectorIndexingStatuses: ConnectorIndexingStatus<
+    ClickupConfig,
+    ClickupCredentialJson
+  >[] = connectorIndexingStatuses.filter(
+    (connectorIndexingStatus) =>
+      connectorIndexingStatus.connector.source === "clickup"
+  );
+
+  const clickupCredential: Credential<ClickupCredentialJson> | undefined =
+    credentialsData.find(
+      (credential) => credential.credential_json?.clickup_api_token
+    );
+
+  return (
+    <>
+      {popup}
+      <Title className="mb-2 mt-6 ml-auto mr-auto">
+        Step 1: Provide Credentials
+      </Title>
+
+      {clickupCredential ? (
+        <>
+          <div className="flex mb-1 text-sm">
+            <Text className="my-auto">Existing Clickup API Token: </Text>
+            <Text className="ml-1 italic my-auto">
+              {clickupCredential.credential_json.clickup_api_token}
+            </Text>
+            <button
+              className="ml-1 hover:bg-hover rounded p-1"
+              onClick={async () => {
+                if (clickupConnectorIndexingStatuses.length > 0) {
+                  setPopup({
+                    type: "error",
+                    message:
+                      "Must delete all connectors before deleting credentials",
+                  });
+                  return;
+                }
+                await adminDeleteCredential(clickupCredential.id);
+                refreshCredentials();
+              }}
+            >
+              <TrashIcon />
+            </button>
+          </div>
+        </>
+      ) : (
+        <>
+          <Text className="mb-4">
+            To use the Clickup connector, you must first provide the API token
+            and Team ID corresponding to your Clickup setup. See setup guide{" "}
+            <a
+              className="text-link"
+              href="https://docs.danswer.dev/connectors/clickup"
+              target="_blank"
+            >
+              here
+            </a>{" "}
+            for more detail.
+          </Text>
+          <Card className="mt-2">
+            <CredentialForm<ClickupCredentialJson>
+              formBody={
+                <>
+                  <TextFormField
+                    name="clickup_api_token"
+                    label="Clickup API Token:"
+                    type="password"
+                  />
+                  <TextFormField name="clickup_team_id" label="Team ID:" />
+                </>
+              }
+              validationSchema={Yup.object().shape({
+                clickup_api_token: Yup.string().required(
+                  "Please enter your Clickup API token"
+                ),
+                clickup_team_id: Yup.string().required(
+                  "Please enter your Team ID"
+                ),
+              })}
+              initialValues={{
+                clickup_api_token: "",
+                clickup_team_id: "",
+              }}
+              onSubmit={(isSuccess) => {
+                if (isSuccess) {
+                  refreshCredentials();
+                }
+              }}
+            />
+          </Card>
+        </>
+      )}
+
+      <Title className="mb-2 mt-6 ml-auto mr-auto">
+        Step 2: Do you want to search particular space(s), folder(s), list(s),
+        or entire workspace?
+      </Title>
+
+      {clickupConnectorIndexingStatuses.length > 0 && (
+        <>
+          <Text className="mb-2">
+            We index the latest articles from either the entire workspace, or
+            specified space(s), folder(s), list(s) listed below regularly.
+          </Text>
+          <div className="mb-2">
+            <ConnectorsTable<ClickupConfig, ClickupCredentialJson>
+              connectorIndexingStatuses={clickupConnectorIndexingStatuses}
+              liveCredential={clickupCredential}
+              getCredential={(credential) =>
+                credential.credential_json.clickup_api_token
+              }
+              specialColumns={[
+                {
+                  header: "Connector Type",
+                  key: "connector_type",
+                  getValue: (ccPairStatus) =>
+                    ccPairStatus.connector.connector_specific_config
+                      .connector_type,
+                },
+                {
+                  header: "ID(s)",
+                  key: "connector_ids",
+                  getValue: (ccPairStatus) =>
+                    ccPairStatus.connector.connector_specific_config
+                      .connector_ids &&
+                    ccPairStatus.connector.connector_specific_config
+                      .connector_ids.length > 0
+                      ? ccPairStatus.connector.connector_specific_config.connector_ids.join(
+                          ", "
+                        )
+                      : "",
+                },
+                {
+                  header: "Retrieve Task Comments?",
+                  key: "retrieve_task_comments",
+                  getValue: (ccPairStatus) =>
+                    ccPairStatus.connector.connector_specific_config
+                      .retrieve_task_comments
+                      ? "Yes"
+                      : "No",
+                },
+              ]}
+              onUpdate={() =>
+                mutate("/api/manage/admin/connector/indexing-status")
+              }
+              onCredentialLink={async (connectorId) => {
+                if (clickupCredential) {
+                  await linkCredential(connectorId, clickupCredential.id);
+                  mutate("/api/manage/admin/connector/indexing-status");
+                }
+              }}
+            />
+          </div>
+          <Divider />
+        </>
+      )}
+
+      {clickupCredential ? (
+        <Card className="mt-4">
+          <h2 className="font-bold mb-3">Connect to a New Workspace</h2>
+          <ConnectorForm<ClickupConfig>
+            nameBuilder={(values) =>
+              values.connector_ids
+                ? `ClickupConnector-${
+                    values.connector_type
+                  }-${values.connector_ids.join("_")}`
+                : `ClickupConnector-${values.connector_type}`
+            }
+            source="clickup"
+            inputType="poll"
+            formBody={
+              <>
+                <SelectorFormField
+                  name="connector_type"
+                  label="Connector Type:"
+                  options={[
+                    {
+                      name: "Entire Workspace",
+                      value: "workspace",
+                      description:
+                        "Recursively index all tasks from the entire workspace",
+                    },
+                    {
+                      name: "Space(s)",
+                      value: "space",
+                      description:
+                        "Index tasks only from the specified space id(s).",
+                    },
+                    {
+                      name: "Folder(s)",
+                      value: "folder",
+                      description:
+                        "Index tasks only from the specified folder id(s).",
+                    },
+                    {
+                      name: "List(s)",
+                      value: "list",
+                      description:
+                        "Index tasks only from the specified list id(s).",
+                    },
+                  ]}
+                />
+              </>
+            }
+            formBodyBuilder={(values) => {
+              return (
+                <>
+                  <Divider />
+                  {TextArrayFieldBuilder({
+                    name: "connector_ids",
+                    label: "ID(s):",
+                    subtext: "Specify 0 or more id(s) to index from.",
+                  })(values)}
+                  <BooleanFormField
+                    name="retrieve_task_comments"
+                    label="Retrieve Task Comments?"
+                    subtext={
+                      "If checked, then all the comments for each task will also be retrieved and indexed."
+                    }
+                  />
+                </>
+              );
+            }}
+            validationSchema={Yup.object().shape({
+              connector_type: Yup.string()
+                .oneOf(["workspace", "space", "folder", "list"])
+                .required("Please select the connector_type to index"),
+              connector_ids: Yup.array()
+                .of(Yup.string().required("ID(s) must be strings"))
+                .test(
+                  "idsRequired",
+                  "At least 1 ID is required if space, folder or list is selected",
+                  function (value) {
+                    if (this.parent.connector_type === "workspace") return true;
+                    else if (value !== undefined && value.length > 0)
+                      return true;
+                    setPopup({
+                      type: "error",
+                      message: `Add at least one ${this.parent.connector_type} ID`,
+                    });
+                    return false;
+                  }
+                ),
+              retrieve_task_comments: Yup.boolean().required(),
+            })}
+            initialValues={{
+              connector_type: "workspace",
+              connector_ids: [],
+              retrieve_task_comments: true,
+            }}
+            refreshFreq={10 * 60} // 10 minutes
+            credentialId={clickupCredential.id}
+          />
+        </Card>
+      ) : (
+        <Text>
+          Please provide your Clickup API token and Team ID in Step 1 first!
+          Once done with that, you can then specify whether you want to make the
+          entire workspace, or specified space(s), folder(s), list(s)
+          searchable.
+        </Text>
+      )}
+    </>
+  );
+};
+
+export default function Page() {
+  return (
+    <div className="mx-auto container">
+      <div className="mb-4">
+        <HealthCheckBanner />
+      </div>
+
+      <AdminPageTitle icon={<ClickupIcon size={32} />} title="Clickup" />
+
+      <MainSection />
+    </div>
+  );
+}

web/src/components/icons/icons.tsx

@@ -57,6 +57,7 @@ import teamsIcon from "../../../public/Teams.png";
 import mediawikiIcon from "../../../public/MediaWiki.svg";
 import wikipediaIcon from "../../../public/Wikipedia.svg";
 import discourseIcon from "../../../public/Discourse.png";
+import clickupIcon from "../../../public/Clickup.svg";
 import { FaRobot } from "react-icons/fa";
 
 interface IconProps {
@@ -654,6 +655,20 @@ export const AxeroIcon = ({
   </div>
 );
 
+export const ClickupIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return (
+    <div
+      style={{ width: `${size + 4}px`, height: `${size + 4}px` }}
+      className={`w-[${size + 4}px] h-[${size + 4}px] -m-0.5 ` + className}
+    >
+      <Image src={clickupIcon} alt="Logo" width="96" height="96" />
+    </div>
+  );
+};
+
 export const MediaWikiIcon = ({
   size = 16,
   className = defaultTailwindCSS,

web/src/lib/sources.ts

@@ -1,6 +1,7 @@
 import {
   AxeroIcon,
   BookstackIcon,
+  ClickupIcon,
   ConfluenceIcon,
   DiscourseIcon,
   Document360Icon,
@@ -195,6 +196,11 @@ const SOURCE_METADATA_MAP: SourceMap = {
     displayName: "Request Tracker",
     category: SourceCategory.AppConnection,
   },
+  clickup: {
+    icon: ClickupIcon,
+    displayName: "Clickup",
+    category: SourceCategory.AppConnection,
+  },
 };
 
 function fillSourceMetadata(

web/src/lib/types.ts

@@ -48,6 +48,8 @@ export type ValidSources =
   | "zendesk"
   | "discourse"
   | "axero"
+  | "clickup"
+  | "axero"
   | "wikipedia"
   | "mediawiki";
 
@@ -189,6 +191,12 @@ export interface Document360Config {
   categories?: string[];
 }
 
+export interface ClickupConfig {
+  connector_type: "list" | "folder" | "space" | "workspace";
+  connector_ids?: string[];
+  retrieve_task_comments: boolean;
+}
+
 export interface GoogleSitesConfig {
   zip_path: string;
   base_url: string;
@@ -364,6 +372,11 @@ export interface Document360CredentialJson {
   document360_api_token: string;
 }
 
+export interface ClickupCredentialJson {
+  clickup_api_token: string;
+  clickup_team_id: string;
+}
+
 export interface ZendeskCredentialJson {
   zendesk_subdomain: string;
   zendesk_email: string;