diff --git a/.gitignore b/.gitignore index b582ec01c6..ebda0a7c83 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .env .DS_store -.venv \ No newline at end of file +.venv +.mypy_cache diff --git a/backend/.gitignore b/backend/.gitignore index ead9ceb200..b365a5c894 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -1,4 +1,5 @@ __pycache__/ +.mypy_cache .idea/ site_crawls/ .ipynb_checkpoints/ diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py index 46752517a5..23362a5763 100644 --- a/backend/danswer/configs/constants.py +++ b/backend/danswer/configs/constants.py @@ -41,6 +41,7 @@ class DocumentSource(str, Enum): SLACK = "slack" WEB = "web" GOOGLE_DRIVE = "google_drive" + REQUESTTRACKER = "requesttracker" GITHUB = "github" GURU = "guru" BOOKSTACK = "bookstack" diff --git a/backend/danswer/connectors/factory.py b/backend/danswer/connectors/factory.py index 102e29470a..c08e88c974 100644 --- a/backend/danswer/connectors/factory.py +++ b/backend/danswer/connectors/factory.py @@ -21,6 +21,7 @@ from danswer.connectors.linear.connector import LinearConnector from danswer.connectors.models import InputType from danswer.connectors.notion.connector import NotionConnector from danswer.connectors.productboard.connector import ProductboardConnector +from danswer.connectors.requesttracker.connector import RequestTrackerConnector from danswer.connectors.slab.connector import SlabConnector from danswer.connectors.slack.connector import SlackLoadConnector from danswer.connectors.slack.connector import SlackPollConnector @@ -53,6 +54,7 @@ def identify_connector_class( DocumentSource.SLAB: SlabConnector, DocumentSource.NOTION: NotionConnector, DocumentSource.ZULIP: ZulipConnector, + DocumentSource.REQUESTTRACKER: RequestTrackerConnector, DocumentSource.GURU: GuruConnector, DocumentSource.LINEAR: LinearConnector, DocumentSource.HUBSPOT: HubSpotConnector, diff --git a/backend/danswer/connectors/requesttracker/.gitignore b/backend/danswer/connectors/requesttracker/.gitignore new file mode 100644 index 0000000000..4c49bd78f1 --- /dev/null +++ b/backend/danswer/connectors/requesttracker/.gitignore @@ -0,0 +1 @@ +.env diff --git a/backend/danswer/connectors/requesttracker/__init__.py b/backend/danswer/connectors/requesttracker/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/danswer/connectors/requesttracker/connector.py b/backend/danswer/connectors/requesttracker/connector.py new file mode 100644 index 0000000000..6b20504d32 --- /dev/null +++ b/backend/danswer/connectors/requesttracker/connector.py @@ -0,0 +1,152 @@ +from datetime import datetime +from datetime import timezone +from logging import DEBUG as LOG_LVL_DEBUG +from typing import Any +from typing import List +from typing import Optional + +from rt.rest1 import ALL_QUEUES +from rt.rest1 import Rt + +from danswer.configs.app_configs import INDEX_BATCH_SIZE +from danswer.configs.constants import DocumentSource +from danswer.connectors.interfaces import GenerateDocumentsOutput +from danswer.connectors.interfaces import PollConnector +from danswer.connectors.interfaces import SecondsSinceUnixEpoch +from danswer.connectors.models import ConnectorMissingCredentialError +from danswer.connectors.models import Document +from danswer.connectors.models import Section +from danswer.utils.logger import setup_logger + +logger = setup_logger() + + +class RequestTrackerError(Exception): + pass + + +class RequestTrackerConnector(PollConnector): + def __init__( + self, + batch_size: int = INDEX_BATCH_SIZE, + ) -> None: + self.batch_size = batch_size + + def txn_link(self, tid: int, txn: int) -> str: + return f"{self.rt_base_url}/Ticket/Display.html?id={tid}&txn={txn}" + + def build_doc_sections_from_txn( + self, connection: Rt, ticket_id: int + ) -> List[Section]: + Sections: List[Section] = [] + + get_history_resp = connection.get_history(ticket_id) + + if get_history_resp is None: + raise RequestTrackerError(f"Ticket {ticket_id} cannot be found") + + for tx in get_history_resp: + Sections.append( + Section( + link=self.txn_link(ticket_id, int(tx["id"])), + text="\n".join( + [ + f"{k}:\n{v}\n" if k != "Attachments" else "" + for (k, v) in tx.items() + ] + ), + ) + ) + return Sections + + def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]: + self.rt_username = credentials.get("requesttracker_username") + self.rt_password = credentials.get("requesttracker_password") + self.rt_base_url = credentials.get("requesttracker_base_url") + return None + + # This does not include RT file attachments yet. + def _process_tickets( + self, start: datetime, end: datetime + ) -> GenerateDocumentsOutput: + if any([self.rt_username, self.rt_password, self.rt_base_url]) is None: + raise ConnectorMissingCredentialError("requesttracker") + + Rt0 = Rt( + f"{self.rt_base_url}/REST/1.0/", + self.rt_username, + self.rt_password, + ) + + Rt0.login() + + d0 = start.strftime("%Y-%m-%d %H:%M:%S") + d1 = end.strftime("%Y-%m-%d %H:%M:%S") + + tickets = Rt0.search( + Queue=ALL_QUEUES, + raw_query=f"Updated > '{d0}' AND Updated < '{d1}'", + ) + + doc_batch: List[Document] = [] + + for ticket in tickets: + ticket_keys_to_omit = ["id", "Subject"] + tid: int = int(ticket["numerical_id"]) + ticketLink: str = f"{self.rt_base_url}/Ticket/Display.html?id={tid}" + logger.info(f"Processing ticket {tid}") + doc = Document( + id=ticket["id"], + sections=[Section(link=ticketLink, text=f"{ticket['Subject']}\n")] + + self.build_doc_sections_from_txn(Rt0, tid), + source=DocumentSource.REQUESTTRACKER, + semantic_identifier=ticket["Subject"], + metadata={ + key: value + for key, value in ticket.items() + if key not in ticket_keys_to_omit + }, + ) + + doc_batch.append(doc) + + if len(doc_batch) >= self.batch_size: + yield doc_batch + doc_batch = [] + + if doc_batch: + yield doc_batch + + def poll_source( + self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch + ) -> GenerateDocumentsOutput: + # Keep query short, only look behind 1 day at maximum + one_day_ago: float = end - (24 * 60 * 60) + _start: float = start if start > one_day_ago else one_day_ago + start_datetime = datetime.fromtimestamp(_start, tz=timezone.utc) + end_datetime = datetime.fromtimestamp(end, tz=timezone.utc) + yield from self._process_tickets(start_datetime, end_datetime) + + +if __name__ == "__main__": + import time + import os + from dotenv import load_dotenv + + load_dotenv() + logger.setLevel(LOG_LVL_DEBUG) + rt_connector = RequestTrackerConnector() + rt_connector.load_credentials( + { + "requesttracker_username": os.getenv("RT_USERNAME"), + "requesttracker_password": os.getenv("RT_PASSWORD"), + "requesttracker_base_url": os.getenv("RT_BASE_URL"), + } + ) + + current = time.time() + one_day_ago = current - (24 * 60 * 60) # 1 days + latest_docs = rt_connector.poll_source(one_day_ago, current) + + for doc in latest_docs: + print(doc) diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt index 08f93f5663..9c7be4ce64 100644 --- a/backend/requirements/default.txt +++ b/backend/requirements/default.txt @@ -37,6 +37,7 @@ pydantic==1.10.7 PyGithub==1.58.2 pypdf==3.17.0 pytest-playwright==0.3.2 +python-dotenv==1.0.0 python-multipart==0.0.6 requests==2.31.0 requests-oauthlib==1.3.1 @@ -44,6 +45,7 @@ retry==0.9.2 # This pulls in py which is in CVE-2022-42969, must remove py from rfc3986==1.5.0 # need to pin `safetensors` version, since the latest versions requires # building from source using Rust +rt==3.1.2 safetensors==0.3.1 sentence-transformers==2.2.2 slack-sdk==3.20.2 diff --git a/web/public/RequestTracker.png b/web/public/RequestTracker.png new file mode 100644 index 0000000000..95d6680e95 Binary files /dev/null and b/web/public/RequestTracker.png differ diff --git a/web/src/app/admin/connectors/requesttracker/page.tsx b/web/src/app/admin/connectors/requesttracker/page.tsx new file mode 100644 index 0000000000..32575e11c6 --- /dev/null +++ b/web/src/app/admin/connectors/requesttracker/page.tsx @@ -0,0 +1,236 @@ +"use client"; + +import * as Yup from "yup"; +import { TrashIcon, RequestTrackerIcon } from "@/components/icons/icons"; // Make sure you have a Document360 icon +import { fetcher } from "@/lib/fetcher"; +import useSWR, { useSWRConfig } from "swr"; +import { LoadingAnimation } from "@/components/Loading"; +import { HealthCheckBanner } from "@/components/health/healthcheck"; +import { + RequestTrackerConfig, + RequestTrackerCredentialJson, + ConnectorIndexingStatus, + Credential, +} from "@/lib/types"; // Modify or create these types as required +import { adminDeleteCredential, linkCredential } from "@/lib/credential"; +import { CredentialForm } from "@/components/admin/connectors/CredentialForm"; +import { + TextFormField, + TextArrayFieldBuilder, +} from "@/components/admin/connectors/Field"; +import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable"; +import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; +import { usePublicCredentials } from "@/lib/hooks"; + +const MainSection = () => { + const { mutate } = useSWRConfig(); + const { + data: connectorIndexingStatuses, + isLoading: isConnectorIndexingStatusesLoading, + error: isConnectorIndexingStatusesError, + } = useSWR[]>( + "/api/manage/admin/connector/indexing-status", + fetcher + ); + + const { + data: credentialsData, + isLoading: isCredentialsLoading, + error: isCredentialsError, + refreshCredentials, + } = usePublicCredentials(); + + if ( + (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) || + (!credentialsData && isCredentialsLoading) + ) { + return ; + } + + if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) { + return
Failed to load connectors
; + } + + if (isCredentialsError || !credentialsData) { + return
Failed to load credentials
; + } + + const requestTrackerConnectorIndexingStatuses: ConnectorIndexingStatus< + RequestTrackerConfig, + RequestTrackerCredentialJson + >[] = connectorIndexingStatuses.filter( + (connectorIndexingStatus) => + connectorIndexingStatus.connector.source === "requesttracker" + ); + + const requestTrackerCredential: + | Credential + | undefined = credentialsData.find( + (credential) => credential.credential_json?.requesttracker_username + ); + + return ( + <> +

+ Step 1: Provide Request Tracker credentials +

+ {requestTrackerCredential ? ( + <> +
+

Existing Request Tracker username:

+

+ {requestTrackerCredential.credential_json.requesttracker_username} +

+ +
+ + ) : ( + <> +

+ To use the Request Tracker connector, provide a Request Tracker + username, password, and base url. +

+

+ This connector currently supports{" "} + + Request Tracker REST API 1.0 + + ,{" "} + not the latest REST API 2.0 introduced in Request Tracker 5.0 + . +

+
+ + formBody={ + <> + + + + + } + validationSchema={Yup.object().shape({ + requesttracker_username: Yup.string().required( + "Please enter your Request Tracker username" + ), + requesttracker_password: Yup.string().required( + "Please enter your Request Tracker password" + ), + requesttracker_base_url: Yup.string() + .url() + .required( + "Please enter the base url of your RT installation" + ), + })} + initialValues={{ + requesttracker_username: "", + requesttracker_password: "", + requesttracker_base_url: "", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + refreshCredentials(); + } + }} + /> +
+ + )} + + {requestTrackerConnectorIndexingStatuses.length > 0 && ( + <> +

+ We index the most recently updated tickets from each Request Tracker + instance listed below regularly. +

+

+ The initial poll at this time retrieves tickets updated in the past + hour. All subsequent polls execute every ten minutes. This should be + configurable in the future. +

+
+ + connectorIndexingStatuses={ + requestTrackerConnectorIndexingStatuses + } + liveCredential={requestTrackerCredential} + getCredential={(credential) => + credential.credential_json.requesttracker_base_url + } + onUpdate={() => + mutate("/api/manage/admin/connector/indexing-status") + } + onCredentialLink={async (connectorId) => { + if (requestTrackerCredential) { + await linkCredential( + connectorId, + requestTrackerCredential.id + ); + mutate("/api/manage/admin/connector/indexing-status"); + } + }} + /> +
+ + )} + + {requestTrackerCredential && + requestTrackerConnectorIndexingStatuses.length === 0 ? ( +
+

+ Step 2: (Re)initialize connection to Request Tracker installation +

+ + nameBuilder={(values) => + `RequestTracker-${requestTrackerCredential.credential_json.requesttracker_base_url}` + } + ccPairNameBuilder={(values) => + `Request Tracker ${requestTrackerCredential.credential_json.requesttracker_base_url}` + } + source="requesttracker" + inputType="poll" + validationSchema={Yup.object().shape({})} + formBody={<>} + initialValues={{}} + credentialId={requestTrackerCredential.id} + refreshFreq={10 * 60} // 10 minutes + /> +
+ ) : ( + <> + )} + + ); +}; + +export default function Page() { + return ( +
+
+ +
+
+ +

Request Tracker

+
+ +
+ ); +} diff --git a/web/src/components/admin/Layout.tsx b/web/src/components/admin/Layout.tsx index 7000ede1cc..d13a5928fb 100644 --- a/web/src/components/admin/Layout.tsx +++ b/web/src/components/admin/Layout.tsx @@ -23,6 +23,7 @@ import { BookmarkIcon, CPUIcon, Document360Icon, + RequestTrackerIcon, GoogleSitesIcon, GongIcon, ZoomInIcon, @@ -223,6 +224,15 @@ export async function Layout({ children }: { children: React.ReactNode }) { ), link: "/admin/connectors/hubspot", }, + { + name: ( +
+ +
Request Tracker
+
+ ), + link: "/admin/connectors/requesttracker", + }, { name: (
diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx index 3d62cd7bd7..a8f90438f3 100644 --- a/web/src/components/icons/icons.tsx +++ b/web/src/components/icons/icons.tsx @@ -40,6 +40,7 @@ import jiraSVG from "../../../public/Jira.svg"; import confluenceSVG from "../../../public/Confluence.svg"; import guruIcon from "../../../public/Guru.svg"; import gongIcon from "../../../public/Gong.png"; +import requestTrackerIcon from "../../../public/RequestTracker.png"; import zulipIcon from "../../../public/Zulip.png"; import linearIcon from "../../../public/Linear.png"; import hubSpotIcon from "../../../public/HubSpot.png"; @@ -427,6 +428,18 @@ export const GuruIcon = ({
); +export const RequestTrackerIcon = ({ + size = 16, + className = defaultTailwindCSS, +}: IconProps) => ( +
+ Logo +
+); + export const GongIcon = ({ size = 16, className = defaultTailwindCSS, diff --git a/web/src/components/search/Filters.tsx b/web/src/components/search/Filters.tsx index fa9596cffd..4ab2866a9b 100644 --- a/web/src/components/search/Filters.tsx +++ b/web/src/components/search/Filters.tsx @@ -26,6 +26,7 @@ const sources: Source[] = [ { displayName: "Linear", internalName: "linear" }, { displayName: "HubSpot", internalName: "hubspot" }, { displayName: "Document360", internalName: "document360" }, + { displayName: "Request Tracker", internalName: "requesttracker" }, { displayName: "Google Sites", internalName: "google_sites" }, ]; diff --git a/web/src/components/source.tsx b/web/src/components/source.tsx index 7162512ea4..bd7a0dd7a0 100644 --- a/web/src/components/source.tsx +++ b/web/src/components/source.tsx @@ -18,6 +18,7 @@ import { HubSpotIcon, Document360Icon, GoogleSitesIcon, + RequestTrackerIcon, ZendeskIcon, } from "./icons/icons"; @@ -131,6 +132,12 @@ export const getSourceMetadata = (sourceType: ValidSources): SourceMetadata => { displayName: "Document360", adminPageLink: "/admin/connectors/document360", }; + case "requesttracker": + return { + icon: RequestTrackerIcon, + displayName: "Request Tracker", + adminPageLink: "/admin/connectors/requesttracker", + }; case "google_sites": return { icon: GoogleSitesIcon, diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 028c9c2530..f692424c18 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -24,6 +24,7 @@ export type ValidSources = | "linear" | "hubspot" | "document360" + | "requesttracker" | "file" | "google_sites" | "zendesk"; @@ -119,6 +120,8 @@ export interface NotionConfig {} export interface HubSpotConfig {} +export interface RequestTrackerConfig {} + export interface Document360Config { workspace: string; categories?: string[]; @@ -240,6 +243,12 @@ export interface HubSpotCredentialJson { hubspot_access_token: string; } +export interface RequestTrackerCredentialJson { + requesttracker_username: string; + requesttracker_password: string; + requesttracker_base_url: string; +} + export interface Document360CredentialJson { portal_id: string; document360_api_token: string;