Request Tracker Connector (#709)

Contributed by Evan! Thanks for the contribution!

- Minor linting and rebasing done by Yuhong, everything else from Evan

---------

Co-authored-by: Evan Sarmiento <e.sarmiento@soax.com>
Co-authored-by: Evan <esarmien@fas.harvard.edu>

.gitignore

@@ -1,3 +1,4 @@
 .env
 .DS_store
 .venv
+.mypy_cache

backend/.gitignore

@@ -1,4 +1,5 @@
 __pycache__/
+.mypy_cache
 .idea/
 site_crawls/
 .ipynb_checkpoints/

backend/danswer/configs/constants.py

@@ -41,6 +41,7 @@ class DocumentSource(str, Enum):
     SLACK = "slack"
     WEB = "web"
     GOOGLE_DRIVE = "google_drive"
+    REQUESTTRACKER = "requesttracker"
     GITHUB = "github"
     GURU = "guru"
     BOOKSTACK = "bookstack"

backend/danswer/connectors/factory.py

@@ -21,6 +21,7 @@ from danswer.connectors.linear.connector import LinearConnector
 from danswer.connectors.models import InputType
 from danswer.connectors.notion.connector import NotionConnector
 from danswer.connectors.productboard.connector import ProductboardConnector
+from danswer.connectors.requesttracker.connector import RequestTrackerConnector
 from danswer.connectors.slab.connector import SlabConnector
 from danswer.connectors.slack.connector import SlackLoadConnector
 from danswer.connectors.slack.connector import SlackPollConnector
@@ -53,6 +54,7 @@ def identify_connector_class(
         DocumentSource.SLAB: SlabConnector,
         DocumentSource.NOTION: NotionConnector,
         DocumentSource.ZULIP: ZulipConnector,
+        DocumentSource.REQUESTTRACKER: RequestTrackerConnector,
         DocumentSource.GURU: GuruConnector,
         DocumentSource.LINEAR: LinearConnector,
         DocumentSource.HUBSPOT: HubSpotConnector,

backend/danswer/connectors/requesttracker/.gitignore

@@ -0,0 +1 @@
+.env

backend/danswer/connectors/requesttracker/connector.py

@@ -0,0 +1,152 @@
+from datetime import datetime
+from datetime import timezone
+from logging import DEBUG as LOG_LVL_DEBUG
+from typing import Any
+from typing import List
+from typing import Optional
+
+from rt.rest1 import ALL_QUEUES
+from rt.rest1 import Rt
+
+from danswer.configs.app_configs import INDEX_BATCH_SIZE
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.interfaces import GenerateDocumentsOutput
+from danswer.connectors.interfaces import PollConnector
+from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+from danswer.connectors.models import ConnectorMissingCredentialError
+from danswer.connectors.models import Document
+from danswer.connectors.models import Section
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+class RequestTrackerError(Exception):
+    pass
+
+
+class RequestTrackerConnector(PollConnector):
+    def __init__(
+        self,
+        batch_size: int = INDEX_BATCH_SIZE,
+    ) -> None:
+        self.batch_size = batch_size
+
+    def txn_link(self, tid: int, txn: int) -> str:
+        return f"{self.rt_base_url}/Ticket/Display.html?id={tid}&txn={txn}"
+
+    def build_doc_sections_from_txn(
+        self, connection: Rt, ticket_id: int
+    ) -> List[Section]:
+        Sections: List[Section] = []
+
+        get_history_resp = connection.get_history(ticket_id)
+
+        if get_history_resp is None:
+            raise RequestTrackerError(f"Ticket {ticket_id} cannot be found")
+
+        for tx in get_history_resp:
+            Sections.append(
+                Section(
+                    link=self.txn_link(ticket_id, int(tx["id"])),
+                    text="\n".join(
+                        [
+                            f"{k}:\n{v}\n" if k != "Attachments" else ""
+                            for (k, v) in tx.items()
+                        ]
+                    ),
+                )
+            )
+        return Sections
+
+    def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]:
+        self.rt_username = credentials.get("requesttracker_username")
+        self.rt_password = credentials.get("requesttracker_password")
+        self.rt_base_url = credentials.get("requesttracker_base_url")
+        return None
+
+    # This does not include RT file attachments yet.
+    def _process_tickets(
+        self, start: datetime, end: datetime
+    ) -> GenerateDocumentsOutput:
+        if any([self.rt_username, self.rt_password, self.rt_base_url]) is None:
+            raise ConnectorMissingCredentialError("requesttracker")
+
+        Rt0 = Rt(
+            f"{self.rt_base_url}/REST/1.0/",
+            self.rt_username,
+            self.rt_password,
+        )
+
+        Rt0.login()
+
+        d0 = start.strftime("%Y-%m-%d %H:%M:%S")
+        d1 = end.strftime("%Y-%m-%d %H:%M:%S")
+
+        tickets = Rt0.search(
+            Queue=ALL_QUEUES,
+            raw_query=f"Updated > '{d0}' AND Updated < '{d1}'",
+        )
+
+        doc_batch: List[Document] = []
+
+        for ticket in tickets:
+            ticket_keys_to_omit = ["id", "Subject"]
+            tid: int = int(ticket["numerical_id"])
+            ticketLink: str = f"{self.rt_base_url}/Ticket/Display.html?id={tid}"
+            logger.info(f"Processing ticket {tid}")
+            doc = Document(
+                id=ticket["id"],
+                sections=[Section(link=ticketLink, text=f"{ticket['Subject']}\n")]
+                + self.build_doc_sections_from_txn(Rt0, tid),
+                source=DocumentSource.REQUESTTRACKER,
+                semantic_identifier=ticket["Subject"],
+                metadata={
+                    key: value
+                    for key, value in ticket.items()
+                    if key not in ticket_keys_to_omit
+                },
+            )
+
+            doc_batch.append(doc)
+
+            if len(doc_batch) >= self.batch_size:
+                yield doc_batch
+                doc_batch = []
+
+        if doc_batch:
+            yield doc_batch
+
+    def poll_source(
+        self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
+    ) -> GenerateDocumentsOutput:
+        # Keep query short, only look behind 1 day at maximum
+        one_day_ago: float = end - (24 * 60 * 60)
+        _start: float = start if start > one_day_ago else one_day_ago
+        start_datetime = datetime.fromtimestamp(_start, tz=timezone.utc)
+        end_datetime = datetime.fromtimestamp(end, tz=timezone.utc)
+        yield from self._process_tickets(start_datetime, end_datetime)
+
+
+if __name__ == "__main__":
+    import time
+    import os
+    from dotenv import load_dotenv
+
+    load_dotenv()
+    logger.setLevel(LOG_LVL_DEBUG)
+    rt_connector = RequestTrackerConnector()
+    rt_connector.load_credentials(
+        {
+            "requesttracker_username": os.getenv("RT_USERNAME"),
+            "requesttracker_password": os.getenv("RT_PASSWORD"),
+            "requesttracker_base_url": os.getenv("RT_BASE_URL"),
+        }
+    )
+
+    current = time.time()
+    one_day_ago = current - (24 * 60 * 60)  # 1 days
+    latest_docs = rt_connector.poll_source(one_day_ago, current)
+
+    for doc in latest_docs:
+        print(doc)

backend/requirements/default.txt

@@ -37,6 +37,7 @@ pydantic==1.10.7
 PyGithub==1.58.2
 pypdf==3.17.0
 pytest-playwright==0.3.2
+python-dotenv==1.0.0
 python-multipart==0.0.6
 requests==2.31.0
 requests-oauthlib==1.3.1
@@ -44,6 +45,7 @@ retry==0.9.2  # This pulls in py which is in CVE-2022-42969, must remove py from
 rfc3986==1.5.0
 # need to pin `safetensors` version, since the latest versions requires
 # building from source using Rust
+rt==3.1.2
 safetensors==0.3.1
 sentence-transformers==2.2.2
 slack-sdk==3.20.2

web/src/app/admin/connectors/requesttracker/page.tsx

@@ -0,0 +1,236 @@
+"use client";
+
+import * as Yup from "yup";
+import { TrashIcon, RequestTrackerIcon } from "@/components/icons/icons"; // Make sure you have a Document360 icon
+import { fetcher } from "@/lib/fetcher";
+import useSWR, { useSWRConfig } from "swr";
+import { LoadingAnimation } from "@/components/Loading";
+import { HealthCheckBanner } from "@/components/health/healthcheck";
+import {
+  RequestTrackerConfig,
+  RequestTrackerCredentialJson,
+  ConnectorIndexingStatus,
+  Credential,
+} from "@/lib/types"; // Modify or create these types as required
+import { adminDeleteCredential, linkCredential } from "@/lib/credential";
+import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
+import {
+  TextFormField,
+  TextArrayFieldBuilder,
+} from "@/components/admin/connectors/Field";
+import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
+import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
+import { usePublicCredentials } from "@/lib/hooks";
+
+const MainSection = () => {
+  const { mutate } = useSWRConfig();
+  const {
+    data: connectorIndexingStatuses,
+    isLoading: isConnectorIndexingStatusesLoading,
+    error: isConnectorIndexingStatusesError,
+  } = useSWR<ConnectorIndexingStatus<any, any>[]>(
+    "/api/manage/admin/connector/indexing-status",
+    fetcher
+  );
+
+  const {
+    data: credentialsData,
+    isLoading: isCredentialsLoading,
+    error: isCredentialsError,
+    refreshCredentials,
+  } = usePublicCredentials();
+
+  if (
+    (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) ||
+    (!credentialsData && isCredentialsLoading)
+  ) {
+    return <LoadingAnimation text="Loading" />;
+  }
+
+  if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) {
+    return <div>Failed to load connectors</div>;
+  }
+
+  if (isCredentialsError || !credentialsData) {
+    return <div>Failed to load credentials</div>;
+  }
+
+  const requestTrackerConnectorIndexingStatuses: ConnectorIndexingStatus<
+    RequestTrackerConfig,
+    RequestTrackerCredentialJson
+  >[] = connectorIndexingStatuses.filter(
+    (connectorIndexingStatus) =>
+      connectorIndexingStatus.connector.source === "requesttracker"
+  );
+
+  const requestTrackerCredential:
+    | Credential<RequestTrackerCredentialJson>
+    | undefined = credentialsData.find(
+    (credential) => credential.credential_json?.requesttracker_username
+  );
+
+  return (
+    <>
+      <h2 className="font-bold mb-2 mt-6 ml-auto mr-auto">
+        Step 1: Provide Request Tracker credentials
+      </h2>
+      {requestTrackerCredential ? (
+        <>
+          <div className="flex mb-1 text-sm">
+            <p className="my-auto">Existing Request Tracker username: </p>
+            <p className="ml-1 italic my-auto">
+              {requestTrackerCredential.credential_json.requesttracker_username}
+            </p>
+            <button
+              className="ml-1 hover:bg-gray-700 rounded-full p-1"
+              onClick={async () => {
+                await adminDeleteCredential(requestTrackerCredential.id);
+                refreshCredentials();
+              }}
+            >
+              <TrashIcon />
+            </button>
+          </div>
+        </>
+      ) : (
+        <>
+          <p className="text-sm mb-2">
+            To use the Request Tracker connector, provide a Request Tracker
+            username, password, and base url.
+          </p>
+          <p className="text-sm mb-2">
+            This connector currently supports{" "}
+            <a href="https://rt-wiki.bestpractical.com/wiki/REST">
+              Request Tracker REST API 1.0
+            </a>
+            ,{" "}
+            <b>not the latest REST API 2.0 introduced in Request Tracker 5.0</b>
+            .
+          </p>
+          <div className="border-solid border-gray-600 border rounded-md p-6 mt-2">
+            <CredentialForm<RequestTrackerCredentialJson>
+              formBody={
+                <>
+                  <TextFormField
+                    name="requesttracker_username"
+                    label="Request Tracker username:"
+                  />
+                  <TextFormField
+                    name="requesttracker_password"
+                    label="Request Tracker password:"
+                    type="password"
+                  />
+                  <TextFormField
+                    name="requesttracker_base_url"
+                    label="Request Tracker base url:"
+                  />
+                </>
+              }
+              validationSchema={Yup.object().shape({
+                requesttracker_username: Yup.string().required(
+                  "Please enter your Request Tracker username"
+                ),
+                requesttracker_password: Yup.string().required(
+                  "Please enter your Request Tracker password"
+                ),
+                requesttracker_base_url: Yup.string()
+                  .url()
+                  .required(
+                    "Please enter the base url of your RT installation"
+                  ),
+              })}
+              initialValues={{
+                requesttracker_username: "",
+                requesttracker_password: "",
+                requesttracker_base_url: "",
+              }}
+              onSubmit={(isSuccess) => {
+                if (isSuccess) {
+                  refreshCredentials();
+                }
+              }}
+            />
+          </div>
+        </>
+      )}
+
+      {requestTrackerConnectorIndexingStatuses.length > 0 && (
+        <>
+          <p className="text-sm mb-2">
+            We index the most recently updated tickets from each Request Tracker
+            instance listed below regularly.
+          </p>
+          <p className="text-sm mb-2">
+            The initial poll at this time retrieves tickets updated in the past
+            hour. All subsequent polls execute every ten minutes. This should be
+            configurable in the future.
+          </p>
+          <div className="mb-2">
+            <ConnectorsTable<RequestTrackerConfig, RequestTrackerCredentialJson>
+              connectorIndexingStatuses={
+                requestTrackerConnectorIndexingStatuses
+              }
+              liveCredential={requestTrackerCredential}
+              getCredential={(credential) =>
+                credential.credential_json.requesttracker_base_url
+              }
+              onUpdate={() =>
+                mutate("/api/manage/admin/connector/indexing-status")
+              }
+              onCredentialLink={async (connectorId) => {
+                if (requestTrackerCredential) {
+                  await linkCredential(
+                    connectorId,
+                    requestTrackerCredential.id
+                  );
+                  mutate("/api/manage/admin/connector/indexing-status");
+                }
+              }}
+            />
+          </div>
+        </>
+      )}
+
+      {requestTrackerCredential &&
+      requestTrackerConnectorIndexingStatuses.length === 0 ? (
+        <div className="border-solid border-gray-600 border rounded-md p-6 mt-4">
+          <h2 className="font-bold mb-3">
+            Step 2: (Re)initialize connection to Request Tracker installation
+          </h2>
+          <ConnectorForm<RequestTrackerConfig>
+            nameBuilder={(values) =>
+              `RequestTracker-${requestTrackerCredential.credential_json.requesttracker_base_url}`
+            }
+            ccPairNameBuilder={(values) =>
+              `Request Tracker ${requestTrackerCredential.credential_json.requesttracker_base_url}`
+            }
+            source="requesttracker"
+            inputType="poll"
+            validationSchema={Yup.object().shape({})}
+            formBody={<></>}
+            initialValues={{}}
+            credentialId={requestTrackerCredential.id}
+            refreshFreq={10 * 60} // 10 minutes
+          />
+        </div>
+      ) : (
+        <></>
+      )}
+    </>
+  );
+};
+
+export default function Page() {
+  return (
+    <div className="mx-auto container">
+      <div className="mb-4">
+        <HealthCheckBanner />
+      </div>
+      <div className="border-solid border-gray-600 border-b mb-4 pb-2 flex">
+        <RequestTrackerIcon size={32} />
+        <h1 className="text-3xl font-bold pl-2">Request Tracker</h1>
+      </div>
+      <MainSection />
+    </div>
+  );
+}

web/src/components/admin/Layout.tsx

@@ -23,6 +23,7 @@ import {
   BookmarkIcon,
   CPUIcon,
   Document360Icon,
+  RequestTrackerIcon,
   GoogleSitesIcon,
   GongIcon,
   ZoomInIcon,
@@ -223,6 +224,15 @@ export async function Layout({ children }: { children: React.ReactNode }) {
                   ),
                   link: "/admin/connectors/hubspot",
                 },
+                {
+                  name: (
+                    <div className="flex">
+                      <RequestTrackerIcon size={16} />
+                      <div className="ml-1">Request Tracker</div>
+                    </div>
+                  ),
+                  link: "/admin/connectors/requesttracker",
+                },
                 {
                   name: (
                     <div className="flex">

web/src/components/icons/icons.tsx

@@ -40,6 +40,7 @@ import jiraSVG from "../../../public/Jira.svg";
 import confluenceSVG from "../../../public/Confluence.svg";
 import guruIcon from "../../../public/Guru.svg";
 import gongIcon from "../../../public/Gong.png";
+import requestTrackerIcon from "../../../public/RequestTracker.png";
 import zulipIcon from "../../../public/Zulip.png";
 import linearIcon from "../../../public/Linear.png";
 import hubSpotIcon from "../../../public/HubSpot.png";
@@ -427,6 +428,18 @@ export const GuruIcon = ({
   </div>
 );
 
+export const RequestTrackerIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => (
+  <div
+    style={{ width: `${size}px`, height: `${size}px` }}
+    className={`w-[${size}px] h-[${size}px] ` + className}
+  >
+    <Image src={requestTrackerIcon} alt="Logo" width="96" height="96" />
+  </div>
+);
+
 export const GongIcon = ({
   size = 16,
   className = defaultTailwindCSS,

web/src/components/search/Filters.tsx

@@ -26,6 +26,7 @@ const sources: Source[] = [
   { displayName: "Linear", internalName: "linear" },
   { displayName: "HubSpot", internalName: "hubspot" },
   { displayName: "Document360", internalName: "document360" },
+  { displayName: "Request Tracker", internalName: "requesttracker" },
   { displayName: "Google Sites", internalName: "google_sites" },
 ];
 

web/src/components/source.tsx

@@ -18,6 +18,7 @@ import {
   HubSpotIcon,
   Document360Icon,
   GoogleSitesIcon,
+  RequestTrackerIcon,
   ZendeskIcon,
 } from "./icons/icons";
 
@@ -131,6 +132,12 @@ export const getSourceMetadata = (sourceType: ValidSources): SourceMetadata => {
         displayName: "Document360",
         adminPageLink: "/admin/connectors/document360",
       };
+    case "requesttracker":
+      return {
+        icon: RequestTrackerIcon,
+        displayName: "Request Tracker",
+        adminPageLink: "/admin/connectors/requesttracker",
+      };
     case "google_sites":
       return {
         icon: GoogleSitesIcon,

web/src/lib/types.ts

@@ -24,6 +24,7 @@ export type ValidSources =
   | "linear"
   | "hubspot"
   | "document360"
+  | "requesttracker"
   | "file"
   | "google_sites"
   | "zendesk";
@@ -119,6 +120,8 @@ export interface NotionConfig {}
 
 export interface HubSpotConfig {}
 
+export interface RequestTrackerConfig {}
+
 export interface Document360Config {
   workspace: string;
   categories?: string[];
@@ -240,6 +243,12 @@ export interface HubSpotCredentialJson {
   hubspot_access_token: string;
 }
 
+export interface RequestTrackerCredentialJson {
+  requesttracker_username: string;
+  requesttracker_password: string;
+  requesttracker_base_url: string;
+}
+
 export interface Document360CredentialJson {
   portal_id: string;
   document360_api_token: string;