Request Tracker Connector (#709)

Contributed by Evan! Thanks for the contribution!

- Minor linting and rebasing done by Yuhong, everything else from Evan

---------

Co-authored-by: Evan Sarmiento <e.sarmiento@soax.com>
Co-authored-by: Evan <esarmien@fas.harvard.edu>

backend/.gitignore

@@ -1,4 +1,5 @@
 __pycache__/
+.mypy_cache
 .idea/
 site_crawls/
 .ipynb_checkpoints/

backend/danswer/configs/constants.py

@@ -41,6 +41,7 @@ class DocumentSource(str, Enum):
     SLACK = "slack"
     WEB = "web"
     GOOGLE_DRIVE = "google_drive"
+    REQUESTTRACKER = "requesttracker"
     GITHUB = "github"
     GURU = "guru"
     BOOKSTACK = "bookstack"

backend/danswer/connectors/factory.py

@@ -21,6 +21,7 @@ from danswer.connectors.linear.connector import LinearConnector
 from danswer.connectors.models import InputType
 from danswer.connectors.notion.connector import NotionConnector
 from danswer.connectors.productboard.connector import ProductboardConnector
+from danswer.connectors.requesttracker.connector import RequestTrackerConnector
 from danswer.connectors.slab.connector import SlabConnector
 from danswer.connectors.slack.connector import SlackLoadConnector
 from danswer.connectors.slack.connector import SlackPollConnector
@@ -53,6 +54,7 @@ def identify_connector_class(
         DocumentSource.SLAB: SlabConnector,
         DocumentSource.NOTION: NotionConnector,
         DocumentSource.ZULIP: ZulipConnector,
+        DocumentSource.REQUESTTRACKER: RequestTrackerConnector,
         DocumentSource.GURU: GuruConnector,
         DocumentSource.LINEAR: LinearConnector,
         DocumentSource.HUBSPOT: HubSpotConnector,

backend/danswer/connectors/requesttracker/.gitignore

@@ -0,0 +1 @@
+.env

backend/danswer/connectors/requesttracker/connector.py

@@ -0,0 +1,152 @@
+from datetime import datetime
+from datetime import timezone
+from logging import DEBUG as LOG_LVL_DEBUG
+from typing import Any
+from typing import List
+from typing import Optional
+
+from rt.rest1 import ALL_QUEUES
+from rt.rest1 import Rt
+
+from danswer.configs.app_configs import INDEX_BATCH_SIZE
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.interfaces import GenerateDocumentsOutput
+from danswer.connectors.interfaces import PollConnector
+from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+from danswer.connectors.models import ConnectorMissingCredentialError
+from danswer.connectors.models import Document
+from danswer.connectors.models import Section
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+class RequestTrackerError(Exception):
+    pass
+
+
+class RequestTrackerConnector(PollConnector):
+    def __init__(
+        self,
+        batch_size: int = INDEX_BATCH_SIZE,
+    ) -> None:
+        self.batch_size = batch_size
+
+    def txn_link(self, tid: int, txn: int) -> str:
+        return f"{self.rt_base_url}/Ticket/Display.html?id={tid}&txn={txn}"
+
+    def build_doc_sections_from_txn(
+        self, connection: Rt, ticket_id: int
+    ) -> List[Section]:
+        Sections: List[Section] = []
+
+        get_history_resp = connection.get_history(ticket_id)
+
+        if get_history_resp is None:
+            raise RequestTrackerError(f"Ticket {ticket_id} cannot be found")
+
+        for tx in get_history_resp:
+            Sections.append(
+                Section(
+                    link=self.txn_link(ticket_id, int(tx["id"])),
+                    text="\n".join(
+                        [
+                            f"{k}:\n{v}\n" if k != "Attachments" else ""
+                            for (k, v) in tx.items()
+                        ]
+                    ),
+                )
+            )
+        return Sections
+
+    def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]:
+        self.rt_username = credentials.get("requesttracker_username")
+        self.rt_password = credentials.get("requesttracker_password")
+        self.rt_base_url = credentials.get("requesttracker_base_url")
+        return None
+
+    # This does not include RT file attachments yet.
+    def _process_tickets(
+        self, start: datetime, end: datetime
+    ) -> GenerateDocumentsOutput:
+        if any([self.rt_username, self.rt_password, self.rt_base_url]) is None:
+            raise ConnectorMissingCredentialError("requesttracker")
+
+        Rt0 = Rt(
+            f"{self.rt_base_url}/REST/1.0/",
+            self.rt_username,
+            self.rt_password,
+        )
+
+        Rt0.login()
+
+        d0 = start.strftime("%Y-%m-%d %H:%M:%S")
+        d1 = end.strftime("%Y-%m-%d %H:%M:%S")
+
+        tickets = Rt0.search(
+            Queue=ALL_QUEUES,
+            raw_query=f"Updated > '{d0}' AND Updated < '{d1}'",
+        )
+
+        doc_batch: List[Document] = []
+
+        for ticket in tickets:
+            ticket_keys_to_omit = ["id", "Subject"]
+            tid: int = int(ticket["numerical_id"])
+            ticketLink: str = f"{self.rt_base_url}/Ticket/Display.html?id={tid}"
+            logger.info(f"Processing ticket {tid}")
+            doc = Document(
+                id=ticket["id"],
+                sections=[Section(link=ticketLink, text=f"{ticket['Subject']}\n")]
+                + self.build_doc_sections_from_txn(Rt0, tid),
+                source=DocumentSource.REQUESTTRACKER,
+                semantic_identifier=ticket["Subject"],
+                metadata={
+                    key: value
+                    for key, value in ticket.items()
+                    if key not in ticket_keys_to_omit
+                },
+            )
+
+            doc_batch.append(doc)
+
+            if len(doc_batch) >= self.batch_size:
+                yield doc_batch
+                doc_batch = []
+
+        if doc_batch:
+            yield doc_batch
+
+    def poll_source(
+        self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
+    ) -> GenerateDocumentsOutput:
+        # Keep query short, only look behind 1 day at maximum
+        one_day_ago: float = end - (24 * 60 * 60)
+        _start: float = start if start > one_day_ago else one_day_ago
+        start_datetime = datetime.fromtimestamp(_start, tz=timezone.utc)
+        end_datetime = datetime.fromtimestamp(end, tz=timezone.utc)
+        yield from self._process_tickets(start_datetime, end_datetime)
+
+
+if __name__ == "__main__":
+    import time
+    import os
+    from dotenv import load_dotenv
+
+    load_dotenv()
+    logger.setLevel(LOG_LVL_DEBUG)
+    rt_connector = RequestTrackerConnector()
+    rt_connector.load_credentials(
+        {
+            "requesttracker_username": os.getenv("RT_USERNAME"),
+            "requesttracker_password": os.getenv("RT_PASSWORD"),
+            "requesttracker_base_url": os.getenv("RT_BASE_URL"),
+        }
+    )
+
+    current = time.time()
+    one_day_ago = current - (24 * 60 * 60)  # 1 days
+    latest_docs = rt_connector.poll_source(one_day_ago, current)
+
+    for doc in latest_docs:
+        print(doc)

backend/requirements/default.txt

@@ -37,6 +37,7 @@ pydantic==1.10.7
 PyGithub==1.58.2
 pypdf==3.17.0
 pytest-playwright==0.3.2
+python-dotenv==1.0.0
 python-multipart==0.0.6
 requests==2.31.0
 requests-oauthlib==1.3.1
@@ -44,6 +45,7 @@ retry==0.9.2  # This pulls in py which is in CVE-2022-42969, must remove py from
 rfc3986==1.5.0
 # need to pin `safetensors` version, since the latest versions requires
 # building from source using Rust
+rt==3.1.2
 safetensors==0.3.1
 sentence-transformers==2.2.2
 slack-sdk==3.20.2