Make ticket fetching a seperate function from processing

backend/danswer/connectors/freshdesk/connector.py

@@ -1,6 +1,6 @@
 import requests
 import json
-from datetime import datetime, timezone
+from datetime import datetime
 from typing import Any, List, Optional
 from danswer.file_processing.html_utils import parse_html_page_basic
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
@@ -41,7 +41,7 @@ class FreshdeskConnector(PollConnector, LoadConnector):
         self.password = credentials.get("freshdesk_password")
         return None
     
-    def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
+    def _fetch_tickets(self) -> List[dict]:
         if any([self.api_key, self.domain, self.password]) is None:
             raise ConnectorMissingCredentialError("freshdesk")
         
@@ -52,16 +52,22 @@ class FreshdeskConnector(PollConnector, LoadConnector):
         if response.status_code!= 204:
             tickets = json.loads(response.content)
             logger.info(f"Fetched {len(tickets)} tickets from Freshdesk API")
+            return tickets
+        else:
+            return []
+
+    def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
+        tickets = self._fetch_tickets()
         doc_batch: List[Document] = []
 
         for ticket in tickets:
-                # Convert the "created_at", "updated_at", and "due_by" values to ISO 8601 strings
+            #convert to iso format
             for date_field in ["created_at", "updated_at", "due_by"]:
                 if ticket[date_field].endswith('Z'):
                     ticket[date_field] = ticket[date_field][:-1] + '+00:00'
                 ticket[date_field] = datetime.fromisoformat(ticket[date_field]).strftime("%Y-%m-%d %H:%M:%S")
 
-                # Convert all other values to strings
+            #convert all other values to strings
             ticket = {
                 key: str(value) if not isinstance(value, str) else value
                 for key, value in ticket.items()