Add option to index all Jira projects (#4106)

* Add option to index all Jira projects

* Fix test

* Fix web build

* Address comment

backend/onyx/connectors/onyx_jira/connector.py

@@ -29,7 +29,6 @@ from onyx.connectors.onyx_jira.utils import best_effort_basic_expert_info
 from onyx.connectors.onyx_jira.utils import best_effort_get_field_from_issue
 from onyx.connectors.onyx_jira.utils import build_jira_client
 from onyx.connectors.onyx_jira.utils import build_jira_url
-from onyx.connectors.onyx_jira.utils import extract_jira_project
 from onyx.connectors.onyx_jira.utils import extract_text_from_adf
 from onyx.connectors.onyx_jira.utils import get_comment_strs
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
@@ -160,7 +159,8 @@ def fetch_jira_issues_batch(
 class JiraConnector(LoadConnector, PollConnector, SlimConnector):
     def __init__(
         self,
-        jira_project_url: str,
+        jira_base_url: str,
+        project_key: str | None = None,
         comment_email_blacklist: list[str] | None = None,
         batch_size: int = INDEX_BATCH_SIZE,
         # if a ticket has one of the labels specified in this list, we will just
@@ -169,12 +169,13 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
         labels_to_skip: list[str] = JIRA_CONNECTOR_LABELS_TO_SKIP,
     ) -> None:
         self.batch_size = batch_size
-        self.jira_base, self._jira_project = extract_jira_project(jira_project_url)
-        self._jira_client: JIRA | None = None
+        self.jira_base = jira_base_url.rstrip("/")  # Remove trailing slash if present
+        self.jira_project = project_key
         self._comment_email_blacklist = comment_email_blacklist or []
-
         self.labels_to_skip = set(labels_to_skip)
 
+        self._jira_client: JIRA | None = None
+
     @property
     def comment_email_blacklist(self) -> tuple:
         return tuple(email.strip() for email in self._comment_email_blacklist)
@@ -188,7 +189,9 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
     @property
     def quoted_jira_project(self) -> str:
         # Quote the project name to handle reserved words
-        return f'"{self._jira_project}"'
+        if not self.jira_project:
+            return ""
+        return f'"{self.jira_project}"'
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         self._jira_client = build_jira_client(
@@ -197,8 +200,14 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
         )
         return None
 
+    def _get_jql_query(self) -> str:
+        """Get the JQL query based on whether a specific project is set"""
+        if self.jira_project:
+            return f"project = {self.quoted_jira_project}"
+        return ""  # Empty string means all accessible projects
+
     def load_from_state(self) -> GenerateDocumentsOutput:
-        jql = f"project = {self.quoted_jira_project}"
+        jql = self._get_jql_query()
 
         document_batch = []
         for doc in fetch_jira_issues_batch(
@@ -225,11 +234,10 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
             "%Y-%m-%d %H:%M"
         )
 
+        base_jql = self._get_jql_query()
         jql = (
-            f"project = {self.quoted_jira_project} AND "
-            f"updated >= '{start_date_str}' AND "
-            f"updated <= '{end_date_str}'"
-        )
+            f"{base_jql} AND " if base_jql else ""
+        ) + f"updated >= '{start_date_str}' AND updated <= '{end_date_str}'"
 
         document_batch = []
         for doc in fetch_jira_issues_batch(
@@ -252,7 +260,7 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
         end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
-        jql = f"project = {self.quoted_jira_project}"
+        jql = self._get_jql_query()
 
         slim_doc_batch = []
         for issue in _paginate_jql_search(
@@ -279,43 +287,63 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
         if self._jira_client is None:
             raise ConnectorMissingCredentialError("Jira")
 
-        if not self._jira_project:
-            raise ConnectorValidationError(
-                "Invalid connector settings: 'jira_project' must be provided."
-            )
+        # If a specific project is set, validate it exists
+        if self.jira_project:
+            try:
+                self.jira_client.project(self.jira_project)
+            except Exception as e:
+                status_code = getattr(e, "status_code", None)
 
-        try:
-            self.jira_client.project(self._jira_project)
+                if status_code == 401:
+                    raise CredentialExpiredError(
+                        "Jira credential appears to be expired or invalid (HTTP 401)."
+                    )
+                elif status_code == 403:
+                    raise InsufficientPermissionsError(
+                        "Your Jira token does not have sufficient permissions for this project (HTTP 403)."
+                    )
+                elif status_code == 404:
+                    raise ConnectorValidationError(
+                        f"Jira project not found with key: {self.jira_project}"
+                    )
+                elif status_code == 429:
+                    raise ConnectorValidationError(
+                        "Validation failed due to Jira rate-limits being exceeded. Please try again later."
+                    )
 
-        except Exception as e:
-            status_code = getattr(e, "status_code", None)
+                raise RuntimeError(f"Unexpected Jira error during validation: {e}")
+        else:
+            # If no project specified, validate we can access the Jira API
+            try:
+                # Try to list projects to validate access
+                self.jira_client.projects()
+            except Exception as e:
+                status_code = getattr(e, "status_code", None)
+                if status_code == 401:
+                    raise CredentialExpiredError(
+                        "Jira credential appears to be expired or invalid (HTTP 401)."
+                    )
+                elif status_code == 403:
+                    raise InsufficientPermissionsError(
+                        "Your Jira token does not have sufficient permissions to list projects (HTTP 403)."
+                    )
+                elif status_code == 429:
+                    raise ConnectorValidationError(
+                        "Validation failed due to Jira rate-limits being exceeded. Please try again later."
+                    )
 
-            if status_code == 401:
-                raise CredentialExpiredError(
-                    "Jira credential appears to be expired or invalid (HTTP 401)."
-                )
-            elif status_code == 403:
-                raise InsufficientPermissionsError(
-                    "Your Jira token does not have sufficient permissions for this project (HTTP 403)."
-                )
-            elif status_code == 404:
-                raise ConnectorValidationError(
-                    f"Jira project not found with key: {self._jira_project}"
-                )
-            elif status_code == 429:
-                raise ConnectorValidationError(
-                    "Validation failed due to Jira rate-limits being exceeded. Please try again later."
-                )
-            else:
-                raise Exception(f"Unexpected Jira error during validation: {e}")
+                raise RuntimeError(f"Unexpected Jira error during validation: {e}")
 
 
 if __name__ == "__main__":
     import os
 
     connector = JiraConnector(
-        os.environ["JIRA_PROJECT_URL"], comment_email_blacklist=[]
+        jira_base_url=os.environ["JIRA_BASE_URL"],
+        project_key=os.environ.get("JIRA_PROJECT_KEY"),
+        comment_email_blacklist=[],
     )
+
     connector.load_credentials(
         {
             "jira_user_email": os.environ["JIRA_USER_EMAIL"],