Linear connector (#312)

backend/danswer/configs/constants.py

@@ -32,6 +32,7 @@ class DocumentSource(str, Enum):
     FILE = "file"
     NOTION = "notion"
     ZULIP = "zulip"
+    LINEAR = "linear"
 
 
 class DanswerGenAIModel(str, Enum):

backend/danswer/connectors/factory.py

@@ -13,6 +13,7 @@ from danswer.connectors.interfaces import BaseConnector
 from danswer.connectors.interfaces import EventConnector
 from danswer.connectors.interfaces import LoadConnector
 from danswer.connectors.interfaces import PollConnector
+from danswer.connectors.linear.connector import LinearConnector
 from danswer.connectors.models import InputType
 from danswer.connectors.notion.connector import NotionConnector
 from danswer.connectors.productboard.connector import ProductboardConnector
@@ -22,8 +23,6 @@ from danswer.connectors.slack.connector import SlackPollConnector
 from danswer.connectors.web.connector import WebConnector
 from danswer.connectors.zulip.connector import ZulipConnector
 
-_NUM_SECONDS_IN_DAY = 86400
-
 
 class ConnectorMissingException(Exception):
     pass
@@ -50,6 +49,7 @@ def identify_connector_class(
         DocumentSource.NOTION: NotionConnector,
         DocumentSource.ZULIP: ZulipConnector,
         DocumentSource.GURU: GuruConnector,
+        DocumentSource.LINEAR: LinearConnector,
     }
     connector_by_source = connector_map.get(source, {})
 

backend/danswer/connectors/linear/connector.py

@@ -0,0 +1,217 @@
+import os
+from datetime import datetime
+from datetime import timezone
+from typing import Any
+from typing import cast
+
+import requests
+
+from danswer.configs.app_configs import INDEX_BATCH_SIZE
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.interfaces import GenerateDocumentsOutput
+from danswer.connectors.interfaces import LoadConnector
+from danswer.connectors.interfaces import PollConnector
+from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+from danswer.connectors.models import ConnectorMissingCredentialError
+from danswer.connectors.models import Document
+from danswer.connectors.models import Section
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+_NUM_RETRIES = 5
+_TIMEOUT = 60
+_LINEAR_GRAPHQL_URL = "https://api.linear.app/graphql"
+
+
+def _make_query(request_body: dict[str, Any], api_key: str) -> requests.Response:
+    headers = {
+        "Authorization": api_key,
+        "Content-Type": "application/json",
+    }
+
+    response: requests.Response | None = None
+    for i in range(_NUM_RETRIES):
+        try:
+            response = requests.post(
+                _LINEAR_GRAPHQL_URL,
+                headers=headers,
+                json=request_body,
+                timeout=_TIMEOUT,
+            )
+            if not response.ok:
+                raise RuntimeError(
+                    f"Error fetching issues from Linear: {response.text}"
+                )
+
+            return response
+        except Exception as e:
+            if i == _NUM_RETRIES - 1:
+                raise e
+
+            logger.warning(f"A Linear GraphQL error occurred: {e}. Retrying...")
+
+    raise RuntimeError(
+        "Unexpected execution when querying Linear. This should never happen."
+    )
+
+
+class LinearConnector(LoadConnector, PollConnector):
+    def __init__(
+        self,
+        batch_size: int = INDEX_BATCH_SIZE,
+    ) -> None:
+        self.batch_size = batch_size
+        self.linear_api_key: str | None = None
+
+    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
+        self.linear_api_key = cast(str, credentials["linear_api_key"])
+        return None
+
+    def _process_issues(
+        self, start_str: datetime | None = None, end_str: datetime | None = None
+    ) -> GenerateDocumentsOutput:
+        if self.linear_api_key is None:
+            raise ConnectorMissingCredentialError("Linear")
+
+        lte_filter = f'lte: "{end_str}"' if end_str else ""
+        gte_filter = f'gte: "{start_str}"' if start_str else ""
+        updatedAtFilter = f"""
+            {lte_filter}
+            {gte_filter}
+        """
+
+        query = (
+            """
+            query IterateIssueBatches($first: Int, $after: String) {
+                issues(
+                    orderBy: updatedAt, 
+                    first: $first, 
+                    after: $after,
+                    filter: {
+                        updatedAt: {
+        """
+            + updatedAtFilter
+            + """
+                        },
+
+                    }
+                ) {
+                    edges {
+                        node {
+                            id
+                            createdAt
+                            updatedAt
+                            archivedAt
+                            number
+                            title
+                            priority
+                            estimate
+                            sortOrder
+                            startedAt
+                            completedAt
+                            startedTriageAt
+                            triagedAt
+                            canceledAt
+                            autoClosedAt
+                            autoArchivedAt
+                            dueDate
+                            slaStartedAt
+                            slaBreachesAt
+                            trashed
+                            snoozedUntilAt
+                            team {
+                                name
+                            }
+                            previousIdentifiers
+                            subIssueSortOrder
+                            priorityLabel
+                            identifier
+                            url
+                            branchName
+                            customerTicketCount
+                            description
+                            descriptionData
+                            comments {
+                                nodes {
+                                    url
+                                    body
+                                }
+                            }
+                        }
+                    }
+                    pageInfo {
+                        hasNextPage
+                        endCursor
+                    }
+                }
+            }
+        """
+        )
+
+        has_more = True
+        endCursor = None
+        while has_more:
+            graphql_query = {
+                "query": query,
+                "variables": {
+                    "first": self.batch_size,
+                    "after": endCursor,
+                },
+            }
+            logger.debug(f"Requesting issues from Linear with query: {graphql_query}")
+
+            response = _make_query(graphql_query, self.linear_api_key)
+            response_json = response.json()
+            logger.debug(f"Raw response from Linear: {response_json}")
+            edges = response_json["data"]["issues"]["edges"]
+
+            documents: list[Document] = []
+            for edge in edges:
+                node = edge["node"]
+                documents.append(
+                    Document(
+                        id=node["id"],
+                        sections=[
+                            Section(
+                                link=node["url"],
+                                text=node["description"],
+                            )
+                        ]
+                        + [
+                            Section(
+                                link=node["url"],
+                                text=comment["body"],
+                            )
+                            for comment in node["comments"]["nodes"]
+                        ],
+                        source=DocumentSource.LINEAR,
+                        semantic_identifier=node["identifier"],
+                        metadata={
+                            "updated_at": node["updatedAt"],
+                            "team": node["team"]["name"],
+                        },
+                    )
+                )
+                yield documents
+
+            endCursor = response_json["data"]["issues"]["pageInfo"]["endCursor"]
+            has_more = response_json["data"]["issues"]["pageInfo"]["hasNextPage"]
+
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        yield from self._process_issues()
+
+    def poll_source(
+        self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
+    ) -> GenerateDocumentsOutput:
+        start_time = datetime.fromtimestamp(start, tz=timezone.utc)
+        end_time = datetime.fromtimestamp(end, tz=timezone.utc)
+
+        yield from self._process_issues(start_str=start_time, end_str=end_time)
+
+
+if __name__ == "__main__":
+    connector = LinearConnector()
+    connector.load_credentials({"linear_api_key": os.environ["LINEAR_API_KEY"]})
+    document_batches = connector.load_from_state()
+    print(next(document_batches))

web/src/app/admin/connectors/linear/page.tsx

@@ -0,0 +1,227 @@
+"use client";
+
+import * as Yup from "yup";
+import { LinearIcon, TrashIcon } from "@/components/icons/icons";
+import { TextFormField } from "@/components/admin/connectors/Field";
+import { HealthCheckBanner } from "@/components/health/healthcheck";
+import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
+import {
+  Credential,
+  ConnectorIndexingStatus,
+  LinearCredentialJson,
+} from "@/lib/types";
+import useSWR, { useSWRConfig } from "swr";
+import { fetcher } from "@/lib/fetcher";
+import { LoadingAnimation } from "@/components/Loading";
+import { deleteCredential, linkCredential } from "@/lib/credential";
+import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
+import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
+import { usePopup } from "@/components/admin/connectors/Popup";
+
+const Main = () => {
+  const { popup, setPopup } = usePopup();
+
+  const { mutate } = useSWRConfig();
+  const {
+    data: connectorIndexingStatuses,
+    isLoading: isConnectorIndexingStatusesLoading,
+    error: isConnectorIndexingStatusesError,
+  } = useSWR<ConnectorIndexingStatus<any, any>[]>(
+    "/api/manage/admin/connector/indexing-status",
+    fetcher
+  );
+  const {
+    data: credentialsData,
+    isLoading: isCredentialsLoading,
+    isValidating: isCredentialsValidating,
+    error: isCredentialsError,
+  } = useSWR<Credential<any>[]>("/api/manage/credential", fetcher);
+
+  if (
+    isConnectorIndexingStatusesLoading ||
+    isCredentialsLoading ||
+    isCredentialsValidating
+  ) {
+    return <LoadingAnimation text="Loading" />;
+  }
+
+  if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) {
+    return <div>Failed to load connectors</div>;
+  }
+
+  if (isCredentialsError || !credentialsData) {
+    return <div>Failed to load credentials</div>;
+  }
+
+  const linearConnectorIndexingStatuses: ConnectorIndexingStatus<
+    {},
+    LinearCredentialJson
+  >[] = connectorIndexingStatuses.filter(
+    (connectorIndexingStatus) =>
+      connectorIndexingStatus.connector.source === "linear"
+  );
+  const linearCredential: Credential<LinearCredentialJson> =
+    credentialsData.filter(
+      (credential) => credential.credential_json?.linear_api_key
+    )[0];
+
+  return (
+    <>
+      {popup}
+      <h2 className="font-bold mb-2 mt-6 ml-auto mr-auto">
+        Step 1: Provide your Credentials
+      </h2>
+
+      {linearCredential ? (
+        <>
+          <div className="flex mb-1 text-sm">
+            <p className="my-auto">Existing API Key: </p>
+            <p className="ml-1 italic my-auto max-w-md truncate">
+              {linearCredential.credential_json?.linear_api_key}
+            </p>
+            <button
+              className="ml-1 hover:bg-gray-700 rounded-full p-1"
+              onClick={async () => {
+                if (linearConnectorIndexingStatuses.length > 0) {
+                  setPopup({
+                    type: "error",
+                    message:
+                      "Must delete all connectors before deleting credentials",
+                  });
+                  return;
+                }
+                await deleteCredential(linearCredential.id);
+                mutate("/api/manage/credential");
+              }}
+            >
+              <TrashIcon />
+            </button>
+          </div>
+        </>
+      ) : (
+        <>
+          <p className="text-sm">
+            To use the Linear connector, first follow the guide{" "}
+            <a
+              className="text-blue-500"
+              href="https://docs.danswer.dev/connectors/jira#setting-up"
+            >
+              here
+            </a>{" "}
+            to generate an API Key.
+          </p>
+          <div className="border-solid border-gray-600 border rounded-md p-6 mt-2">
+            <CredentialForm<LinearCredentialJson>
+              formBody={
+                <>
+                  <TextFormField
+                    name="linear_api_key"
+                    label="Linear API Key:"
+                    type="password"
+                  />
+                </>
+              }
+              validationSchema={Yup.object().shape({
+                linear_api_key: Yup.string().required(
+                  "Please enter your Linear API Key!"
+                ),
+              })}
+              initialValues={{
+                linear_api_key: "",
+              }}
+              onSubmit={(isSuccess) => {
+                if (isSuccess) {
+                  mutate("/api/manage/credential");
+                }
+              }}
+            />
+          </div>
+        </>
+      )}
+
+      <h2 className="font-bold mb-2 mt-6 ml-auto mr-auto">
+        Step 2: Start indexing
+      </h2>
+      {linearCredential ? (
+        <>
+          {linearConnectorIndexingStatuses.length > 0 ? (
+            <>
+              <p className="text-sm mb-2">
+                We pull the latest <i>issues</i> and <i>comments</i> every{" "}
+                <b>10</b> minutes.
+              </p>
+              <div className="mb-2">
+                <ConnectorsTable<{}, LinearCredentialJson>
+                  connectorIndexingStatuses={linearConnectorIndexingStatuses}
+                  liveCredential={linearCredential}
+                  getCredential={(credential) => {
+                    return (
+                      <div>
+                        <p>{credential.credential_json.linear_api_key}</p>
+                      </div>
+                    );
+                  }}
+                  onCredentialLink={async (connectorId) => {
+                    if (linearCredential) {
+                      await linkCredential(connectorId, linearCredential.id);
+                      mutate("/api/manage/admin/connector/indexing-status");
+                    }
+                  }}
+                  onUpdate={() =>
+                    mutate("/api/manage/admin/connector/indexing-status")
+                  }
+                />
+              </div>
+            </>
+          ) : (
+            <div className="border-solid border-gray-600 border rounded-md p-6 mt-4">
+              <h2 className="font-bold mb-3">Create Connector</h2>
+              <p className="text-sm mb-4">
+                Press connect below to start the connection Linear. We pull the
+                latest <i>issues</i> and <i>comments</i> every <b>10</b>{" "}
+                minutes.
+              </p>
+              <ConnectorForm<{}>
+                nameBuilder={() => "LinearConnector"}
+                source="linear"
+                inputType="poll"
+                formBody={<></>}
+                validationSchema={Yup.object().shape({})}
+                initialValues={{}}
+                refreshFreq={10 * 60} // 10 minutes
+                onSubmit={async (isSuccess, responseJson) => {
+                  if (isSuccess && responseJson) {
+                    await linkCredential(responseJson.id, linearCredential.id);
+                    mutate("/api/manage/admin/connector/indexing-status");
+                  }
+                }}
+              />
+            </div>
+          )}
+        </>
+      ) : (
+        <>
+          <p className="text-sm">
+            Please provide your access token in Step 1 first! Once done with
+            that, you can then start indexing Linear.
+          </p>
+        </>
+      )}
+    </>
+  );
+};
+
+export default function Page() {
+  return (
+    <div className="mx-auto container">
+      <div className="mb-4">
+        <HealthCheckBanner />
+      </div>
+      <div className="border-solid border-gray-600 border-b mb-4 pb-2 flex">
+        <LinearIcon size={32} />
+        <h1 className="text-3xl font-bold pl-2">Linear</h1>
+      </div>
+      <Main />
+    </div>
+  );
+}

web/src/app/admin/layout.tsx

@@ -16,6 +16,7 @@ import {
   NotionIcon,
   ZulipIcon,
   ProductboardIcon,
+  LinearIcon,
 } from "@/components/icons/icons";
 import { DISABLE_AUTH } from "@/lib/constants";
 import { getCurrentUserSS } from "@/lib/userSS";
@@ -106,6 +107,15 @@ export default async function AdminLayout({
                   ),
                   link: "/admin/connectors/jira",
                 },
+                {
+                  name: (
+                    <div className="flex">
+                      <LinearIcon size={16} />
+                      <div className="ml-1">Linear</div>
+                    </div>
+                  ),
+                  link: "/admin/connectors/linear",
+                },
                 {
                   name: (
                     <div className="flex">

web/src/components/icons/icons.tsx

@@ -21,6 +21,7 @@ import jiraSVG from "../../../public/Jira.svg";
 import confluenceSVG from "../../../public/Confluence.svg";
 import guruIcon from "../../../public/Guru.svg";
 import zulipIcon from "../../../public/Zulip.png";
+import linearIcon from "../../../public/Linear.png";
 
 interface IconProps {
   size?: number;
@@ -237,6 +238,21 @@ export const ProductboardIcon = ({
   );
 };
 
+export const LinearIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return (
+    <div
+      // Linear Icon has a bit more surrounding whitespace than other icons, which is why we need to adjust it here
+      style={{ width: `${size + 4}px`, height: `${size + 4}px` }}
+      className={`w-[${size + 4}px] h-[${size + 4}px] -m-0.5 ` + className}
+    >
+      <Image src={linearIcon} alt="Logo" width="96" height="96" />
+    </div>
+  );
+};
+
 export const SlabIcon = ({
   size = 16,
   className = defaultTailwindCSS,

web/src/components/search/Filters.tsx

@@ -18,6 +18,7 @@ const sources: Source[] = [
   { displayName: "File", internalName: "file" },
   { displayName: "Notion", internalName: "notion" },
   { displayName: "Zulip", internalName: "zulip" },
+  { displayName: "Linear", internalName: "linear" },
 ];
 
 interface SourceSelectorProps {

web/src/components/source.tsx

@@ -8,6 +8,7 @@ import {
   GoogleDriveIcon,
   GuruIcon,
   JiraIcon,
+  LinearIcon,
   NotionIcon,
   ProductboardIcon,
   SlabIcon,
@@ -101,6 +102,12 @@ export const getSourceMetadata = (sourceType: ValidSources): SourceMetadata => {
         displayName: "Guru",
         adminPageLink: "/admin/connectors/guru",
       };
+    case "linear":
+      return {
+        icon: LinearIcon,
+        displayName: "Linear",
+        adminPageLink: "/admin/connectors/linear",
+      };
     default:
       throw new Error("Invalid source type");
   }

web/src/lib/types.ts

@@ -20,6 +20,7 @@ export type ValidSources =
   | "notion"
   | "guru"
   | "zulip"
+  | "linear"
   | "file";
 export type ValidInputTypes = "load_state" | "poll" | "event";
 export type ValidStatuses =
@@ -181,6 +182,10 @@ export interface GuruCredentialJson {
   guru_user_token: string;
 }
 
+export interface LinearCredentialJson {
+  linear_api_key: string;
+}
+
 // DELETION
 
 export interface DeletionAttemptSnapshot {