Add Loopio Connector (#850)

Looks good! I couldn't verify that it end-to-end because Loopio still hasn't granted me API access but the code looks good. Thanks a bunch for the contribution!

Would you be open to also writing the docs page for the setup? It's just adding an md file with some images or gifs:
https://github.com/danswer-ai/danswer-docs

I can provide a template branch if that would make it easier, just let me know 🙏

backend/danswer/configs/constants.py

@@ -78,6 +78,7 @@ class DocumentSource(str, Enum):
     GONG = "gong"
     GOOGLE_SITES = "google_sites"
     ZENDESK = "zendesk"
+    LOOPIO = "loopio"
 
 
 class DocumentIndexType(str, Enum):

backend/danswer/connectors/factory.py

@@ -18,6 +18,7 @@ from danswer.connectors.interfaces import EventConnector
 from danswer.connectors.interfaces import LoadConnector
 from danswer.connectors.interfaces import PollConnector
 from danswer.connectors.linear.connector import LinearConnector
+from danswer.connectors.loopio.connector import LoopioConnector
 from danswer.connectors.models import InputType
 from danswer.connectors.notion.connector import NotionConnector
 from danswer.connectors.productboard.connector import ProductboardConnector
@@ -62,6 +63,7 @@ def identify_connector_class(
         DocumentSource.GONG: GongConnector,
         DocumentSource.GOOGLE_SITES: GoogleSitesConnector,
         DocumentSource.ZENDESK: ZendeskConnector,
+        DocumentSource.LOOPIO: LoopioConnector,
     }
     connector_by_source = connector_map.get(source, {})
 

backend/danswer/connectors/loopio/connector.py

@@ -0,0 +1,218 @@
+import json
+from datetime import datetime
+from datetime import timezone
+from typing import Any
+
+from oauthlib.oauth2 import BackendApplicationClient
+from requests_oauthlib import OAuth2Session
+
+from danswer.configs.app_configs import INDEX_BATCH_SIZE
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.cross_connector_utils.html_utils import parse_html_page_basic
+from danswer.connectors.cross_connector_utils.html_utils import (
+    strip_excessive_newlines_and_spaces,
+)
+from danswer.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
+from danswer.connectors.interfaces import GenerateDocumentsOutput
+from danswer.connectors.interfaces import LoadConnector
+from danswer.connectors.interfaces import PollConnector
+from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+from danswer.connectors.models import BasicExpertInfo
+from danswer.connectors.models import ConnectorMissingCredentialError
+from danswer.connectors.models import Document
+from danswer.connectors.models import Section
+from danswer.utils.logger import setup_logger
+
+LOOPIO_API_BASE = "https://api.loopio.com/"
+LOOPIO_AUTH_URL = LOOPIO_API_BASE + "oauth2/access_token"
+LOOPIO_DATA_URL = LOOPIO_API_BASE + "data/"
+
+logger = setup_logger()
+
+
+class LoopioConnector(LoadConnector, PollConnector):
+    def __init__(
+        self,
+        loopio_stack_name: str | None = None,
+        batch_size: int = INDEX_BATCH_SIZE,
+    ) -> None:
+        self.batch_size = batch_size
+        self.loopio_client_id = None
+        self.loopio_client_token = None
+        self.loopio_stack_name = loopio_stack_name
+        self.search_filter = {}
+
+    def _fetch_data(
+        self, resource: str, params: dict[str, str | int]
+    ) -> dict[str, Any]:
+        client = BackendApplicationClient(
+            client_id=self.loopio_client_id, scope=["library:read"]
+        )
+        session = OAuth2Session(client=client)
+        session.fetch_token(
+            token_url=LOOPIO_AUTH_URL,
+            client_id=self.loopio_client_id,
+            client_secret=self.loopio_client_token,
+        )
+        page = 0
+        stop_at_page = 1
+        while (page := page + 1) <= stop_at_page:
+            params["page"] = page
+            response = session.request(
+                "GET",
+                LOOPIO_DATA_URL + resource,
+                headers={"Accept": "application/json"},
+                params=params,
+            )
+            if response.status_code == 400:
+                logger.error(
+                    f"Loopio API returned 400 for {resource} with params {params}",
+                )
+                logger.error(response.text)
+            response.raise_for_status()
+            response_data = json.loads(response.text)
+            stop_at_page = response_data.get("totalPages", 1)
+            yield response_data
+
+    def _build_search_filter(
+        self, stack_name: str, start: str, end: str
+    ) -> dict[str, Any]:
+        filter = {}
+        if start is not None and end is not None:
+            filter["lastUpdatedDate"] = {"gte": start, "lt": end}
+
+        if stack_name is not None:
+            # Right now this is fetching the stacks every time, which is not ideal.
+            # We should update this later to store the ID when we create the Connector
+            for stack in self._fetch_data(resource="v2/stacks", params={}):
+                for item in stack["items"]:
+                    if item["name"] == stack_name:
+                        filter["locations"] = [{"stackID": item["id"]}]
+                        break
+            if "locations" not in filter:
+                raise ValueError(f"Stack {stack_name} not found in Loopio")
+        return filter
+
+    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
+        self.loopio_subdomain = credentials["loopio_subdomain"]
+        self.loopio_client_id = credentials["loopio_client_id"]
+        self.loopio_client_token = credentials["loopio_client_token"]
+        return None
+
+    def _process_entries(
+        self, start: str | None = None, end: str | None = None
+    ) -> GenerateDocumentsOutput:
+        if self.loopio_client_id is None or self.loopio_client_token is None:
+            raise ConnectorMissingCredentialError("Loopio")
+
+        filter = self._build_search_filter(
+            stack_name=self.loopio_stack_name, start=start, end=end
+        )
+        params: dict[str, str | int] = {"pageSize": self.batch_size}
+        params["filter"] = json.dumps(filter)
+
+        doc_batch: list[Document] = []
+        for library_entries in self._fetch_data(
+            resource="v2/libraryEntries", params=params
+        ):
+            for entry in library_entries.get("items", []):
+                link = f"https://{self.loopio_subdomain}.loopio.com/library?entry={entry['id']}"
+                topic = "/".join(
+                    part["name"] for part in entry["location"].values() if part
+                )
+
+                answer = parse_html_page_basic(entry.get("answer", {}).get("text", ""))
+                questions = [
+                    question.get("text").replace("\xa0", " ")
+                    for question in entry["questions"]
+                    if question.get("text")
+                ]
+                questions_string = strip_excessive_newlines_and_spaces(
+                    "\n".join(questions)
+                )
+                content_text = f"{answer}\n\nRelated Questions: {questions_string}"
+                content_text = strip_excessive_newlines_and_spaces(
+                    content_text.replace("\xa0", " ")
+                )
+
+                last_updated = time_str_to_utc(entry["lastUpdatedDate"])
+                last_reviewed = (
+                    time_str_to_utc(entry["lastReviewedDate"])
+                    if entry.get("lastReviewedDate")
+                    else None
+                )
+
+                # For Danswer, we decay document score overtime, either last_updated or
+                # last_reviewed is a good enough signal for the document's recency
+                latest_time = (
+                    max(last_reviewed, last_updated) if last_reviewed else last_updated
+                )
+                creator = entry.get("creator")
+                last_updated_by = entry.get("lastUpdatedBy")
+                last_reviewed_by = entry.get("lastReviewedBy")
+
+                primary_owners: list[BasicExpertInfo] = [
+                    {"display_name": owner.get("name")}
+                    for owner in [creator, last_updated_by]
+                    if owner is not None
+                ]
+                secondary_owners: list[BasicExpertInfo] = [
+                    {"display_name": owner.get("name")}
+                    for owner in [last_reviewed_by]
+                    if owner is not None
+                ]
+                doc_batch.append(
+                    Document(
+                        id=entry["id"],
+                        sections=[Section(link=link, text=content_text)],
+                        source=DocumentSource.LOOPIO,
+                        semantic_identifier=questions[0],
+                        doc_updated_at=latest_time,
+                        primary_owners=primary_owners,
+                        secondary_owners=secondary_owners,
+                        metadata={
+                            "topic": topic,
+                            "questions": "\n".join(questions),
+                            "creator": creator.get("name") if creator else None,
+                        },
+                    )
+                )
+
+            if len(doc_batch) >= self.batch_size:
+                yield doc_batch
+                doc_batch = []
+        if len(doc_batch) > 0:
+            yield doc_batch
+
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        return self._process_entries()
+
+    def poll_source(
+        self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
+    ) -> GenerateDocumentsOutput:
+        start_time = datetime.fromtimestamp(start, tz=timezone.utc).isoformat(
+            timespec="seconds"
+        )
+        end_time = datetime.fromtimestamp(end, tz=timezone.utc).isoformat(
+            timespec="seconds"
+        )
+
+        return self._process_entries(start_time, end_time)
+
+
+if __name__ == "__main__":
+    import os
+
+    connector = LoopioConnector(
+        loopio_stack_name=os.environ.get("LOOPIO_STACK_NAME", None)
+    )
+    connector.load_credentials(
+        {
+            "loopio_client_id": os.environ["LOOPIO_CLIENT_ID"],
+            "loopio_client_token": os.environ["LOOPIO_CLIENT_TOKEN"],
+            "loopio_subdomain": os.environ["LOOPIO_SUBDOMAIN"],
+        }
+    )
+
+    latest_docs = connector.load_from_state()
+    print(next(latest_docs))

web/src/app/admin/connectors/loopio/page.tsx

@@ -0,0 +1,252 @@
+"use client";
+
+import * as Yup from "yup";
+import { LoopioIcon, TrashIcon } from "@/components/icons/icons";
+import { TextFormField } from "@/components/admin/connectors/Field";
+import { HealthCheckBanner } from "@/components/health/healthcheck";
+import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
+import {
+  Credential,
+  ConnectorIndexingStatus,
+  LoopioConfig,
+  LoopioCredentialJson,
+} from "@/lib/types";
+import useSWR, { useSWRConfig } from "swr";
+import { fetcher } from "@/lib/fetcher";
+import { LoadingAnimation } from "@/components/Loading";
+import { adminDeleteCredential, linkCredential } from "@/lib/credential";
+import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
+import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
+import { usePopup } from "@/components/admin/connectors/Popup";
+import { usePublicCredentials } from "@/lib/hooks";
+
+const Main = () => {
+  const { popup, setPopup } = usePopup();
+
+  const { mutate } = useSWRConfig();
+  const {
+    data: connectorIndexingStatuses,
+    isLoading: isConnectorIndexingStatusesLoading,
+    error: isConnectorIndexingStatusesError,
+  } = useSWR<ConnectorIndexingStatus<any, any>[]>(
+    "/api/manage/admin/connector/indexing-status",
+    fetcher
+  );
+
+  const {
+    data: credentialsData,
+    isLoading: isCredentialsLoading,
+    isValidating: isCredentialsValidating,
+    error: isCredentialsError,
+    refreshCredentials,
+  } = usePublicCredentials();
+
+  if (
+    isConnectorIndexingStatusesLoading ||
+    isCredentialsLoading ||
+    isCredentialsValidating
+  ) {
+    return <LoadingAnimation text="Loading" />;
+  }
+
+  if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) {
+    return <div>Failed to load connectors</div>;
+  }
+
+  if (isCredentialsError || !credentialsData) {
+    return <div>Failed to load credentials</div>;
+  }
+
+  const loopioConnectorIndexingStatuses: ConnectorIndexingStatus<
+    LoopioConfig,
+    LoopioCredentialJson
+  >[] = connectorIndexingStatuses.filter(
+    (connectorIndexingStatus) =>
+      connectorIndexingStatus.connector.source === "loopio"
+  );
+  const loopioCredential: Credential<LoopioCredentialJson> | undefined =
+    credentialsData.find(
+      (credential) => credential.credential_json?.loopio_client_id
+    );
+
+  return (
+    <>
+      {popup}
+      <p className="text-sm">
+        This connector allows you to sync your Loopio Library Entries into
+        Danswer
+      </p>
+
+      <h2 className="font-bold mb-2 mt-6 ml-auto mr-auto">
+        Step 1: Provide your API Access info
+      </h2>
+
+      {loopioCredential ? (
+        <>
+          <div className="flex mb-1 text-sm">
+            <p className="my-auto">Existing App Key Secret: </p>
+            <p className="ml-1 italic my-auto max-w-md truncate">
+              {loopioCredential.credential_json?.loopio_client_token}
+            </p>
+            <button
+              className="ml-1 hover:bg-gray-700 rounded-full p-1"
+              onClick={async () => {
+                if (loopioConnectorIndexingStatuses.length > 0) {
+                  setPopup({
+                    type: "error",
+                    message:
+                      "Must delete all connectors before deleting credentials",
+                  });
+                  return;
+                }
+                await adminDeleteCredential(loopioCredential.id);
+                refreshCredentials();
+              }}
+            >
+              <TrashIcon />
+            </button>
+          </div>
+        </>
+      ) : (
+        <>
+          <div className="border-solid border-gray-600 border rounded-md p-6 mt-2">
+            <CredentialForm<LoopioCredentialJson>
+              formBody={
+                <>
+                  <TextFormField
+                    name="loopio_subdomain"
+                    label="Account Subdomain:"
+                  />
+                  <TextFormField name="loopio_client_id" label="App Key ID:" />
+                  <TextFormField
+                    name="loopio_client_token"
+                    label="App Key Secret:"
+                    type="password"
+                  />
+                </>
+              }
+              validationSchema={Yup.object().shape({
+                loopio_subdomain: Yup.string().required(
+                  "Please enter your Loopio Account subdomain"
+                ),
+                loopio_client_id: Yup.string().required(
+                  "Please enter your Loopio App Key ID"
+                ),
+                loopio_client_token: Yup.string().required(
+                  "Please enter your Loopio App Key Secret"
+                ),
+              })}
+              initialValues={{
+                loopio_subdomain: "",
+                loopio_client_id: "",
+                loopio_client_token: "",
+              }}
+              onSubmit={(isSuccess) => {
+                if (isSuccess) {
+                  refreshCredentials();
+                }
+              }}
+            />
+          </div>
+        </>
+      )}
+
+      <h2 className="font-bold mt-6 ml-auto mr-auto">
+        Step 2: Which Stack do you want to make searchable?
+      </h2>
+      <p className="text-sm mb-2">
+        Leave this blank if you want to index all stacks.
+      </p>
+
+      {loopioConnectorIndexingStatuses.length > 0 && (
+        <>
+          <p className="text-sm mb-2">
+            We pull the latest library entries every <b>24</b> hours.
+          </p>
+          <div className="mb-2">
+            <ConnectorsTable<LoopioConfig, LoopioCredentialJson>
+              connectorIndexingStatuses={loopioConnectorIndexingStatuses}
+              liveCredential={loopioCredential}
+              getCredential={(credential) =>
+                credential.credential_json.loopio_client_id
+              }
+              specialColumns={[
+                {
+                  header: "Stack",
+                  key: "loopio_stack_name",
+                  getValue: (ccPairStatus) =>
+                    ccPairStatus.connector.connector_specific_config
+                      .loopio_stack_name || "All stacks",
+                },
+              ]}
+              includeName={true}
+              onUpdate={() =>
+                mutate("/api/manage/admin/connector/indexing-status")
+              }
+              onCredentialLink={async (connectorId) => {
+                if (loopioCredential) {
+                  await linkCredential(connectorId, loopioCredential.id);
+                  mutate("/api/manage/admin/connector/indexing-status");
+                }
+              }}
+            />
+          </div>
+        </>
+      )}
+
+      {loopioCredential ? (
+        <>
+          <div className="border-solid border-gray-600 border rounded-md p-6 mt-4">
+            <h2 className="font-bold mb-3">Create a new Loopio Connector</h2>
+            <ConnectorForm<LoopioConfig>
+              nameBuilder={(values) =>
+                values?.loopio_stack_name
+                  ? `LoopioConnector-${values.loopio_stack_name}-Stack`
+                  : `LoopioConnector-AllStacks`
+              }
+              source="loopio"
+              inputType="poll"
+              formBody={
+                <>
+                  <TextFormField
+                    name="loopio_stack_name"
+                    label="[Optional] Loopio Stack name:"
+                    subtext=" Must be exact match to the name in Library Management, leave this blank if you want to index all Stacks"
+                  />
+                </>
+              }
+              validationSchema={Yup.object().shape({
+                loopio_stack_name: Yup.string(),
+              })}
+              initialValues={{
+                loopio_stack_name: "",
+              }}
+              refreshFreq={60 * 60 * 24} // 24 hours
+              credentialId={loopioCredential.id}
+            />
+          </div>
+        </>
+      ) : (
+        <p className="text-sm">
+          Please provide your API Access Info in Step 1 first! Once done with
+          that, you can start indexing your Loopio library.
+        </p>
+      )}
+    </>
+  );
+};
+
+export default function Page() {
+  return (
+    <div className="mx-auto container">
+      <div className="mb-4">
+        <HealthCheckBanner />
+      </div>
+      <div className="border-solid border-gray-600 border-b mb-4 pb-2 flex">
+        <LoopioIcon size={32} />
+        <h1 className="text-3xl font-bold pl-2">Loopio</h1>
+      </div>
+      <Main />
+    </div>
+  );
+}

web/src/components/icons/icons.tsx

@@ -302,6 +302,20 @@ export const ConnectorIcon = ({
 //  COMPANY LOGOS
 //
 
+export const LoopioIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return (
+    <div
+      style={{ width: `${size}px`, height: `${size}px` }}
+      className={`w-[${size}px] h-[${size}px] dark:invert ` + className}
+    >
+      <Image src="/Loopio.png" alt="Logo" width="96" height="96" />
+    </div>
+  );
+};
+
 export const SlackIcon = ({
   size = 16,
   className = defaultTailwindCSS,

web/src/lib/sources.ts

@@ -12,6 +12,7 @@ import {
   HubSpotIcon,
   JiraIcon,
   LinearIcon,
+  LoopioIcon,
   NotionIcon,
   ProductboardIcon,
   RequestTrackerIcon,
@@ -134,6 +135,11 @@ const SOURCE_METADATA_MAP: SourceMap = {
     displayName: "Request Tracker",
     category: SourceCategory.AppConnection,
   },
+  loopio: {
+    icon: LoopioIcon,
+    displayName: "Loopio",
+    category: SourceCategory.AppConnection,
+  },
 };
 
 function fillSourceMetadata(

web/src/lib/types.ts

@@ -29,6 +29,7 @@ export type ValidSources =
   | "requesttracker"
   | "file"
   | "google_sites"
+  | "loopio"
   | "zendesk";
 
 export type ValidInputTypes = "load_state" | "poll" | "event";
@@ -109,6 +110,10 @@ export interface GongConfig {
   workspaces?: string[];
 }
 
+export interface LoopioConfig {
+  loopio_stack_name?: string;
+}
+
 export interface FileConfig {
   file_locations: string[];
 }
@@ -239,6 +244,12 @@ export interface GongCredentialJson {
   gong_access_key_secret: string;
 }
 
+export interface LoopioCredentialJson {
+  loopio_subdomain: string;
+  loopio_client_id: string;
+  loopio_client_token: string;
+}
+
 export interface LinearCredentialJson {
   linear_api_key: string;
 }