Axero Spaces (#1276)

.github/workflows/docker-build-push-backend-container-on-tag.yml

@@ -38,5 +38,6 @@ jobs:
     - name: Run Trivy vulnerability scanner
       uses: aquasecurity/trivy-action@master
       with:
+        # To run locally: trivy image --severity HIGH,CRITICAL danswer/danswer-backend
         image-ref: docker.io/danswer/danswer-backend:${{ github.ref_name }}
         severity: 'CRITICAL,HIGH'

backend/danswer/connectors/axero/connector.py

@@ -38,6 +38,7 @@ def _get_entities(
     axero_base_url: str,
     start: datetime,
     end: datetime,
+    space_id: str | None = None,
 ) -> list[dict]:
     endpoint = axero_base_url + "api/content/list"
     page_num = 1
@@ -51,6 +52,10 @@ def _get_entities(
             "SortOrder": "1",  # descending
             "StartPage": str(page_num),
         }
+
+        if space_id is not None:
+            params["SpaceID"] = space_id
+
         res = requests.get(endpoint, headers=_get_auth_header(api_key), params=params)
         res.raise_for_status()
 
@@ -116,7 +121,8 @@ def _translate_content_to_doc(content: dict) -> Document:
 class AxeroConnector(PollConnector):
     def __init__(
         self,
-        base_url: str,
+        # Strings of the integer ids of the spaces
+        spaces: list[str] | None = None,
         include_article: bool = True,
         include_blog: bool = True,
         include_wiki: bool = True,
@@ -129,20 +135,24 @@ class AxeroConnector(PollConnector):
         self.include_wiki = include_wiki
         self.include_forum = include_forum
         self.batch_size = batch_size
+        self.space_ids = spaces
         self.axero_key = None
-
-        if not base_url.endswith("/"):
-            base_url += "/"
-        self.base_url = base_url
+        self.base_url = None
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         self.axero_key = credentials["axero_api_token"]
+        # As the API key specifically applies to a particular deployment, this is
+        # included as part of the credential
+        base_url = credentials["base_url"]
+        if not base_url.endswith("/"):
+            base_url += "/"
+        self.base_url = base_url
         return None
 
     def poll_source(
         self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
     ) -> GenerateDocumentsOutput:
-        if not self.axero_key:
+        if not self.axero_key or not self.base_url:
             raise ConnectorMissingCredentialError("Axero")
 
         start_datetime = datetime.utcfromtimestamp(start).replace(tzinfo=timezone.utc)
@@ -158,26 +168,35 @@ class AxeroConnector(PollConnector):
         if self.include_forum:
             raise NotImplementedError("Forums for Axero not supported currently")
 
-        for entity in entity_types:
-            articles = _get_entities(
-                entity_type=entity,
-                api_key=self.axero_key,
-                axero_base_url=self.base_url,
-                start=start_datetime,
-                end=end_datetime,
-            )
-            yield from process_in_batches(
-                objects=articles,
-                process_function=_translate_content_to_doc,
-                batch_size=self.batch_size,
-            )
+        iterable_space_ids = self.space_ids if self.space_ids else [None]
+
+        for space_id in iterable_space_ids:
+            for entity in entity_types:
+                axero_obj = _get_entities(
+                    entity_type=entity,
+                    api_key=self.axero_key,
+                    axero_base_url=self.base_url,
+                    start=start_datetime,
+                    end=end_datetime,
+                    space_id=space_id,
+                )
+                yield from process_in_batches(
+                    objects=axero_obj,
+                    process_function=_translate_content_to_doc,
+                    batch_size=self.batch_size,
+                )
 
 
 if __name__ == "__main__":
     import os
 
-    connector = AxeroConnector(base_url=os.environ["AXERO_BASE_URL"])
-    connector.load_credentials({"axero_api_token": os.environ["AXERO_API_TOKEN"]})
+    connector = AxeroConnector()
+    connector.load_credentials(
+        {
+            "axero_api_token": os.environ["AXERO_API_TOKEN"],
+            "base_url": os.environ["AXERO_BASE_URL"],
+        }
+    )
     current = time.time()
 
     one_year_ago = current - 24 * 60 * 60 * 360

web/src/app/admin/connectors/axero/page.tsx

@@ -1,30 +1,32 @@
 "use client";
 
 import * as Yup from "yup";
-import { AxeroIcon, LinearIcon, TrashIcon } from "@/components/icons/icons";
-import { TextFormField } from "@/components/admin/connectors/Field";
+import { AxeroIcon, TrashIcon } from "@/components/icons/icons";
+import { fetcher } from "@/lib/fetcher";
+import useSWR, { useSWRConfig } from "swr";
+import { LoadingAnimation } from "@/components/Loading";
 import { HealthCheckBanner } from "@/components/health/healthcheck";
+import {
+  AxeroConfig,
+  AxeroCredentialJson,
+  ConnectorIndexingStatus,
+  Credential,
+} from "@/lib/types";
+import { adminDeleteCredential, linkCredential } from "@/lib/credential";
 import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
 import {
-  Credential,
-  ConnectorIndexingStatus,
-  LinearCredentialJson,
-  AxeroCredentialJson,
-} from "@/lib/types";
-import useSWR, { useSWRConfig } from "swr";
-import { fetcher } from "@/lib/fetcher";
-import { LoadingAnimation } from "@/components/Loading";
-import { adminDeleteCredential, linkCredential } from "@/lib/credential";
-import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
+  TextFormField,
+  TextArrayFieldBuilder,
+  BooleanFormField,
+  TextArrayField,
+} from "@/components/admin/connectors/Field";
 import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
-import { usePopup } from "@/components/admin/connectors/Popup";
+import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
 import { usePublicCredentials } from "@/lib/hooks";
-import { Card, Text, Title } from "@tremor/react";
+import { Button, Card, Divider, Text, Title } from "@tremor/react";
 import { AdminPageTitle } from "@/components/admin/Title";
 
-const Main = () => {
-  const { popup, setPopup } = usePopup();
-
+const MainSection = () => {
   const { mutate } = useSWRConfig();
   const {
     data: connectorIndexingStatuses,
@@ -32,21 +34,19 @@ const Main = () => {
     error: isConnectorIndexingStatusesError,
   } = useSWR<ConnectorIndexingStatus<any, any>[]>(
     "/api/manage/admin/connector/indexing-status",
-    fetcher,
-    { refreshInterval: 5000 } // 5 seconds
+    fetcher
   );
+
   const {
     data: credentialsData,
     isLoading: isCredentialsLoading,
     error: isCredentialsError,
-    isValidating: isCredentialsValidating,
     refreshCredentials,
   } = usePublicCredentials();
 
   if (
-    isConnectorIndexingStatusesLoading ||
-    isCredentialsLoading ||
-    isCredentialsValidating
+    (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) ||
+    (!credentialsData && isCredentialsLoading)
   ) {
     return <LoadingAnimation text="Loading" />;
   }
@@ -60,7 +60,7 @@ const Main = () => {
   }
 
   const axeroConnectorIndexingStatuses: ConnectorIndexingStatus<
-    {},
+    AxeroConfig,
     AxeroCredentialJson
   >[] = connectorIndexingStatuses.filter(
     (connectorIndexingStatus) =>
@@ -73,40 +73,32 @@ const Main = () => {
 
   return (
     <>
-      {popup}
       <Title className="mb-2 mt-6 ml-auto mr-auto">
-        Step 1: Provide your Credentials
+        Step 1: Provide Axero API Key
       </Title>
-
       {axeroCredential ? (
         <>
           <div className="flex mb-1 text-sm">
-            <Text className="my-auto">Existing API Key: </Text>
-            <Text className="ml-1 italic my-auto max-w-md truncate">
-              {axeroCredential.credential_json?.axero_api_token}
+            <Text className="my-auto">Existing Axero API Key: </Text>
+            <Text className="ml-1 italic my-auto">
+              {axeroCredential.credential_json.axero_api_token}
             </Text>
-            <button
-              className="ml-1 hover:bg-hover rounded p-1"
+            <Button
+              size="xs"
+              color="red"
+              className="ml-3 text-inverted"
               onClick={async () => {
-                if (axeroConnectorIndexingStatuses.length > 0) {
-                  setPopup({
-                    type: "error",
-                    message:
-                      "Must delete all connectors before deleting credentials",
-                  });
-                  return;
-                }
                 await adminDeleteCredential(axeroCredential.id);
                 refreshCredentials();
               }}
             >
               <TrashIcon />
-            </button>
+            </Button>
           </div>
         </>
       ) : (
         <>
-          <Text>
+          <p className="text-sm mb-4">
             To use the Axero connector, first follow the guide{" "}
             <a
               className="text-blue-500"
@@ -116,11 +108,12 @@ const Main = () => {
               here
             </a>{" "}
             to generate an API Key.
-          </Text>
-          <Card className="mt-4">
+          </p>
+          <Card>
             <CredentialForm<AxeroCredentialJson>
               formBody={
                 <>
+                  <TextFormField name="base_url" label="Axero Base URL:" />
                   <TextFormField
                     name="axero_api_token"
                     label="Axero API Key:"
@@ -129,11 +122,15 @@ const Main = () => {
                 </>
               }
               validationSchema={Yup.object().shape({
+                base_url: Yup.string().required(
+                  "Please enter the base URL of your Axero instance"
+                ),
                 axero_api_token: Yup.string().required(
-                  "Please enter your Axero API Key!"
+                  "Please enter your Axero API Token"
                 ),
               })}
               initialValues={{
+                base_url: "",
                 axero_api_token: "",
               }}
               onSubmit={(isSuccess) => {
@@ -147,80 +144,94 @@ const Main = () => {
       )}
 
       <Title className="mb-2 mt-6 ml-auto mr-auto">
-        Step 2: Start indexing
+        Step 2: Which spaces do you want to connect?
       </Title>
-      {axeroCredential ? (
+
+      {axeroConnectorIndexingStatuses.length > 0 && (
         <>
-          {axeroConnectorIndexingStatuses.length > 0 ? (
-            <>
-              <Text className="mb-2">
-                We pull the latest <i>Articles</i>, <i>Blogs</i>, and{" "}
-                <i>Wikis</i> every <b>10</b> minutes.
-              </Text>
-              <div className="mb-2">
-                <ConnectorsTable<{}, AxeroCredentialJson>
-                  connectorIndexingStatuses={axeroConnectorIndexingStatuses}
-                  liveCredential={axeroCredential}
-                  getCredential={(credential) => {
-                    return (
-                      <div>
-                        <p>{credential.credential_json.axero_api_token}</p>
-                      </div>
-                    );
-                  }}
-                  onCredentialLink={async (connectorId) => {
-                    if (axeroCredential) {
-                      await linkCredential(connectorId, axeroCredential.id);
-                      mutate("/api/manage/admin/connector/indexing-status");
-                    }
-                  }}
-                  onUpdate={() =>
-                    mutate("/api/manage/admin/connector/indexing-status")
-                  }
-                />
-              </div>
-            </>
-          ) : (
-            <Card className="mt-4">
-              <h2 className="font-bold mb-3">Create Connector</h2>
-              <p className="text-sm mb-4">
-                Press connect below to start the connection Axero. We pull the
-                latest <i>Articles</i>, <i>Blogs</i>, and <i>Wikis</i> every{" "}
-                <b>10</b> minutes.
-              </p>
-              <ConnectorForm<{}>
-                nameBuilder={() => "AxeroConnector"}
-                ccPairNameBuilder={() => "Axero"}
-                source="axero"
-                inputType="poll"
-                formBody={
-                  <>
-                    <TextFormField
-                      name="base_url"
-                      label="Axero Base URL"
-                      subtext="The base URL you use to visit Axero."
-                      placeholder="E.g. https://my-company.axero.com"
-                    />
-                  </>
-                }
-                validationSchema={Yup.object().shape({})}
-                initialValues={{
-                  base_url: "",
-                }}
-                refreshFreq={10 * 60} // 10 minutes
-                credentialId={axeroCredential.id}
-              />
-            </Card>
-          )}
-        </>
-      ) : (
-        <>
-          <Text>
-            Please provide your access token in Step 1 first! Once done with
-            that, you can then start indexing Linear.
+          <Text className="mb-2">
+            We pull the latest <i>Articles</i>, <i>Blogs</i>, and <i>Wikis</i>{" "}
+            every <b>10</b> minutes.
           </Text>
+          <div className="mb-2">
+            <ConnectorsTable<AxeroConfig, AxeroCredentialJson>
+              connectorIndexingStatuses={axeroConnectorIndexingStatuses}
+              liveCredential={axeroCredential}
+              getCredential={(credential) =>
+                credential.credential_json.axero_api_token
+              }
+              specialColumns={[
+                {
+                  header: "Space",
+                  key: "spaces",
+                  getValue: (ccPairStatus) => {
+                    const connectorConfig =
+                      ccPairStatus.connector.connector_specific_config;
+                    return connectorConfig.spaces &&
+                      connectorConfig.spaces.length > 0
+                      ? connectorConfig.spaces.join(", ")
+                      : "";
+                  },
+                },
+              ]}
+              onUpdate={() =>
+                mutate("/api/manage/admin/connector/indexing-status")
+              }
+              onCredentialLink={async (connectorId) => {
+                if (axeroCredential) {
+                  await linkCredential(connectorId, axeroCredential.id);
+                  mutate("/api/manage/admin/connector/indexing-status");
+                }
+              }}
+            />
+          </div>
+          <Divider />
         </>
       )}
+
+      {axeroCredential ? (
+        <Card>
+          <h2 className="font-bold mb-3">Configure an Axero Connector</h2>
+          <ConnectorForm<AxeroConfig>
+            nameBuilder={(values) =>
+              values.spaces
+                ? `AxeroConnector-${values.spaces.join("_")}`
+                : `AxeroConnector`
+            }
+            source="axero"
+            inputType="poll"
+            formBodyBuilder={(values) => {
+              return (
+                <>
+                  <Divider />
+                  {TextArrayFieldBuilder({
+                    name: "spaces",
+                    label: "Space IDs:",
+                    subtext: `
+                      Specify zero or more Spaces to index (by the Space IDs). If no Space IDs
+                      are specified, all Spaces will be indexed.`,
+                  })(values)}
+                </>
+              );
+            }}
+            validationSchema={Yup.object().shape({
+              spaces: Yup.array()
+                .of(Yup.string().required("Space Ids cannot be empty"))
+                .required(),
+            })}
+            initialValues={{
+              spaces: [],
+            }}
+            refreshFreq={10 * 60} // 10 minutes
+            credentialId={axeroCredential.id}
+          />
+        </Card>
+      ) : (
+        <Text>
+          Please provide your Axero API Token in Step 1 first! Once done with
+          that, you can then specify which spaces you want to connect.
+        </Text>
+      )}
     </>
   );
 };
@@ -234,7 +245,7 @@ export default function Page() {
 
       <AdminPageTitle icon={<AxeroIcon size={32} />} title="Axero" />
 
-      <Main />
+      <MainSection />
     </div>
   );
 }

web/src/lib/types.ts

@@ -113,6 +113,10 @@ export interface SharepointConfig {
   sites?: string[];
 }
 
+export interface AxeroConfig {
+  spaces?: string[];
+}
+
 export interface ProductboardConfig {}
 
 export interface SlackConfig {
@@ -329,6 +333,7 @@ export interface SharepointCredentialJson {
 }
 
 export interface AxeroCredentialJson {
+  base_url: string;
   axero_api_token: string;
 }