From 783696a6714af909f1fe830bc09633ad6baa7d75 Mon Sep 17 00:00:00 2001 From: Yuhong Sun Date: Sun, 31 Mar 2024 14:45:20 -0700 Subject: [PATCH] Axero Spaces (#1276) --- ...er-build-push-backend-container-on-tag.yml | 1 + backend/danswer/connectors/axero/connector.py | 61 +++-- web/src/app/admin/connectors/axero/page.tsx | 241 +++++++++--------- web/src/lib/types.ts | 5 + 4 files changed, 172 insertions(+), 136 deletions(-) diff --git a/.github/workflows/docker-build-push-backend-container-on-tag.yml b/.github/workflows/docker-build-push-backend-container-on-tag.yml index e95c143fb49..82aa24e6ab5 100644 --- a/.github/workflows/docker-build-push-backend-container-on-tag.yml +++ b/.github/workflows/docker-build-push-backend-container-on-tag.yml @@ -38,5 +38,6 @@ jobs: - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master with: + # To run locally: trivy image --severity HIGH,CRITICAL danswer/danswer-backend image-ref: docker.io/danswer/danswer-backend:${{ github.ref_name }} severity: 'CRITICAL,HIGH' diff --git a/backend/danswer/connectors/axero/connector.py b/backend/danswer/connectors/axero/connector.py index e19fb39eb9c..fcb43955892 100644 --- a/backend/danswer/connectors/axero/connector.py +++ b/backend/danswer/connectors/axero/connector.py @@ -38,6 +38,7 @@ def _get_entities( axero_base_url: str, start: datetime, end: datetime, + space_id: str | None = None, ) -> list[dict]: endpoint = axero_base_url + "api/content/list" page_num = 1 @@ -51,6 +52,10 @@ def _get_entities( "SortOrder": "1", # descending "StartPage": str(page_num), } + + if space_id is not None: + params["SpaceID"] = space_id + res = requests.get(endpoint, headers=_get_auth_header(api_key), params=params) res.raise_for_status() @@ -116,7 +121,8 @@ def _translate_content_to_doc(content: dict) -> Document: class AxeroConnector(PollConnector): def __init__( self, - base_url: str, + # Strings of the integer ids of the spaces + spaces: list[str] | None = None, include_article: bool = True, include_blog: bool = True, include_wiki: bool = True, @@ -129,20 +135,24 @@ class AxeroConnector(PollConnector): self.include_wiki = include_wiki self.include_forum = include_forum self.batch_size = batch_size + self.space_ids = spaces self.axero_key = None - - if not base_url.endswith("/"): - base_url += "/" - self.base_url = base_url + self.base_url = None def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None: self.axero_key = credentials["axero_api_token"] + # As the API key specifically applies to a particular deployment, this is + # included as part of the credential + base_url = credentials["base_url"] + if not base_url.endswith("/"): + base_url += "/" + self.base_url = base_url return None def poll_source( self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch ) -> GenerateDocumentsOutput: - if not self.axero_key: + if not self.axero_key or not self.base_url: raise ConnectorMissingCredentialError("Axero") start_datetime = datetime.utcfromtimestamp(start).replace(tzinfo=timezone.utc) @@ -158,26 +168,35 @@ class AxeroConnector(PollConnector): if self.include_forum: raise NotImplementedError("Forums for Axero not supported currently") - for entity in entity_types: - articles = _get_entities( - entity_type=entity, - api_key=self.axero_key, - axero_base_url=self.base_url, - start=start_datetime, - end=end_datetime, - ) - yield from process_in_batches( - objects=articles, - process_function=_translate_content_to_doc, - batch_size=self.batch_size, - ) + iterable_space_ids = self.space_ids if self.space_ids else [None] + + for space_id in iterable_space_ids: + for entity in entity_types: + axero_obj = _get_entities( + entity_type=entity, + api_key=self.axero_key, + axero_base_url=self.base_url, + start=start_datetime, + end=end_datetime, + space_id=space_id, + ) + yield from process_in_batches( + objects=axero_obj, + process_function=_translate_content_to_doc, + batch_size=self.batch_size, + ) if __name__ == "__main__": import os - connector = AxeroConnector(base_url=os.environ["AXERO_BASE_URL"]) - connector.load_credentials({"axero_api_token": os.environ["AXERO_API_TOKEN"]}) + connector = AxeroConnector() + connector.load_credentials( + { + "axero_api_token": os.environ["AXERO_API_TOKEN"], + "base_url": os.environ["AXERO_BASE_URL"], + } + ) current = time.time() one_year_ago = current - 24 * 60 * 60 * 360 diff --git a/web/src/app/admin/connectors/axero/page.tsx b/web/src/app/admin/connectors/axero/page.tsx index 532345d8320..b434f8528d0 100644 --- a/web/src/app/admin/connectors/axero/page.tsx +++ b/web/src/app/admin/connectors/axero/page.tsx @@ -1,30 +1,32 @@ "use client"; import * as Yup from "yup"; -import { AxeroIcon, LinearIcon, TrashIcon } from "@/components/icons/icons"; -import { TextFormField } from "@/components/admin/connectors/Field"; +import { AxeroIcon, TrashIcon } from "@/components/icons/icons"; +import { fetcher } from "@/lib/fetcher"; +import useSWR, { useSWRConfig } from "swr"; +import { LoadingAnimation } from "@/components/Loading"; import { HealthCheckBanner } from "@/components/health/healthcheck"; +import { + AxeroConfig, + AxeroCredentialJson, + ConnectorIndexingStatus, + Credential, +} from "@/lib/types"; +import { adminDeleteCredential, linkCredential } from "@/lib/credential"; import { CredentialForm } from "@/components/admin/connectors/CredentialForm"; import { - Credential, - ConnectorIndexingStatus, - LinearCredentialJson, - AxeroCredentialJson, -} from "@/lib/types"; -import useSWR, { useSWRConfig } from "swr"; -import { fetcher } from "@/lib/fetcher"; -import { LoadingAnimation } from "@/components/Loading"; -import { adminDeleteCredential, linkCredential } from "@/lib/credential"; -import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; + TextFormField, + TextArrayFieldBuilder, + BooleanFormField, + TextArrayField, +} from "@/components/admin/connectors/Field"; import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable"; -import { usePopup } from "@/components/admin/connectors/Popup"; +import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; import { usePublicCredentials } from "@/lib/hooks"; -import { Card, Text, Title } from "@tremor/react"; +import { Button, Card, Divider, Text, Title } from "@tremor/react"; import { AdminPageTitle } from "@/components/admin/Title"; -const Main = () => { - const { popup, setPopup } = usePopup(); - +const MainSection = () => { const { mutate } = useSWRConfig(); const { data: connectorIndexingStatuses, @@ -32,21 +34,19 @@ const Main = () => { error: isConnectorIndexingStatusesError, } = useSWR[]>( "/api/manage/admin/connector/indexing-status", - fetcher, - { refreshInterval: 5000 } // 5 seconds + fetcher ); + const { data: credentialsData, isLoading: isCredentialsLoading, error: isCredentialsError, - isValidating: isCredentialsValidating, refreshCredentials, } = usePublicCredentials(); if ( - isConnectorIndexingStatusesLoading || - isCredentialsLoading || - isCredentialsValidating + (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) || + (!credentialsData && isCredentialsLoading) ) { return ; } @@ -60,7 +60,7 @@ const Main = () => { } const axeroConnectorIndexingStatuses: ConnectorIndexingStatus< - {}, + AxeroConfig, AxeroCredentialJson >[] = connectorIndexingStatuses.filter( (connectorIndexingStatus) => @@ -73,40 +73,32 @@ const Main = () => { return ( <> - {popup} - Step 1: Provide your Credentials + Step 1: Provide Axero API Key - {axeroCredential ? ( <>
- Existing API Key: - - {axeroCredential.credential_json?.axero_api_token} + Existing Axero API Key: + + {axeroCredential.credential_json.axero_api_token} - +
) : ( <> - +

To use the Axero connector, first follow the guide{" "} { here {" "} to generate an API Key. - - +

+ formBody={ <> + { } validationSchema={Yup.object().shape({ + base_url: Yup.string().required( + "Please enter the base URL of your Axero instance" + ), axero_api_token: Yup.string().required( - "Please enter your Axero API Key!" + "Please enter your Axero API Token" ), })} initialValues={{ + base_url: "", axero_api_token: "", }} onSubmit={(isSuccess) => { @@ -147,80 +144,94 @@ const Main = () => { )} - Step 2: Start indexing + Step 2: Which spaces do you want to connect? - {axeroCredential ? ( + + {axeroConnectorIndexingStatuses.length > 0 && ( <> - {axeroConnectorIndexingStatuses.length > 0 ? ( - <> - - We pull the latest Articles, Blogs, and{" "} - Wikis every 10 minutes. - -
- - connectorIndexingStatuses={axeroConnectorIndexingStatuses} - liveCredential={axeroCredential} - getCredential={(credential) => { - return ( -
-

{credential.credential_json.axero_api_token}

-
- ); - }} - onCredentialLink={async (connectorId) => { - if (axeroCredential) { - await linkCredential(connectorId, axeroCredential.id); - mutate("/api/manage/admin/connector/indexing-status"); - } - }} - onUpdate={() => - mutate("/api/manage/admin/connector/indexing-status") - } - /> -
- - ) : ( - -

Create Connector

-

- Press connect below to start the connection Axero. We pull the - latest Articles, Blogs, and Wikis every{" "} - 10 minutes. -

- - nameBuilder={() => "AxeroConnector"} - ccPairNameBuilder={() => "Axero"} - source="axero" - inputType="poll" - formBody={ - <> - - - } - validationSchema={Yup.object().shape({})} - initialValues={{ - base_url: "", - }} - refreshFreq={10 * 60} // 10 minutes - credentialId={axeroCredential.id} - /> - - )} - - ) : ( - <> - - Please provide your access token in Step 1 first! Once done with - that, you can then start indexing Linear. + + We pull the latest Articles, Blogs, and Wikis{" "} + every 10 minutes. +
+ + connectorIndexingStatuses={axeroConnectorIndexingStatuses} + liveCredential={axeroCredential} + getCredential={(credential) => + credential.credential_json.axero_api_token + } + specialColumns={[ + { + header: "Space", + key: "spaces", + getValue: (ccPairStatus) => { + const connectorConfig = + ccPairStatus.connector.connector_specific_config; + return connectorConfig.spaces && + connectorConfig.spaces.length > 0 + ? connectorConfig.spaces.join(", ") + : ""; + }, + }, + ]} + onUpdate={() => + mutate("/api/manage/admin/connector/indexing-status") + } + onCredentialLink={async (connectorId) => { + if (axeroCredential) { + await linkCredential(connectorId, axeroCredential.id); + mutate("/api/manage/admin/connector/indexing-status"); + } + }} + /> +
+ )} + + {axeroCredential ? ( + +

Configure an Axero Connector

+ + nameBuilder={(values) => + values.spaces + ? `AxeroConnector-${values.spaces.join("_")}` + : `AxeroConnector` + } + source="axero" + inputType="poll" + formBodyBuilder={(values) => { + return ( + <> + + {TextArrayFieldBuilder({ + name: "spaces", + label: "Space IDs:", + subtext: ` + Specify zero or more Spaces to index (by the Space IDs). If no Space IDs + are specified, all Spaces will be indexed.`, + })(values)} + + ); + }} + validationSchema={Yup.object().shape({ + spaces: Yup.array() + .of(Yup.string().required("Space Ids cannot be empty")) + .required(), + })} + initialValues={{ + spaces: [], + }} + refreshFreq={10 * 60} // 10 minutes + credentialId={axeroCredential.id} + /> + + ) : ( + + Please provide your Axero API Token in Step 1 first! Once done with + that, you can then specify which spaces you want to connect. + + )} ); }; @@ -234,7 +245,7 @@ export default function Page() { } title="Axero" /> -
+ ); } diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index f06e4515a29..d09ad6c9063 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -113,6 +113,10 @@ export interface SharepointConfig { sites?: string[]; } +export interface AxeroConfig { + spaces?: string[]; +} + export interface ProductboardConfig {} export interface SlackConfig { @@ -329,6 +333,7 @@ export interface SharepointCredentialJson { } export interface AxeroCredentialJson { + base_url: string; axero_api_token: string; }