mirror of
https://github.com/danswer-ai/danswer.git
synced 2025-09-27 12:29:41 +02:00
Add MediaWiki and Wikipedia Connectors (#1250)
* Add MediaWikiConnector first draft * Add MediaWikiConnector first draft * Add MediaWikiConnector first draft * Add MediaWikiConnector sections for each document * Add MediaWikiConnector to constants and factory * Integrate MediaWikiConnector with connectors page * Unit tests + bug fixes * Allow adding multiple mediawikiconnectors * add wikipedia connector * add wikipedia connector to factory * improve docstrings of mediawiki connector backend * improve docstrings of mediawiki connector backend * move wikipedia and mediawiki icon locations in admin page * undo accidental commit of modified docker compose yaml
This commit is contained in:
Andrew Sansom
GitHub
43
web/public/MediaWiki.svg
Normal file
43
web/public/MediaWiki.svg
Normal file
@@ -0,0 +1,43 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="100" height="100" viewBox="0 0 100 100">
|
||||
<defs>
|
||||
<linearGradient id="gradient" gradientUnits="userSpaceOnUse" x1="0" x2="100" y1="100" y2="0">
|
||||
<stop offset="0%" stop-color="#0a00b2"/>
|
||||
<stop offset="50%" stop-color="#ff0000"/>
|
||||
<stop offset="100%" stop-color="#fffc00"/>
|
||||
</linearGradient>
|
||||
<style>
|
||||
.petal {
|
||||
opacity: 0.65;
|
||||
}
|
||||
.petals {
|
||||
fill: url(#gradient);
|
||||
}
|
||||
</style>
|
||||
</defs>
|
||||
<g id="petals" class="petals">
|
||||
<path class="petal" d="M33.6,14.8a16.938,16.938,0,0,0,3.116,11.142,11.457,11.457,0,0,0,6.858,4.3,3.033,3.033,0,0,0,2.385-.713,11.924,11.924,0,0,0,3.634-10.837c-1.257-7.54-6.19-12.43-12.4-17A32.468,32.468,0,0,0,33.6,14.8Z"/>
|
||||
<path class="petal" d="M25.117,20.232a16.938,16.938,0,0,0,5.893,9.956,11.457,11.457,0,0,0,7.738,2.381,3.033,3.033,0,0,0,2.119-1.306,11.924,11.924,0,0,0,.705-11.409C38.406,12.9,32.376,9.449,25.2,6.642A32.468,32.468,0,0,0,25.117,20.232Z"/>
|
||||
<path class="petal" d="M18.329,27.677A16.937,16.937,0,0,0,26.6,35.769a11.457,11.457,0,0,0,8.09.3,3.033,3.033,0,0,0,1.709-1.81,11.924,11.924,0,0,0-2.271-11.2c-4.859-5.9-11.576-7.67-19.237-8.523A32.466,32.466,0,0,0,18.329,27.677Z"/>
|
||||
<path class="petal" d="M13.7,36.626A16.938,16.938,0,0,0,23.781,42.3a11.457,11.457,0,0,0,7.891-1.807A3.033,3.033,0,0,0,32.854,38.3a11.924,11.924,0,0,0-5.093-10.233c-6.221-4.443-13.167-4.412-20.787-3.254A32.468,32.468,0,0,0,13.7,36.626Z"/>
|
||||
<path class="petal" d="M11.543,46.468a16.938,16.938,0,0,0,11.208,2.873,11.457,11.457,0,0,0,7.155-3.788,3.034,3.034,0,0,0,.575-2.422,11.924,11.924,0,0,0-7.568-8.566C15.753,31.884,9.052,33.711,1.99,36.8A32.468,32.468,0,0,0,11.543,46.468Z"/>
|
||||
<path class="petal" d="M12.008,56.532a16.938,16.938,0,0,0,11.569-.126,11.457,11.457,0,0,0,5.931-5.51,3.033,3.033,0,0,0-.072-2.488,11.924,11.924,0,0,0-9.527-6.315C12.3,41.356,6.3,44.855.279,49.669A32.467,32.467,0,0,0,12.008,56.532Z"/>
|
||||
<path class="petal" d="M15.062,66.134A16.938,16.938,0,0,0,26.2,63.018a11.457,11.457,0,0,0,4.3-6.858,3.033,3.033,0,0,0-.713-2.385,11.924,11.924,0,0,0-10.837-3.634c-7.54,1.257-12.43,6.19-17,12.4A32.468,32.468,0,0,0,15.062,66.134Z"/>
|
||||
<path class="petal" d="M20.5,74.618a16.938,16.938,0,0,0,9.956-5.893,11.457,11.457,0,0,0,2.381-7.738,3.033,3.033,0,0,0-1.306-2.119,11.925,11.925,0,0,0-11.409-.705c-6.958,3.166-10.4,9.2-13.212,16.376A32.466,32.466,0,0,0,20.5,74.618Z"/>
|
||||
<path class="petal" d="M27.943,81.406a16.938,16.938,0,0,0,8.092-8.269,11.457,11.457,0,0,0,.3-8.09,3.033,3.033,0,0,0-1.81-1.709,11.924,11.924,0,0,0-11.2,2.271c-5.9,4.859-7.67,11.576-8.523,19.237A32.467,32.467,0,0,0,27.943,81.406Z"/>
|
||||
<path class="petal" d="M36.891,86.035a16.938,16.938,0,0,0,5.676-10.082,11.457,11.457,0,0,0-1.807-7.891,3.033,3.033,0,0,0-2.191-1.182,11.924,11.924,0,0,0-10.233,5.094c-4.443,6.221-4.412,13.167-3.254,20.787A32.467,32.467,0,0,0,36.891,86.035Z"/>
|
||||
<path class="petal" d="M46.733,88.191a16.937,16.937,0,0,0,2.873-11.207,11.457,11.457,0,0,0-3.788-7.155,3.033,3.033,0,0,0-2.422-.575,11.924,11.924,0,0,0-8.566,7.568c-2.681,7.159-.854,13.86,2.237,20.921A32.465,32.465,0,0,0,46.733,88.191Z"/>
|
||||
<path class="petal" d="M56.8,87.726a16.937,16.937,0,0,0-.125-11.569,11.457,11.457,0,0,0-5.511-5.931,3.033,3.033,0,0,0-2.488.072,11.924,11.924,0,0,0-6.315,9.528c-0.737,7.609,2.762,13.609,7.576,19.629A32.466,32.466,0,0,0,56.8,87.726Z"/>
|
||||
<path class="petal" d="M66.4,84.672A16.938,16.938,0,0,0,63.284,73.53a11.457,11.457,0,0,0-6.858-4.3,3.033,3.033,0,0,0-2.385.713,11.924,11.924,0,0,0-3.634,10.837c1.257,7.54,6.19,12.43,12.4,17A32.468,32.468,0,0,0,66.4,84.672Z"/>
|
||||
<path class="petal" d="M74.883,79.237a16.937,16.937,0,0,0-5.893-9.956A11.456,11.456,0,0,0,61.252,66.9a3.033,3.033,0,0,0-2.119,1.306,11.924,11.924,0,0,0-.705,11.409c3.166,6.958,9.2,10.4,16.375,13.212A32.468,32.468,0,0,0,74.883,79.237Z"/>
|
||||
<path class="petal" d="M81.671,71.792A16.938,16.938,0,0,0,73.4,63.7a11.457,11.457,0,0,0-8.09-.3,3.033,3.033,0,0,0-1.708,1.81,11.924,11.924,0,0,0,2.271,11.2c4.859,5.9,11.576,7.67,19.237,8.523A32.466,32.466,0,0,0,81.671,71.792Z"/>
|
||||
<path class="petal" d="M86.3,62.843a16.938,16.938,0,0,0-10.082-5.676,11.457,11.457,0,0,0-7.891,1.807,3.033,3.033,0,0,0-1.182,2.191A11.924,11.924,0,0,0,72.239,71.4c6.221,4.443,13.167,4.412,20.787,3.254A32.467,32.467,0,0,0,86.3,62.843Z"/>
|
||||
<path class="petal" d="M88.457,53a16.938,16.938,0,0,0-11.207-2.873,11.457,11.457,0,0,0-7.155,3.788,3.033,3.033,0,0,0-.574,2.422A11.925,11.925,0,0,0,77.088,64.9c7.158,2.681,13.86.854,20.921-2.237A32.467,32.467,0,0,0,88.457,53Z"/>
|
||||
<path class="petal" d="M87.992,42.936a16.938,16.938,0,0,0-11.569.126,11.457,11.457,0,0,0-5.931,5.511,3.033,3.033,0,0,0,.072,2.488,11.924,11.924,0,0,0,9.527,6.315C87.7,58.113,93.7,54.614,99.721,49.8A32.468,32.468,0,0,0,87.992,42.936Z"/>
|
||||
<path class="petal" d="M84.938,33.335A16.938,16.938,0,0,0,73.8,36.451a11.457,11.457,0,0,0-4.3,6.858,3.033,3.033,0,0,0,.714,2.385,11.924,11.924,0,0,0,10.837,3.634c7.54-1.257,12.43-6.19,17-12.4A32.466,32.466,0,0,0,84.938,33.335Z"/>
|
||||
<path class="petal" d="M79.5,24.851a16.938,16.938,0,0,0-9.956,5.893,11.457,11.457,0,0,0-2.381,7.738A3.033,3.033,0,0,0,68.472,40.6a11.924,11.924,0,0,0,11.409.705c6.958-3.166,10.4-9.2,13.212-16.375A32.468,32.468,0,0,0,79.5,24.851Z"/>
|
||||
<path class="petal" d="M72.057,18.063a16.938,16.938,0,0,0-8.092,8.269,11.457,11.457,0,0,0-.3,8.09,3.033,3.033,0,0,0,1.81,1.709,11.924,11.924,0,0,0,11.2-2.271c5.9-4.859,7.67-11.576,8.523-19.237A32.467,32.467,0,0,0,72.057,18.063Z"/>
|
||||
<path class="petal" d="M63.109,13.434a16.937,16.937,0,0,0-5.676,10.082,11.457,11.457,0,0,0,1.807,7.891,3.033,3.033,0,0,0,2.191,1.182A11.924,11.924,0,0,0,71.664,27.5c4.443-6.221,4.412-13.167,3.254-20.787A32.466,32.466,0,0,0,63.109,13.434Z"/>
|
||||
<path class="petal" d="M53.267,11.278a16.937,16.937,0,0,0-2.873,11.207,11.456,11.456,0,0,0,3.788,7.155,3.033,3.033,0,0,0,2.422.575,11.924,11.924,0,0,0,8.566-7.568c2.681-7.159.854-13.86-2.237-20.921A32.466,32.466,0,0,0,53.267,11.278Z"/>
|
||||
<path class="petal" d="M43.2,11.743a16.938,16.938,0,0,0,.126,11.569,11.457,11.457,0,0,0,5.511,5.931,3.033,3.033,0,0,0,2.488-.072,11.924,11.924,0,0,0,6.315-9.528C58.379,12.034,54.88,6.034,50.066.014A32.468,32.468,0,0,0,43.2,11.743Z"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 6.0 KiB |
1
web/public/Wikipedia.svg
Normal file
1
web/public/Wikipedia.svg
Normal file
File diff suppressed because one or more lines are too long
208
web/src/app/admin/connectors/mediawiki/page.tsx
Normal file
208
web/src/app/admin/connectors/mediawiki/page.tsx
Normal file
@@ -0,0 +1,208 @@
|
||||
"use client";
|
||||
|
||||
import * as Yup from "yup";
|
||||
import { MediaWikiIcon, TrashIcon } from "@/components/icons/icons";
|
||||
import {
|
||||
TextArrayField,
|
||||
TextArrayFieldBuilder,
|
||||
TextFormField,
|
||||
} from "@/components/admin/connectors/Field";
|
||||
import { HealthCheckBanner } from "@/components/health/healthcheck";
|
||||
import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
|
||||
import {
|
||||
MediaWikiCredentialJson,
|
||||
MediaWikiConfig,
|
||||
ConnectorIndexingStatus,
|
||||
Credential,
|
||||
} from "@/lib/types";
|
||||
import useSWR, { useSWRConfig } from "swr";
|
||||
import { fetcher } from "@/lib/fetcher";
|
||||
import { LoadingAnimation } from "@/components/Loading";
|
||||
import { adminDeleteCredential, linkCredential } from "@/lib/credential";
|
||||
import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
|
||||
import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
|
||||
import { usePopup } from "@/components/admin/connectors/Popup";
|
||||
import { usePublicCredentials } from "@/lib/hooks";
|
||||
import { AdminPageTitle } from "@/components/admin/Title";
|
||||
import { Card, Text, Title } from "@tremor/react";
|
||||
|
||||
const Main = () => {
|
||||
const { popup, setPopup } = usePopup();
|
||||
|
||||
const { mutate } = useSWRConfig();
|
||||
const {
|
||||
data: connectorIndexingStatuses,
|
||||
isLoading: isConnectorIndexingStatusesLoading,
|
||||
error: isConnectorIndexingStatusesError,
|
||||
} = useSWR<ConnectorIndexingStatus<any, any>[]>(
|
||||
"/api/manage/admin/connector/indexing-status",
|
||||
fetcher
|
||||
);
|
||||
const {
|
||||
data: credentialsData,
|
||||
isLoading: isCredentialsLoading,
|
||||
error: isCredentialsError,
|
||||
refreshCredentials,
|
||||
} = usePublicCredentials();
|
||||
|
||||
if (
|
||||
(!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) ||
|
||||
(!credentialsData && isCredentialsLoading)
|
||||
) {
|
||||
return <LoadingAnimation text="Loading" />;
|
||||
}
|
||||
|
||||
if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) {
|
||||
return <div>Failed to load connectors</div>;
|
||||
}
|
||||
|
||||
if (isCredentialsError || !credentialsData) {
|
||||
return <div>Failed to load credentials</div>;
|
||||
}
|
||||
|
||||
const mediawikiConnectorIndexingStatuses: ConnectorIndexingStatus<
|
||||
MediaWikiConfig,
|
||||
MediaWikiCredentialJson
|
||||
>[] = connectorIndexingStatuses.filter(
|
||||
(connectorIndexingStatus) =>
|
||||
connectorIndexingStatus.connector.source === "mediawiki"
|
||||
);
|
||||
const mediawikiCredential: Credential<MediaWikiCredentialJson> | undefined =
|
||||
credentialsData.find((credential) => true);
|
||||
|
||||
return (
|
||||
<>
|
||||
{popup}
|
||||
{mediawikiConnectorIndexingStatuses.length > 0 && (
|
||||
<>
|
||||
<Title className="mb-2 mt-6 ml-auto mr-auto">
|
||||
MediaWiki indexing status
|
||||
</Title>
|
||||
<Text className="mb-2">
|
||||
The latest page, chapter, book and shelf changes are fetched every
|
||||
10 minutes.
|
||||
</Text>
|
||||
<div className="mb-2">
|
||||
<ConnectorsTable<MediaWikiConfig, MediaWikiCredentialJson>
|
||||
connectorIndexingStatuses={mediawikiConnectorIndexingStatuses}
|
||||
liveCredential={mediawikiCredential}
|
||||
getCredential={(credential) => {
|
||||
return <div></div>;
|
||||
}}
|
||||
onCredentialLink={async (connectorId) => {
|
||||
if (mediawikiCredential) {
|
||||
await linkCredential(connectorId, mediawikiCredential.id);
|
||||
mutate("/api/manage/admin/connector/indexing-status");
|
||||
}
|
||||
}}
|
||||
onUpdate={() =>
|
||||
mutate("/api/manage/admin/connector/indexing-status")
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{mediawikiCredential && (
|
||||
<>
|
||||
<Card className="mt-4">
|
||||
<h2 className="font-bold mb-3">Create Connection</h2>
|
||||
<Text className="mb-4">
|
||||
Press connect below to start the connection to your MediaWiki
|
||||
instance.
|
||||
</Text>
|
||||
<ConnectorForm<MediaWikiConfig>
|
||||
nameBuilder={(values) =>
|
||||
`MediaWikiConnector-${values.connector_name}`
|
||||
}
|
||||
ccPairNameBuilder={(values) =>
|
||||
`MediaWikiConnector-${values.connector_name}`
|
||||
}
|
||||
source="mediawiki"
|
||||
inputType="poll"
|
||||
formBodyBuilder={(values) => (
|
||||
<div>
|
||||
<TextFormField
|
||||
name="connector_name"
|
||||
label="Connector Name:"
|
||||
/>
|
||||
<TextFormField name="hostname" label="MediaWiki Site URL:" />
|
||||
<TextFormField
|
||||
name="language_code"
|
||||
label="MediaWiki Site Language Code (e.g. 'en', 'sp', etc...):"
|
||||
/>
|
||||
{TextArrayFieldBuilder({
|
||||
name: "pages",
|
||||
label: "Pages to index:",
|
||||
subtext:
|
||||
"Specify 0 or more names of pages to index. Only specify the name of the page, not its url.",
|
||||
})(values)}
|
||||
{TextArrayFieldBuilder({
|
||||
name: "categories",
|
||||
label: "Categories to index:",
|
||||
subtext:
|
||||
"Specify 0 or more names of categories to index. For most MediaWiki sites, these are pages" +
|
||||
" with a name of the form 'Category: XYZ', that are lists of other pages/categories. Only" +
|
||||
" specify the name of the category, not its url.",
|
||||
})(values)}
|
||||
<TextFormField
|
||||
name="recurse_depth"
|
||||
label="Recursion Depth:"
|
||||
type="number"
|
||||
subtext="When indexing categories that have sub-categories, this will determine how may levels to index. Specify 0 to only index the category itself (i.e. no recursion). Specify -1 for unlimited recursion depth. Note, that in some rare instances, a category might contain itself in its dependencies, which will cause an infinite loop. Only use -1 if you confident that this will not happen."
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
validationSchema={Yup.object().shape({
|
||||
connector_name: Yup.string().required(
|
||||
"Please enter a name for your MediaWiki connector."
|
||||
),
|
||||
hostname: Yup.string().required(
|
||||
"Please enter the base URL for your MediaWiki site"
|
||||
),
|
||||
language_code: Yup.string().default("en"),
|
||||
categories: Yup.array().of(
|
||||
Yup.string().required(
|
||||
"Please enter categories to index from your MediaWiki site"
|
||||
)
|
||||
),
|
||||
pages: Yup.array().of(
|
||||
Yup.string().required(
|
||||
"Please enter pages to index from your MediaWiki site"
|
||||
)
|
||||
),
|
||||
recurse_depth: Yup.number().required(
|
||||
"Please enter the recursion depth for your MediaWiki site."
|
||||
),
|
||||
})}
|
||||
initialValues={{
|
||||
connector_name: "",
|
||||
hostname: "",
|
||||
language_code: "en",
|
||||
categories: [],
|
||||
pages: [],
|
||||
recurse_depth: 0,
|
||||
}}
|
||||
refreshFreq={10 * 60} // 10 minutes
|
||||
credentialId={mediawikiCredential.id}
|
||||
/>
|
||||
</Card>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default function Page() {
|
||||
return (
|
||||
<div className="mx-auto container">
|
||||
<div className="mb-4">
|
||||
<HealthCheckBanner />
|
||||
</div>
|
||||
|
||||
<AdminPageTitle icon={<MediaWikiIcon size={32} />} title="MediaWiki" />
|
||||
|
||||
<Main />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
203
web/src/app/admin/connectors/wikipedia/page.tsx
Normal file
203
web/src/app/admin/connectors/wikipedia/page.tsx
Normal file
@@ -0,0 +1,203 @@
|
||||
"use client";
|
||||
|
||||
import * as Yup from "yup";
|
||||
import { WikipediaIcon, TrashIcon } from "@/components/icons/icons";
|
||||
import {
|
||||
TextArrayField,
|
||||
TextArrayFieldBuilder,
|
||||
TextFormField,
|
||||
} from "@/components/admin/connectors/Field";
|
||||
import { HealthCheckBanner } from "@/components/health/healthcheck";
|
||||
import { CredentialForm } from "@/components/admin/connectors/CredentialForm";
|
||||
import {
|
||||
WikipediaCredentialJson,
|
||||
WikipediaConfig,
|
||||
ConnectorIndexingStatus,
|
||||
Credential,
|
||||
} from "@/lib/types";
|
||||
import useSWR, { useSWRConfig } from "swr";
|
||||
import { fetcher } from "@/lib/fetcher";
|
||||
import { LoadingAnimation } from "@/components/Loading";
|
||||
import { adminDeleteCredential, linkCredential } from "@/lib/credential";
|
||||
import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
|
||||
import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
|
||||
import { usePopup } from "@/components/admin/connectors/Popup";
|
||||
import { usePublicCredentials } from "@/lib/hooks";
|
||||
import { AdminPageTitle } from "@/components/admin/Title";
|
||||
import { Card, Text, Title } from "@tremor/react";
|
||||
|
||||
const Main = () => {
|
||||
const { popup, setPopup } = usePopup();
|
||||
|
||||
const { mutate } = useSWRConfig();
|
||||
const {
|
||||
data: connectorIndexingStatuses,
|
||||
isLoading: isConnectorIndexingStatusesLoading,
|
||||
error: isConnectorIndexingStatusesError,
|
||||
} = useSWR<ConnectorIndexingStatus<any, any>[]>(
|
||||
"/api/manage/admin/connector/indexing-status",
|
||||
fetcher
|
||||
);
|
||||
const {
|
||||
data: credentialsData,
|
||||
isLoading: isCredentialsLoading,
|
||||
error: isCredentialsError,
|
||||
refreshCredentials,
|
||||
} = usePublicCredentials();
|
||||
|
||||
if (
|
||||
(!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) ||
|
||||
(!credentialsData && isCredentialsLoading)
|
||||
) {
|
||||
return <LoadingAnimation text="Loading" />;
|
||||
}
|
||||
|
||||
if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) {
|
||||
return <div>Failed to load connectors</div>;
|
||||
}
|
||||
|
||||
if (isCredentialsError || !credentialsData) {
|
||||
return <div>Failed to load credentials</div>;
|
||||
}
|
||||
|
||||
const wikipediaConnectorIndexingStatuses: ConnectorIndexingStatus<
|
||||
WikipediaConfig,
|
||||
WikipediaCredentialJson
|
||||
>[] = connectorIndexingStatuses.filter(
|
||||
(connectorIndexingStatus) =>
|
||||
connectorIndexingStatus.connector.source === "wikipedia"
|
||||
);
|
||||
const wikipediaCredential: Credential<WikipediaCredentialJson> | undefined =
|
||||
credentialsData.find((credential) => true);
|
||||
|
||||
return (
|
||||
<>
|
||||
{popup}
|
||||
{wikipediaConnectorIndexingStatuses.length > 0 && (
|
||||
<>
|
||||
<Title className="mb-2 mt-6 ml-auto mr-auto">
|
||||
Wikipedia indexing status
|
||||
</Title>
|
||||
<Text className="mb-2">
|
||||
The latest page, chapter, book and shelf changes are fetched every
|
||||
10 minutes.
|
||||
</Text>
|
||||
<div className="mb-2">
|
||||
<ConnectorsTable<WikipediaConfig, WikipediaCredentialJson>
|
||||
connectorIndexingStatuses={wikipediaConnectorIndexingStatuses}
|
||||
liveCredential={wikipediaCredential}
|
||||
getCredential={(credential) => {
|
||||
return <div></div>;
|
||||
}}
|
||||
onCredentialLink={async (connectorId) => {
|
||||
if (wikipediaCredential) {
|
||||
await linkCredential(connectorId, wikipediaCredential.id);
|
||||
mutate("/api/manage/admin/connector/indexing-status");
|
||||
}
|
||||
}}
|
||||
onUpdate={() =>
|
||||
mutate("/api/manage/admin/connector/indexing-status")
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{wikipediaCredential && (
|
||||
<>
|
||||
<Card className="mt-4">
|
||||
<h2 className="font-bold mb-3">Create Connection</h2>
|
||||
<Text className="mb-4">
|
||||
Press connect below to start the connection to your Wikipedia
|
||||
instance.
|
||||
</Text>
|
||||
<ConnectorForm<WikipediaConfig>
|
||||
nameBuilder={(values) =>
|
||||
`WikipediaConnector-${values.connector_name}`
|
||||
}
|
||||
ccPairNameBuilder={(values) =>
|
||||
`WikipediaConnector-${values.connector_name}`
|
||||
}
|
||||
source="wikipedia"
|
||||
inputType="poll"
|
||||
formBodyBuilder={(values) => (
|
||||
<div>
|
||||
<TextFormField
|
||||
name="connector_name"
|
||||
label="Connector Name:"
|
||||
/>
|
||||
<TextFormField
|
||||
name="language_code"
|
||||
label="Wikipedia Site Language Code (e.g. 'en', 'sp', etc...):"
|
||||
/>
|
||||
{TextArrayFieldBuilder({
|
||||
name: "pages",
|
||||
label: "Pages to index:",
|
||||
subtext:
|
||||
"Specify 0 or more names of pages to index. Only specify the name of the page, not its url.",
|
||||
})(values)}
|
||||
{TextArrayFieldBuilder({
|
||||
name: "categories",
|
||||
label: "Categories to index:",
|
||||
subtext:
|
||||
"Specify 0 or more names of categories to index. These are pages" +
|
||||
" with a name of the form 'Category: XYZ', that are lists of other pages/categories. Only" +
|
||||
" specify the name of the category, not its url.",
|
||||
})(values)}
|
||||
<TextFormField
|
||||
name="recurse_depth"
|
||||
label="Recursion Depth:"
|
||||
type="number"
|
||||
subtext="When indexing categories that have sub-categories, this will determine how may levels to index. Specify 0 to only index the category itself (i.e. no recursion). Specify -1 for unlimited recursion depth. Note, that in some rare instances, a category might contain itself in its dependencies, which will cause an infinite loop. Only use -1 if you confident that this will not happen."
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
validationSchema={Yup.object().shape({
|
||||
connector_name: Yup.string().required(
|
||||
"Please enter a name for your Wikipedia connector."
|
||||
),
|
||||
language_code: Yup.string().default("en"),
|
||||
categories: Yup.array().of(
|
||||
Yup.string().required(
|
||||
"Please enter categories to index from your Wikipedia site"
|
||||
)
|
||||
),
|
||||
pages: Yup.array().of(
|
||||
Yup.string().required(
|
||||
"Please enter pages to index from your Wikipedia site"
|
||||
)
|
||||
),
|
||||
recurse_depth: Yup.number().required(
|
||||
"Please enter the recursion depth for your Wikipedia site."
|
||||
),
|
||||
})}
|
||||
initialValues={{
|
||||
connector_name: "",
|
||||
language_code: "en",
|
||||
categories: [],
|
||||
pages: [],
|
||||
recurse_depth: 0,
|
||||
}}
|
||||
refreshFreq={10 * 60} // 10 minutes
|
||||
credentialId={wikipediaCredential.id}
|
||||
/>
|
||||
</Card>
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default function Page() {
|
||||
return (
|
||||
<div className="mx-auto container">
|
||||
<div className="mb-4">
|
||||
<HealthCheckBanner />
|
||||
</div>
|
||||
|
||||
<AdminPageTitle icon={<WikipediaIcon size={32} />} title="Wikipedia" />
|
||||
|
||||
<Main />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -52,6 +52,8 @@ import document360Icon from "../../../public/Document360.png";
|
||||
import googleSitesIcon from "../../../public/GoogleSites.png";
|
||||
import zendeskIcon from "../../../public/Zendesk.svg";
|
||||
import sharepointIcon from "../../../public/Sharepoint.png";
|
||||
import mediawikiIcon from "../../../public/MediaWiki.svg";
|
||||
import wikipediaIcon from "../../../public/Wikipedia.svg";
|
||||
import discourseIcon from "../../../public/Discourse.png";
|
||||
import { FaRobot } from "react-icons/fa";
|
||||
|
||||
@@ -625,3 +627,27 @@ export const AxeroIcon = ({
|
||||
<Image src="/Axero.jpeg" alt="Logo" width="96" height="96" />
|
||||
</div>
|
||||
);
|
||||
|
||||
export const MediaWikiIcon = ({
|
||||
size = 16,
|
||||
className = defaultTailwindCSS,
|
||||
}: IconProps) => (
|
||||
<div
|
||||
style={{ width: `${size}px`, height: `${size}px` }}
|
||||
className={`w-[${size}px] h-[${size}px] ` + className}
|
||||
>
|
||||
<Image src={mediawikiIcon} alt="Logo" width="96" height="96" />
|
||||
</div>
|
||||
);
|
||||
|
||||
export const WikipediaIcon = ({
|
||||
size = 16,
|
||||
className = defaultTailwindCSS,
|
||||
}: IconProps) => (
|
||||
<div
|
||||
style={{ width: `${size}px`, height: `${size}px` }}
|
||||
className={`w-[${size}px] h-[${size}px] ` + className}
|
||||
>
|
||||
<Image src={wikipediaIcon} alt="Logo" width="96" height="96" />
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -25,6 +25,8 @@ import {
|
||||
SlackIcon,
|
||||
ZendeskIcon,
|
||||
ZulipIcon,
|
||||
MediaWikiIcon,
|
||||
WikipediaIcon,
|
||||
} from "@/components/icons/icons";
|
||||
import { ValidSources } from "./types";
|
||||
import { SourceCategory, SourceMetadata } from "./search/interfaces";
|
||||
@@ -166,6 +168,16 @@ const SOURCE_METADATA_MAP: SourceMap = {
|
||||
displayName: "Axero",
|
||||
category: SourceCategory.AppConnection,
|
||||
},
|
||||
wikipedia: {
|
||||
icon: WikipediaIcon,
|
||||
displayName: "Wikipedia",
|
||||
category: SourceCategory.AppConnection,
|
||||
},
|
||||
mediawiki: {
|
||||
icon: MediaWikiIcon,
|
||||
displayName: "MediaWiki",
|
||||
category: SourceCategory.AppConnection,
|
||||
},
|
||||
requesttracker: {
|
||||
icon: RequestTrackerIcon,
|
||||
displayName: "Request Tracker",
|
||||
|
||||
@@ -40,7 +40,9 @@ export type ValidSources =
|
||||
| "sharepoint"
|
||||
| "zendesk"
|
||||
| "discourse"
|
||||
| "axero";
|
||||
| "axero"
|
||||
| "wikipedia"
|
||||
| "mediawiki";
|
||||
|
||||
export type ValidInputTypes = "load_state" | "poll" | "event";
|
||||
export type ValidStatuses =
|
||||
@@ -179,6 +181,19 @@ export interface GoogleSitesConfig {
|
||||
|
||||
export interface ZendeskConfig {}
|
||||
|
||||
export interface MediaWikiBaseConfig {
|
||||
connector_name: string;
|
||||
language_code: string;
|
||||
categories?: string[];
|
||||
pages?: string[];
|
||||
recurse_depth?: number;
|
||||
}
|
||||
export interface MediaWikiConfig extends MediaWikiBaseConfig {
|
||||
hostname: string;
|
||||
}
|
||||
|
||||
export interface WikipediaConfig extends MediaWikiBaseConfig {}
|
||||
|
||||
export interface IndexAttemptSnapshot {
|
||||
id: number;
|
||||
status: ValidStatuses | null;
|
||||
@@ -353,6 +368,9 @@ export interface AxeroCredentialJson {
|
||||
axero_api_token: string;
|
||||
}
|
||||
|
||||
export interface MediaWikiCredentialJson {}
|
||||
export interface WikipediaCredentialJson extends MediaWikiCredentialJson {}
|
||||
|
||||
// DELETION
|
||||
|
||||
export interface DeletionAttemptSnapshot {
|
||||
|
||||
Reference in New Issue
Block a user