diff --git a/web/src/app/admin/connectors/web/page.tsx b/web/src/app/admin/connectors/web/page.tsx index 0886c2a12..cbb0c1660 100644 --- a/web/src/app/admin/connectors/web/page.tsx +++ b/web/src/app/admin/connectors/web/page.tsx @@ -6,12 +6,20 @@ import * as Yup from "yup"; import { LoadingAnimation } from "@/components/Loading"; import { GlobeIcon } from "@/components/icons/icons"; import { fetcher } from "@/lib/fetcher"; -import { TextFormField } from "@/components/admin/connectors/Field"; +import { + SelectorFormField, + TextFormField, +} from "@/components/admin/connectors/Field"; import { HealthCheckBanner } from "@/components/health/healthcheck"; import { ConnectorIndexingStatus, WebConfig } from "@/lib/types"; import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable"; import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; -import { linkCredential } from "@/lib/credential"; + +const SCRAPE_TYPE_TO_PRETTY_NAME = { + recursive: "Recursive", + single: "Single Page", + sitemap: "Sitemap", +}; export default function Web() { const { mutate } = useSWRConfig(); @@ -56,15 +64,42 @@ export default function Web() { formBody={ <> + } validationSchema={Yup.object().shape({ base_url: Yup.string().required( "Please enter the website URL to scrape e.g. https://docs.danswer.dev/" ), + web_connector_type: Yup.string() + .oneOf(["recursive", "single", "sitemap"]) + .optional(), })} initialValues={{ base_url: "", + web_connector_type: undefined, }} refreshFreq={60 * 60 * 24} // 1 day /> @@ -93,6 +128,16 @@ export default function Web() { ), }, + { + header: "Scrape Method", + key: "web_connector_type", + getValue: (connector) => + connector.connector_specific_config.web_connector_type + ? SCRAPE_TYPE_TO_PRETTY_NAME[ + connector.connector_specific_config.web_connector_type + ] + : "Recursive", + }, ]} onUpdate={() => mutate("/api/manage/admin/connector/indexing-status")} /> diff --git a/web/src/components/admin/connectors/Field.tsx b/web/src/components/admin/connectors/Field.tsx index f6cf11849..4ea9b4e7b 100644 --- a/web/src/components/admin/connectors/Field.tsx +++ b/web/src/components/admin/connectors/Field.tsx @@ -1,7 +1,16 @@ import { Button } from "@/components/Button"; -import { ArrayHelpers, ErrorMessage, Field, FieldArray } from "formik"; +import { + ArrayHelpers, + ErrorMessage, + Field, + FieldArray, + useField, + useFormikContext, +} from "formik"; import * as Yup from "yup"; import { FormBodyBuilder } from "./types"; +import { FC, useEffect, useRef, useState } from "react"; +import { ChevronDownIcon } from "@/components/icons/icons"; interface TextFormFieldProps { name: string; @@ -172,3 +181,147 @@ export function TextArrayFieldBuilder( ); return _TextArrayField; } + +interface Option { + name: string; + value: string; + description?: string; +} + +interface SelectorFormFieldProps { + name: string; + label: string; + options: Option[]; + subtext?: string; +} + +export function SelectorFormField({ + name, + label, + options, + subtext, +}: SelectorFormFieldProps) { + const [field] = useField(name); + const { setFieldValue } = useFormikContext(); + + return ( +
+ + + setFieldValue(name, selected.value)} + /> + + +
+ ); +} + +interface DropdownProps { + options: Option[]; + selected: string; + onSelect: (selected: Option) => void; +} + +const Dropdown: FC = ({ options, selected, onSelect }) => { + const [isOpen, setIsOpen] = useState(false); + const dropdownRef = useRef(null); + + const selectedName = options.find( + (option) => option.value === selected + )?.name; + + const handleSelect = (option: Option) => { + onSelect(option); + setIsOpen(false); + }; + + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if ( + dropdownRef.current && + !dropdownRef.current.contains(event.target as Node) + ) { + setIsOpen(false); + } + }; + + document.addEventListener("mousedown", handleClickOutside); + return () => { + document.removeEventListener("mousedown", handleClickOutside); + }; + }, []); + + return ( +
+
+ +
+ + {isOpen ? ( +
+
+ {options.map((option, index) => ( + + ))} +
+
+ ) : null} +
+ ); +}; diff --git a/web/src/components/admin/connectors/table/ConnectorsTable.tsx b/web/src/components/admin/connectors/table/ConnectorsTable.tsx index 8148bb321..ecc8a28de 100644 --- a/web/src/components/admin/connectors/table/ConnectorsTable.tsx +++ b/web/src/components/admin/connectors/table/ConnectorsTable.tsx @@ -100,7 +100,9 @@ export function StatusRow({ interface ColumnSpecification { header: string; key: string; - getValue: (connector: Connector) => JSX.Element | string; + getValue: ( + connector: Connector + ) => JSX.Element | string | undefined; } interface ConnectorsTableProps { diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index e1e9211c0..74c189b45 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -56,6 +56,7 @@ export interface Connector extends ConnectorBase { export interface WebConfig { base_url: string; + web_connector_type?: "recursive" | "single" | "sitemap"; } export interface GithubConfig {