diff --git a/web/src/app/admin/connectors/web/page.tsx b/web/src/app/admin/connectors/web/page.tsx index 0886c2a12..cbb0c1660 100644 --- a/web/src/app/admin/connectors/web/page.tsx +++ b/web/src/app/admin/connectors/web/page.tsx @@ -6,12 +6,20 @@ import * as Yup from "yup"; import { LoadingAnimation } from "@/components/Loading"; import { GlobeIcon } from "@/components/icons/icons"; import { fetcher } from "@/lib/fetcher"; -import { TextFormField } from "@/components/admin/connectors/Field"; +import { + SelectorFormField, + TextFormField, +} from "@/components/admin/connectors/Field"; import { HealthCheckBanner } from "@/components/health/healthcheck"; import { ConnectorIndexingStatus, WebConfig } from "@/lib/types"; import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable"; import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; -import { linkCredential } from "@/lib/credential"; + +const SCRAPE_TYPE_TO_PRETTY_NAME = { + recursive: "Recursive", + single: "Single Page", + sitemap: "Sitemap", +}; export default function Web() { const { mutate } = useSWRConfig(); @@ -56,15 +64,42 @@ export default function Web() { formBody={ <> + > } validationSchema={Yup.object().shape({ base_url: Yup.string().required( "Please enter the website URL to scrape e.g. https://docs.danswer.dev/" ), + web_connector_type: Yup.string() + .oneOf(["recursive", "single", "sitemap"]) + .optional(), })} initialValues={{ base_url: "", + web_connector_type: undefined, }} refreshFreq={60 * 60 * 24} // 1 day /> @@ -93,6 +128,16 @@ export default function Web() { ), }, + { + header: "Scrape Method", + key: "web_connector_type", + getValue: (connector) => + connector.connector_specific_config.web_connector_type + ? SCRAPE_TYPE_TO_PRETTY_NAME[ + connector.connector_specific_config.web_connector_type + ] + : "Recursive", + }, ]} onUpdate={() => mutate("/api/manage/admin/connector/indexing-status")} /> diff --git a/web/src/components/admin/connectors/Field.tsx b/web/src/components/admin/connectors/Field.tsx index f6cf11849..4ea9b4e7b 100644 --- a/web/src/components/admin/connectors/Field.tsx +++ b/web/src/components/admin/connectors/Field.tsx @@ -1,7 +1,16 @@ import { Button } from "@/components/Button"; -import { ArrayHelpers, ErrorMessage, Field, FieldArray } from "formik"; +import { + ArrayHelpers, + ErrorMessage, + Field, + FieldArray, + useField, + useFormikContext, +} from "formik"; import * as Yup from "yup"; import { FormBodyBuilder } from "./types"; +import { FC, useEffect, useRef, useState } from "react"; +import { ChevronDownIcon } from "@/components/icons/icons"; interface TextFormFieldProps { name: string; @@ -172,3 +181,147 @@ export function TextArrayFieldBuilder( ); return _TextArrayField; } + +interface Option { + name: string; + value: string; + description?: string; +} + +interface SelectorFormFieldProps { + name: string; + label: string; + options: Option[]; + subtext?: string; +} + +export function SelectorFormField({ + name, + label, + options, + subtext, +}: SelectorFormFieldProps) { + const [field] = useField(name); + const { setFieldValue } = useFormikContext(); + + return ( + + + + {label} + {subtext && {subtext}} + + + + setFieldValue(name, selected.value)} + /> + + + + ); +} + +interface DropdownProps { + options: Option[]; + selected: string; + onSelect: (selected: Option) => void; +} + +const Dropdown: FC = ({ options, selected, onSelect }) => { + const [isOpen, setIsOpen] = useState(false); + const dropdownRef = useRef(null); + + const selectedName = options.find( + (option) => option.value === selected + )?.name; + + const handleSelect = (option: Option) => { + onSelect(option); + setIsOpen(false); + }; + + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if ( + dropdownRef.current && + !dropdownRef.current.contains(event.target as Node) + ) { + setIsOpen(false); + } + }; + + document.addEventListener("mousedown", handleClickOutside); + return () => { + document.removeEventListener("mousedown", handleClickOutside); + }; + }, []); + + return ( + + + setIsOpen(!isOpen)} + > + {selectedName ? {selectedName} : "Select an option..."} + + + + + {isOpen ? ( + + + {options.map((option, index) => ( + handleSelect(option)} + className={ + `w-full text-left block px-4 py-2.5 text-sm hover:bg-gray-800` + + (index !== 0 ? " border-t-2 border-gray-600" : "") + } + role="menuitem" + > + {option.name} + {option.description && ( + + + {option.description} + + + )} + + ))} + + + ) : null} + + ); +}; diff --git a/web/src/components/admin/connectors/table/ConnectorsTable.tsx b/web/src/components/admin/connectors/table/ConnectorsTable.tsx index 8148bb321..ecc8a28de 100644 --- a/web/src/components/admin/connectors/table/ConnectorsTable.tsx +++ b/web/src/components/admin/connectors/table/ConnectorsTable.tsx @@ -100,7 +100,9 @@ export function StatusRow({ interface ColumnSpecification { header: string; key: string; - getValue: (connector: Connector) => JSX.Element | string; + getValue: ( + connector: Connector + ) => JSX.Element | string | undefined; } interface ConnectorsTableProps { diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index e1e9211c0..74c189b45 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -56,6 +56,7 @@ export interface Connector extends ConnectorBase { export interface WebConfig { base_url: string; + web_connector_type?: "recursive" | "single" | "sitemap"; } export interface GithubConfig {
{subtext}
{selectedName}
{option.name}
+ {option.description} +