Add option to select Scrape Type in the web connector UI

web/src/app/admin/connectors/web/page.tsx

@@ -6,12 +6,20 @@ import * as Yup from "yup";
 import { LoadingAnimation } from "@/components/Loading";
 import { GlobeIcon } from "@/components/icons/icons";
 import { fetcher } from "@/lib/fetcher";
-import { TextFormField } from "@/components/admin/connectors/Field";
+import {
+  SelectorFormField,
+  TextFormField,
+} from "@/components/admin/connectors/Field";
 import { HealthCheckBanner } from "@/components/health/healthcheck";
 import { ConnectorIndexingStatus, WebConfig } from "@/lib/types";
 import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable";
 import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm";
-import { linkCredential } from "@/lib/credential";
+
+const SCRAPE_TYPE_TO_PRETTY_NAME = {
+  recursive: "Recursive",
+  single: "Single Page",
+  sitemap: "Sitemap",
+};
 
 export default function Web() {
   const { mutate } = useSWRConfig();
@@ -56,15 +64,42 @@ export default function Web() {
           formBody={
             <>
               <TextFormField name="base_url" label="URL to Index:" />
+              <SelectorFormField
+                name="web_connector_type"
+                label="Scrape Method:"
+                options={[
+                  {
+                    name: "Recursive",
+                    value: "recursive",
+                    description:
+                      "Recursively index all pages that share the same base URL.",
+                  },
+                  {
+                    name: "Single Page",
+                    value: "single",
+                    description: "Index only the specified page.",
+                  },
+                  {
+                    name: "Sitemap",
+                    value: "sitemap",
+                    description:
+                      "Assumes the URL to Index points to a Sitemap. Will try and index all pages that are a mentioned in the sitemap.",
+                  },
+                ]}
+              />
             </>
           }
           validationSchema={Yup.object().shape({
             base_url: Yup.string().required(
               "Please enter the website URL to scrape e.g. https://docs.danswer.dev/"
             ),
+            web_connector_type: Yup.string()
+              .oneOf(["recursive", "single", "sitemap"])
+              .optional(),
           })}
           initialValues={{
             base_url: "",
+            web_connector_type: undefined,
           }}
           refreshFreq={60 * 60 * 24} // 1 day
         />
@@ -93,6 +128,16 @@ export default function Web() {
                 </a>
               ),
             },
+            {
+              header: "Scrape Method",
+              key: "web_connector_type",
+              getValue: (connector) =>
+                connector.connector_specific_config.web_connector_type
+                  ? SCRAPE_TYPE_TO_PRETTY_NAME[
+                      connector.connector_specific_config.web_connector_type
+                    ]
+                  : "Recursive",
+            },
           ]}
           onUpdate={() => mutate("/api/manage/admin/connector/indexing-status")}
         />

web/src/components/admin/connectors/Field.tsx

@@ -1,7 +1,16 @@
 import { Button } from "@/components/Button";
-import { ArrayHelpers, ErrorMessage, Field, FieldArray } from "formik";
+import {
+  ArrayHelpers,
+  ErrorMessage,
+  Field,
+  FieldArray,
+  useField,
+  useFormikContext,
+} from "formik";
 import * as Yup from "yup";
 import { FormBodyBuilder } from "./types";
+import { FC, useEffect, useRef, useState } from "react";
+import { ChevronDownIcon } from "@/components/icons/icons";
 
 interface TextFormFieldProps {
   name: string;
@@ -172,3 +181,147 @@ export function TextArrayFieldBuilder<T extends Yup.AnyObject>(
   );
   return _TextArrayField;
 }
+
+interface Option {
+  name: string;
+  value: string;
+  description?: string;
+}
+
+interface SelectorFormFieldProps {
+  name: string;
+  label: string;
+  options: Option[];
+  subtext?: string;
+}
+
+export function SelectorFormField({
+  name,
+  label,
+  options,
+  subtext,
+}: SelectorFormFieldProps) {
+  const [field] = useField<string>(name);
+  const { setFieldValue } = useFormikContext();
+
+  return (
+    <div className="mb-4">
+      <label className="flex mb-2">
+        <div>
+          {label}
+          {subtext && <p className="text-xs">{subtext}</p>}
+        </div>
+      </label>
+
+      <Dropdown
+        options={options}
+        selected={field.value}
+        onSelect={(selected) => setFieldValue(name, selected.value)}
+      />
+
+      <ErrorMessage
+        name={name}
+        component="div"
+        className="text-red-500 text-sm mt-1"
+      />
+    </div>
+  );
+}
+
+interface DropdownProps {
+  options: Option[];
+  selected: string;
+  onSelect: (selected: Option) => void;
+}
+
+const Dropdown: FC<DropdownProps> = ({ options, selected, onSelect }) => {
+  const [isOpen, setIsOpen] = useState(false);
+  const dropdownRef = useRef<HTMLDivElement>(null);
+
+  const selectedName = options.find(
+    (option) => option.value === selected
+  )?.name;
+
+  const handleSelect = (option: Option) => {
+    onSelect(option);
+    setIsOpen(false);
+  };
+
+  useEffect(() => {
+    const handleClickOutside = (event: MouseEvent) => {
+      if (
+        dropdownRef.current &&
+        !dropdownRef.current.contains(event.target as Node)
+      ) {
+        setIsOpen(false);
+      }
+    };
+
+    document.addEventListener("mousedown", handleClickOutside);
+    return () => {
+      document.removeEventListener("mousedown", handleClickOutside);
+    };
+  }, []);
+
+  return (
+    <div className="relative inline-block text-left w-full" ref={dropdownRef}>
+      <div>
+        <button
+          type="button"
+          className={`inline-flex 
+          justify-center 
+          w-full 
+          px-4 
+          py-3
+          text-sm 
+          bg-gray-700 
+          border 
+          border-gray-300 
+          rounded-md 
+          shadow-sm 
+          hover:bg-gray-700 
+          focus:ring focus:ring-offset-0 focus:ring-1 focus:ring-offset-gray-800 focus:ring-blue-800
+          `}
+          id="options-menu"
+          aria-expanded="true"
+          aria-haspopup="true"
+          onClick={() => setIsOpen(!isOpen)}
+        >
+          {selectedName ? <p>{selectedName}</p> : "Select an option..."}
+          <ChevronDownIcon className="text-gray-400 my-auto ml-auto" />
+        </button>
+      </div>
+
+      {isOpen ? (
+        <div className="origin-top-right absolute left-0 mt-3 w-full rounded-md shadow-lg bg-gray-700 border-2 border-gray-600">
+          <div
+            role="menu"
+            aria-orientation="vertical"
+            aria-labelledby="options-menu"
+          >
+            {options.map((option, index) => (
+              <button
+                key={index}
+                onClick={() => handleSelect(option)}
+                className={
+                  `w-full text-left block px-4 py-2.5 text-sm hover:bg-gray-800` +
+                  (index !== 0 ? " border-t-2 border-gray-600" : "")
+                }
+                role="menuitem"
+              >
+                <p className="font-medium">{option.name}</p>
+                {option.description && (
+                  <div>
+                    <p className="text-xs text-gray-300">
+                      {option.description}
+                    </p>
+                  </div>
+                )}
+              </button>
+            ))}
+          </div>
+        </div>
+      ) : null}
+    </div>
+  );
+};

web/src/components/admin/connectors/table/ConnectorsTable.tsx

@@ -100,7 +100,9 @@ export function StatusRow<ConnectorConfigType, ConnectorCredentialType>({
 interface ColumnSpecification<ConnectorConfigType> {
   header: string;
   key: string;
-  getValue: (connector: Connector<ConnectorConfigType>) => JSX.Element | string;
+  getValue: (
+    connector: Connector<ConnectorConfigType>
+  ) => JSX.Element | string | undefined;
 }
 
 interface ConnectorsTableProps<ConnectorConfigType, ConnectorCredentialType> {

web/src/lib/types.ts

@@ -56,6 +56,7 @@ export interface Connector<T> extends ConnectorBase<T> {
 
 export interface WebConfig {
   base_url: string;
+  web_connector_type?: "recursive" | "single" | "sitemap";
 }
 
 export interface GithubConfig {