highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-09-27 20:38:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add option to not re-index (#4157)

Browse Source 
* Add option to not re-index

* Add quantizaton / dimensionality override support

* Fix build / ut
This commit is contained in:
Chris Weaver
2025-03-03 10:54:11 -08:00
committed by GitHub
parent 39fd6919ad
commit f25e1e80f6
40 changed files with 1020 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
web/src/app/admin/configuration/search/UpgradingPage.tsx
View File

@@ -108,15 +108,13 @@ export default function UpgradingPage({
>
<div>
<div>
Are you sure you want to cancel?
<br />
<br />
Cancelling will revert to the previous model and all progress will
be lost.
Are you sure you want to cancel? Cancelling will revert to the
previous model and all progress will be lost.
</div>
<div className="flex">
<Button onClick={onCancel} variant="submit">
Confirm
<div className="mt-12 gap-x-2 w-full justify-end flex">
<Button onClick={onCancel}>Confirm</Button>
<Button onClick={() => setIsCancelling(false)} variant="outline">
Cancel
</Button>
</div>
</div>
@@ -141,30 +139,46 @@ export default function UpgradingPage({
</Button>
{connectors && connectors.length > 0 ? (
<>
{failedIndexingStatus && failedIndexingStatus.length > 0 && (
<FailedReIndexAttempts
failedIndexingStatuses={failedIndexingStatus}
setPopup={setPopup}
/>
)}
futureEmbeddingModel.background_reindex_enabled ? (
<>
{failedIndexingStatus && failedIndexingStatus.length > 0 && (
<FailedReIndexAttempts
failedIndexingStatuses={failedIndexingStatus}
setPopup={setPopup}
/>
)}
<Text className="my-4">
The table below shows the re-indexing progress of all existing
connectors. Once all connectors have been re-indexed
successfully, the new model will be used for all search
queries. Until then, we will use the old model so that no
downtime is necessary during this transition.
</Text>
<Text className="my-4">
The table below shows the re-indexing progress of all
existing connectors. Once all connectors have been
re-indexed successfully, the new model will be used for all
search queries. Until then, we will use the old model so
that no downtime is necessary during this transition.
</Text>
{sortedReindexingProgress ? (
<ReindexingProgressTable
reindexingProgress={sortedReindexingProgress}
/>
) : (
<ErrorCallout errorTitle="Failed to fetch reindexing progress" />
)}
</>
{sortedReindexingProgress ? (
<ReindexingProgressTable
reindexingProgress={sortedReindexingProgress}
/>
) : (
<ErrorCallout errorTitle="Failed to fetch re-indexing progress" />
)}
</>
) : (
<div className="mt-8">
<h3 className="text-lg font-semibold mb-2">
Switching Embedding Models
</h3>
<p className="mb-4 text-text-800">
You&apos;re currently switching embedding models, and
you&apos;ve selected the instant switch option. The
transition will complete shortly.
</p>
<p className="text-text-600">
The new model will be active soon.
</p>
</div>
)
) : (
<div className="mt-8 p-6 bg-background-100 border border-border-strong rounded-lg max-w-2xl">
<h3 className="text-lg font-semibold mb-2">

19
web/src/app/admin/connectors/[connector]/pages/ConnectorInput/NumberInput.tsx
View File

@@ -1,5 +1,5 @@
import { SubLabel } from "@/components/admin/connectors/Field";
import { Field } from "formik";
import { Label, SubLabel } from "@/components/admin/connectors/Field";
import { ErrorMessage, Field } from "formik";
export default function NumberInput({
label,
@@ -16,10 +16,12 @@ export default function NumberInput({
}) {
return (
<div className="w-full flex flex-col">
<label className="block text-base font-medium text-text-700 dark:text-neutral-100 mb-1">
{label}
{optional && <span className="text-text-500 ml-1">(optional)</span>}
</label>
<Label>
<>
{label}
{optional && <span className="text-text-500 ml-1">(optional)</span>}
</>
</Label>
{description && <SubLabel>{description}</SubLabel>}
<Field
@@ -34,6 +36,11 @@ export default function NumberInput({
invalid:border-pink-500 invalid:text-pink-600
focus:invalid:border-pink-500 focus:invalid:ring-pink-500`}
/>
<ErrorMessage
name={name}
component="div"
className="text-error text-sm mt-1"
/>
</div>
);
}

42
web/src/app/admin/connectors/[connector]/pages/formelements/NumberInput.tsx
View File

@@ -1,42 +0,0 @@
import { SubLabel } from "@/components/admin/connectors/Field";
import { Field } from "formik";
export default function NumberInput({
label,
value,
optional,
description,
name,
showNeverIfZero,
}: {
value?: number;
label: string;
name: string;
optional?: boolean;
description?: string;
showNeverIfZero?: boolean;
}) {
return (
<div className="w-full flex flex-col">
<label className="block text-base font-medium text-text-700 mb-1">
{label}
{optional && <span className="text-text-500 ml-1">(optional)</span>}
</label>
{description && <SubLabel>{description}</SubLabel>}
<Field
type="number"
name={name}
min="-1"
value={value === 0 && showNeverIfZero ? "Never" : value}
className={`mt-2 block w-full px-3 py-2
bg-white border border-background-300 rounded-md
text-sm shadow-sm placeholder-text-400
focus:outline-none focus:border-sky-500 focus:ring-1 focus:ring-sky-500
disabled:bg-background-50 disabled:text-text-500 disabled:border-background-200 disabled:shadow-none
invalid:border-pink-500 invalid:text-pink-600
focus:invalid:border-pink-500 focus:invalid:ring-pink-500`}
/>
</div>
);
}

40
web/src/app/admin/embeddings/EmbeddingModelSelectionForm.tsx
View File

@@ -103,42 +103,6 @@ export function EmbeddingModelSelection({
{ refreshInterval: 5000 } // 5 seconds
);
const { data: connectors } = useSWR<Connector<any>[]>(
"/api/manage/connector",
errorHandlingFetcher,
{ refreshInterval: 5000 } // 5 seconds
);
const onConfirmSelection = async (model: EmbeddingModelDescriptor) => {
const response = await fetch(
"/api/search-settings/set-new-search-settings",
{
method: "POST",
body: JSON.stringify({ ...model, index_name: null }),
headers: {
"Content-Type": "application/json",
},
}
);
if (response.ok) {
setShowTentativeModel(null);
mutate("/api/search-settings/get-secondary-search-settings");
if (!connectors || !connectors.length) {
setShowAddConnectorPopup(true);
}
} else {
alert(`Failed to update embedding model - ${await response.text()}`);
}
};
const onSelectOpenSource = async (model: HostedEmbeddingModel) => {
if (selectedProvider?.model_name === INVALID_OLD_MODEL) {
await onConfirmSelection(model);
} else {
setShowTentativeOpenProvider(model);
}
};
return (
<div className="p-2">
{alreadySelectedModel && (
@@ -270,7 +234,9 @@ export function EmbeddingModelSelection({
{modelTab == "open" && (
<OpenEmbeddingPage
selectedProvider={selectedProvider}
onSelectOpenSource={onSelectOpenSource}
onSelectOpenSource={(model: HostedEmbeddingModel) => {
setShowTentativeOpenProvider(model);
}}
/>
)}

83
web/src/app/admin/embeddings/RerankingFormPage.tsx
View File

@@ -30,6 +30,10 @@ interface RerankingDetailsFormProps {
originalRerankingDetails: RerankingDetails;
modelTab: "open" | "cloud" | null;
setModelTab: Dispatch<SetStateAction<"open" | "cloud" | null>>;
onValidationChange?: (
isValid: boolean,
errors: Record<string, string>
) => void;
}
const RerankingDetailsForm = forwardRef<
@@ -43,6 +47,7 @@ const RerankingDetailsForm = forwardRef<
currentRerankingDetails,
modelTab,
setModelTab,
onValidationChange,
},
ref
) => {
@@ -55,26 +60,78 @@ const RerankingDetailsForm = forwardRef<
const combinedSettings = useContext(SettingsContext);
const gpuEnabled = combinedSettings?.settings.gpu_enabled;
// Define the validation schema
const validationSchema = Yup.object().shape({
rerank_model_name: Yup.string().nullable(),
rerank_provider_type: Yup.mixed<RerankerProvider>()
.nullable()
.oneOf(Object.values(RerankerProvider))
.optional(),
rerank_api_key: Yup.string()
.nullable()
.test(
"required-if-cohere",
"API Key is required for Cohere reranking",
function (value) {
const { rerank_provider_type } = this.parent;
return (
rerank_provider_type !== RerankerProvider.COHERE ||
(value !== null && value !== "")
);
}
),
rerank_api_url: Yup.string()
.url("Must be a valid URL")
.matches(/^https?:\/\//, "URL must start with http:// or https://")
.nullable()
.test(
"required-if-litellm",
"API URL is required for LiteLLM reranking",
function (value) {
const { rerank_provider_type } = this.parent;
return (
rerank_provider_type !== RerankerProvider.LITELLM ||
(value !== null && value !== "")
);
}
),
});
return (
<Formik
innerRef={ref}
initialValues={currentRerankingDetails}
validationSchema={Yup.object().shape({
rerank_model_name: Yup.string().nullable(),
rerank_provider_type: Yup.mixed<RerankerProvider>()
.nullable()
.oneOf(Object.values(RerankerProvider))
.optional(),
api_key: Yup.string().nullable(),
num_rerank: Yup.number().min(1, "Must be at least 1"),
rerank_api_url: Yup.string()
.url("Must be a valid URL")
.matches(/^https?:\/\//, "URL must start with http:// or https://")
.nullable(),
})}
validationSchema={validationSchema}
onSubmit={async (_, { setSubmitting }) => {
setSubmitting(false);
}}
validate={(values) => {
// Update parent component with values
setRerankingDetails(values);
// Run validation and report errors
if (onValidationChange) {
// We'll return an empty object here since Yup will handle the actual validation
// But we need to check if there are any validation errors
const errors: Record<string, string> = {};
try {
// Manually validate against the schema
validationSchema.validateSync(values, { abortEarly: false });
onValidationChange(true, {});
} catch (validationError) {
if (validationError instanceof Yup.ValidationError) {
validationError.inner.forEach((err) => {
if (err.path) {
errors[err.path] = err.message;
}
});
onValidationChange(false, errors);
}
}
}
return {}; // Return empty object as Formik will handle the errors
}}
enableReinitialize={true}
>
{({ values, setFieldValue, resetForm }) => {

8
web/src/app/admin/embeddings/interfaces.ts
View File

@@ -20,6 +20,11 @@ export enum RerankerProvider {
LITELLM = "litellm",
}
export enum EmbeddingPrecision {
FLOAT = "float",
BFLOAT16 = "bfloat16",
}
export interface AdvancedSearchConfiguration {
index_name: string | null;
multipass_indexing: boolean;
@@ -27,12 +32,15 @@ export interface AdvancedSearchConfiguration {
disable_rerank_for_streaming: boolean;
api_url: string | null;
num_rerank: number;
embedding_precision: EmbeddingPrecision;
reduced_dimension: number | null;
}
export interface SavedSearchSettings
extends RerankingDetails,
AdvancedSearchConfiguration {
provider_type: EmbeddingProvider | null;
background_reindex_enabled: boolean;
}
export interface RerankingModel {

37
web/src/app/admin/embeddings/modals/InstantSwitchConfirmModal.tsx Normal file
View File

@@ -0,0 +1,37 @@
import { Modal } from "@/components/Modal";
import { Button } from "@/components/ui/button";
interface InstantSwitchConfirmModalProps {
onClose: () => void;
onConfirm: () => void;
}
export const InstantSwitchConfirmModal = ({
onClose,
onConfirm,
}: InstantSwitchConfirmModalProps) => {
return (
<Modal
onOutsideClick={onClose}
width="max-w-3xl"
title="Are you sure you want to do an instant switch?"
>
<>
<div>
Instant switching will immediately change the embedding model without
re-indexing. Searches will be over a partial set of documents
(starting with 0 documents) until re-indexing is complete.
<br />
<br />
<b>This is not reversible.</b>
</div>
<div className="flex mt-4 gap-x-2 justify-end">
<Button onClick={onConfirm}>Confirm</Button>
<Button variant="outline" onClick={onClose}>
Cancel
</Button>
</div>
</>
</Modal>
);
};

7
web/src/app/admin/embeddings/modals/ModelSelectionModal.tsx
View File

@@ -51,9 +51,10 @@ export function ModelSelectionConfirmationModal({
</Callout>
)}
<div className="flex mt-8">
<Button className="mx-auto" variant="submit" onClick={onConfirm}>
Yes
<div className="flex mt-8 gap-x-2 justify-end">
<Button onClick={onConfirm}>Confirm</Button>
<Button variant="outline" onClick={onCancel}>
Cancel
</Button>
</div>
</div>

15
web/src/app/admin/embeddings/modals/SelectModelModal.tsx
View File

@@ -21,15 +21,14 @@ export function SelectModelModal({
>
<div className="mb-4">
<Text className="text-lg mb-2">
You&apos;re selecting a new embedding model, {model.model_name}. If
you update to this model, you will need to undergo a complete
re-indexing.
<br />
Are you sure?
You&apos;re selecting a new embedding model, <b>{model.model_name}</b>
. If you update to this model, you will need to undergo a complete
re-indexing. Are you sure?
</Text>
<div className="flex mt-8 justify-end">
<Button variant="submit" onClick={onConfirm}>
Yes
<div className="flex mt-8 justify-end gap-x-2">
<Button onClick={onConfirm}>Confirm</Button>
<Button variant="outline" onClick={onCancel}>
Cancel
</Button>
</div>
</div>

279
web/src/app/admin/embeddings/pages/AdvancedEmbeddingFormPage.tsx
View File

@@ -3,13 +3,15 @@ import { Formik, Form, FormikProps, FieldArray, Field } from "formik";
import * as Yup from "yup";
import { TrashIcon } from "@/components/icons/icons";
import { FaPlus } from "react-icons/fa";
import { AdvancedSearchConfiguration } from "../interfaces";
import { AdvancedSearchConfiguration, EmbeddingPrecision } from "../interfaces";
import {
BooleanFormField,
Label,
SubLabel,
SelectorFormField,
} from "@/components/admin/connectors/Field";
import NumberInput from "../../connectors/[connector]/pages/ConnectorInput/NumberInput";
import { StringOrNumberOption } from "@/components/Dropdown";
interface AdvancedEmbeddingFormPageProps {
updateAdvancedEmbeddingDetails: (
@@ -17,102 +19,207 @@ interface AdvancedEmbeddingFormPageProps {
value: any
) => void;
advancedEmbeddingDetails: AdvancedSearchConfiguration;
embeddingProviderType: string | null;
onValidationChange?: (
isValid: boolean,
errors: Record<string, string>
) => void;
}
// Options for embedding precision based on EmbeddingPrecision enum
const embeddingPrecisionOptions: StringOrNumberOption[] = [
{ name: EmbeddingPrecision.BFLOAT16, value: EmbeddingPrecision.BFLOAT16 },
{ name: EmbeddingPrecision.FLOAT, value: EmbeddingPrecision.FLOAT },
];
const AdvancedEmbeddingFormPage = forwardRef<
FormikProps<any>,
AdvancedEmbeddingFormPageProps
>(({ updateAdvancedEmbeddingDetails, advancedEmbeddingDetails }, ref) => {
return (
<div className="py-4 rounded-lg max-w-4xl px-4 mx-auto">
<Formik
innerRef={ref}
initialValues={advancedEmbeddingDetails}
validationSchema={Yup.object().shape({
multilingual_expansion: Yup.array().of(Yup.string()),
multipass_indexing: Yup.boolean(),
disable_rerank_for_streaming: Yup.boolean(),
num_rerank: Yup.number(),
})}
onSubmit={async (_, { setSubmitting }) => {
setSubmitting(false);
}}
validate={(values) => {
// Call updateAdvancedEmbeddingDetails for each changed field
Object.entries(values).forEach(([key, value]) => {
updateAdvancedEmbeddingDetails(
key as keyof AdvancedSearchConfiguration,
value
);
});
}}
enableReinitialize={true}
>
{({ values }) => (
<Form>
<FieldArray name="multilingual_expansion">
{({ push, remove }) => (
<div className="w-full">
<Label>Multi-lingual Expansion</Label>
>(
(
{
updateAdvancedEmbeddingDetails,
advancedEmbeddingDetails,
embeddingProviderType,
onValidationChange,
},
ref
) => {
return (
<div className="py-4 rounded-lg max-w-4xl px-4 mx-auto">
<Formik
innerRef={ref}
initialValues={advancedEmbeddingDetails}
validationSchema={Yup.object().shape({
multilingual_expansion: Yup.array().of(Yup.string()),
multipass_indexing: Yup.boolean(),
disable_rerank_for_streaming: Yup.boolean(),
num_rerank: Yup.number()
.required("Number of results to rerank is required")
.min(1, "Must be at least 1"),
embedding_precision: Yup.string().nullable(),
reduced_dimension: Yup.number()
.nullable()
.test(
"positive",
"Must be larger than or equal to 256",
(value) => value === null || value === undefined || value >= 256
)
.test(
"openai",
"Reduced Dimensions is only supported for OpenAI embedding models",
(value) => {
return embeddingProviderType === "openai" || value === null;
}
),
})}
onSubmit={async (_, { setSubmitting }) => {
setSubmitting(false);
}}
validate={(values) => {
// Call updateAdvancedEmbeddingDetails for each changed field
Object.entries(values).forEach(([key, value]) => {
updateAdvancedEmbeddingDetails(
key as keyof AdvancedSearchConfiguration,
value
);
});
<SubLabel>Add additional languages to the search.</SubLabel>
{values.multilingual_expansion.map(
(_: any, index: number) => (
<div key={index} className="w-full flex mb-4">
<Field
name={`multilingual_expansion.${index}`}
className={`w-full bg-input text-sm p-2 border border-border-medium rounded-md
// Run validation and report errors
if (onValidationChange) {
// We'll return an empty object here since Yup will handle the actual validation
// But we need to check if there are any validation errors
const errors: Record<string, string> = {};
try {
// Manually validate against the schema
Yup.object()
.shape({
multilingual_expansion: Yup.array().of(Yup.string()),
multipass_indexing: Yup.boolean(),
disable_rerank_for_streaming: Yup.boolean(),
num_rerank: Yup.number()
.required("Number of results to rerank is required")
.min(1, "Must be at least 1"),
embedding_precision: Yup.string().nullable(),
reduced_dimension: Yup.number()
.nullable()
.test(
"positive",
"Must be larger than or equal to 256",
(value) =>
value === null || value === undefined || value >= 256
)
.test(
"openai",
"Reduced Dimensions is only supported for OpenAI embedding models",
(value) => {
return (
embeddingProviderType === "openai" || value === null
);
}
),
})
.validateSync(values, { abortEarly: false });
onValidationChange(true, {});
} catch (validationError) {
if (validationError instanceof Yup.ValidationError) {
validationError.inner.forEach((err) => {
if (err.path) {
errors[err.path] = err.message;
}
});
onValidationChange(false, errors);
}
}
}
return {}; // Return empty object as Formik will handle the errors
}}
enableReinitialize={true}
>
{({ values }) => (
<Form>
<FieldArray name="multilingual_expansion">
{({ push, remove }) => (
<div className="w-full">
<Label>Multi-lingual Expansion</Label>
<SubLabel>Add additional languages to the search.</SubLabel>
{values.multilingual_expansion.map(
(_: any, index: number) => (
<div key={index} className="w-full flex mb-4">
<Field
name={`multilingual_expansion.${index}`}
className={`w-full bg-input text-sm p-2 border border-border-medium rounded-md
focus:ring-2 focus:ring-indigo-500 focus:border-indigo-500 mr-2`}
/>
<button
type="button"
onClick={() => remove(index)}
className={`p-2 my-auto bg-input flex-none rounded-md
/>
<button
type="button"
onClick={() => remove(index)}
className={`p-2 my-auto bg-input flex-none rounded-md
bg-red-500 text-white hover:bg-red-600
focus:outline-none focus:ring-2 focus:ring-red-500 focus:ring-opacity-50`}
>
<TrashIcon className="text-white my-auto" />
</button>
</div>
)
)}
<button
type="button"
onClick={() => push("")}
className={`mt-2 p-2 bg-rose-500 text-xs text-white rounded-md flex items-center
>
<TrashIcon className="text-white my-auto" />
</button>
</div>
)
)}
<button
type="button"
onClick={() => push("")}
className={`mt-2 p-2 bg-rose-500 text-xs text-white rounded-md flex items-center
hover:bg-rose-600 focus:outline-none focus:ring-2 focus:ring-rose-500 focus:ring-opacity-50`}
>
<FaPlus className="mr-2" />
Add Language
</button>
</div>
)}
</FieldArray>
>
<FaPlus className="mr-2" />
Add Language
</button>
</div>
)}
</FieldArray>
<BooleanFormField
subtext="Enable multipass indexing for both mini and large chunks."
optional
label="Multipass Indexing"
name="multipass_indexing"
/>
<BooleanFormField
subtext="Disable reranking for streaming to improve response time."
optional
label="Disable Rerank for Streaming"
name="disable_rerank_for_streaming"
/>
<NumberInput
description="Number of results to rerank"
optional={false}
label="Number of Results to Rerank"
name="num_rerank"
/>
</Form>
)}
</Formik>
</div>
);
});
<BooleanFormField
subtext="Enable multipass indexing for both mini and large chunks."
optional
label="Multipass Indexing"
name="multipass_indexing"
/>
<BooleanFormField
subtext="Disable reranking for streaming to improve response time."
optional
label="Disable Rerank for Streaming"
name="disable_rerank_for_streaming"
/>
<NumberInput
description="Number of results to rerank"
optional={false}
label="Number of Results to Rerank"
name="num_rerank"
/>
<SelectorFormField
name="embedding_precision"
label="Embedding Precision"
options={embeddingPrecisionOptions}
subtext="Select the precision for embedding vectors. Lower precision uses less storage but may reduce accuracy."
/>
<NumberInput
description="Number of dimensions to reduce the embedding to.
Will reduce memory usage but may reduce accuracy.
If not specified, will just use the selected model's default dimensionality without any reduction.
Currently only supported for OpenAI embedding models"
optional={true}
label="Reduced Dimension"
name="reduced_dimension"
/>
</Form>
)}
</Formik>
</div>
);
}
);
export default AdvancedEmbeddingFormPage;
AdvancedEmbeddingFormPage.displayName = "AdvancedEmbeddingFormPage";

340
web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
View File

@@ -3,10 +3,16 @@ import { usePopup } from "@/components/admin/connectors/Popup";
import { HealthCheckBanner } from "@/components/health/healthcheck";
import { EmbeddingModelSelection } from "../EmbeddingModelSelectionForm";
import { useCallback, useEffect, useMemo, useState } from "react";
import { useCallback, useEffect, useMemo, useState, useRef } from "react";
import Text from "@/components/ui/text";
import { Button } from "@/components/ui/button";
import { ArrowLeft, ArrowRight, WarningCircle } from "@phosphor-icons/react";
import {
ArrowLeft,
ArrowRight,
WarningCircle,
CaretDown,
Warning,
} from "@phosphor-icons/react";
import {
CloudEmbeddingModel,
EmbeddingProvider,
@@ -19,16 +25,35 @@ import { ThreeDotsLoader } from "@/components/Loading";
import AdvancedEmbeddingFormPage from "./AdvancedEmbeddingFormPage";
import {
AdvancedSearchConfiguration,
EmbeddingPrecision,
RerankingDetails,
SavedSearchSettings,
} from "../interfaces";
import RerankingDetailsForm from "../RerankingFormPage";
import { useEmbeddingFormContext } from "@/components/context/EmbeddingContext";
import { Modal } from "@/components/Modal";
import { InstantSwitchConfirmModal } from "../modals/InstantSwitchConfirmModal";
import { useRouter } from "next/navigation";
import CardSection from "@/components/admin/CardSection";
import { combineSearchSettings } from "./utils";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import {
Tooltip,
TooltipContent,
TooltipProvider,
TooltipTrigger,
} from "@/components/ui/tooltip";
enum ReindexType {
REINDEX = "reindex",
INSTANT = "instant",
}
export default function EmbeddingForm() {
const { formStep, nextFormStep, prevFormStep } = useEmbeddingFormContext();
@@ -43,6 +68,8 @@ export default function EmbeddingForm() {
disable_rerank_for_streaming: false,
api_url: null,
num_rerank: 0,
embedding_precision: EmbeddingPrecision.FLOAT,
reduced_dimension: null,
});
const [rerankingDetails, setRerankingDetails] = useState<RerankingDetails>({
@@ -52,6 +79,19 @@ export default function EmbeddingForm() {
rerank_api_url: null,
});
const [reindexType, setReindexType] = useState<ReindexType>(
ReindexType.REINDEX
);
const [formErrors, setFormErrors] = useState<Record<string, string>>({});
const [isFormValid, setIsFormValid] = useState(true);
const [rerankFormErrors, setRerankFormErrors] = useState<
Record<string, string>
>({});
const [isRerankFormValid, setIsRerankFormValid] = useState(true);
const advancedFormRef = useRef(null);
const rerankFormRef = useRef(null);
const updateAdvancedEmbeddingDetails = (
key: keyof AdvancedSearchConfiguration,
value: any
@@ -82,6 +122,8 @@ export default function EmbeddingForm() {
};
const [displayPoorModelName, setDisplayPoorModelName] = useState(true);
const [showPoorModel, setShowPoorModel] = useState(false);
const [showInstantSwitchConfirm, setShowInstantSwitchConfirm] =
useState(false);
const [modelTab, setModelTab] = useState<"open" | "cloud" | null>(null);
const {
@@ -115,6 +157,8 @@ export default function EmbeddingForm() {
searchSettings.disable_rerank_for_streaming,
num_rerank: searchSettings.num_rerank,
api_url: null,
embedding_precision: searchSettings.embedding_precision,
reduced_dimension: searchSettings.reduced_dimension,
});
setRerankingDetails({
@@ -146,17 +190,14 @@ export default function EmbeddingForm() {
}
}, [currentEmbeddingModel]);
const handleReindex = async () => {
const update = await updateSearch();
if (update) {
await onConfirm();
}
};
const needsReIndex =
currentEmbeddingModel != selectedProvider ||
searchSettings?.multipass_indexing !=
advancedEmbeddingDetails.multipass_indexing;
advancedEmbeddingDetails.multipass_indexing ||
searchSettings?.embedding_precision !=
advancedEmbeddingDetails.embedding_precision ||
searchSettings?.reduced_dimension !=
advancedEmbeddingDetails.reduced_dimension;
const updateSearch = useCallback(async () => {
if (!selectedProvider) {
@@ -166,18 +207,44 @@ export default function EmbeddingForm() {
selectedProvider,
advancedEmbeddingDetails,
rerankingDetails,
selectedProvider.provider_type?.toLowerCase() as EmbeddingProvider | null
selectedProvider.provider_type?.toLowerCase() as EmbeddingProvider | null,
reindexType === ReindexType.REINDEX
);
const response = await updateSearchSettings(searchSettings);
if (response.ok) {
return true;
} else {
setPopup({ message: "Failed to update search settings", type: "error" });
setPopup({
message: "Failed to update search settings",
type: "error",
});
return false;
}
}, [selectedProvider, advancedEmbeddingDetails, rerankingDetails, setPopup]);
const handleValidationChange = useCallback(
(isValid: boolean, errors: Record<string, string>) => {
setIsFormValid(isValid);
setFormErrors(errors);
},
[]
);
const handleRerankValidationChange = useCallback(
(isValid: boolean, errors: Record<string, string>) => {
setIsRerankFormValid(isValid);
setRerankFormErrors(errors);
},
[]
);
// Combine validation states for both forms
const isOverallFormValid = isFormValid && isRerankFormValid;
const combinedFormErrors = useMemo(() => {
return { ...formErrors, ...rerankFormErrors };
}, [formErrors, rerankFormErrors]);
const ReIndexingButton = useMemo(() => {
const ReIndexingButtonComponent = ({
needsReIndex,
@@ -186,47 +253,204 @@ export default function EmbeddingForm() {
}) => {
return needsReIndex ? (
<div className="flex mx-auto gap-x-1 ml-auto items-center">
<button
className="enabled:cursor-pointer disabled:bg-accent/50 disabled:cursor-not-allowed bg-agent flex gap-x-1 items-center text-white py-2.5 px-3.5 text-sm font-regular rounded-sm"
onClick={handleReindex}
>
Re-index
</button>
<div className="relative group">
<WarningCircle
className="text-text-800 cursor-help"
size={20}
weight="fill"
/>
<div className="absolute z-10 invisible group-hover:visible bg-background-800 text-text-200 text-sm rounded-md shadow-md p-2 right-0 mt-1 w-64">
<p className="font-semibold mb-2">Needs re-indexing due to:</p>
<ul className="list-disc pl-5">
{currentEmbeddingModel != selectedProvider && (
<li>Changed embedding provider</li>
)}
{searchSettings?.multipass_indexing !=
advancedEmbeddingDetails.multipass_indexing && (
<li>Multipass indexing modification</li>
)}
</ul>
</div>
<div className="flex items-center">
<button
onClick={() => {
if (reindexType == ReindexType.INSTANT) {
setShowInstantSwitchConfirm(true);
} else {
handleReIndex();
navigateToEmbeddingPage("search settings");
}
}}
disabled={!isOverallFormValid}
className="
enabled:cursor-pointer
disabled:bg-accent/50
disabled:cursor-not-allowed
bg-agent
flex
items-center
justify-center
text-white
text-sm
font-regular
rounded-l-sm
py-2.5
px-3.5
transition-colors
hover:bg-white/10
text-center
w-32"
>
{reindexType == ReindexType.REINDEX
? "Re-index"
: "Instant Switch"}
</button>
<DropdownMenu>
<DropdownMenuTrigger asChild>
<button
disabled={!isOverallFormValid}
className="
enabled:cursor-pointer
disabled:bg-accent/50
disabled:cursor-not-allowed
bg-agent
flex
items-center
justify-center
text-white
text-sm
font-regular
rounded-r-sm
border-l
border-white/20
py-2.5
px-2
h-[40px]
w-[34px]
transition-colors
hover:bg-white/10"
>
<CaretDown className="h-4 w-4" />
</button>
</DropdownMenuTrigger>
<DropdownMenuContent>
<DropdownMenuItem
onClick={() => {
setReindexType(ReindexType.REINDEX);
}}
>
<TooltipProvider>
<Tooltip>
<TooltipTrigger className="w-full text-left">
(Recommended) Re-index
</TooltipTrigger>
<TooltipContent>
<p>
Re-runs all connectors in the background before
switching over. Takes longer but ensures no
degredation of search during the switch.
</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
</DropdownMenuItem>
<DropdownMenuItem
onClick={() => {
setReindexType(ReindexType.INSTANT);
}}
>
<TooltipProvider>
<Tooltip>
<TooltipTrigger className="w-full text-left">
Instant Switch
</TooltipTrigger>
<TooltipContent>
<p>
Immediately switches to new settings without
re-indexing. Searches will be degraded until the
re-indexing is complete.
</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
{isOverallFormValid && (
<div className="relative group">
<WarningCircle
className="text-text-800 cursor-help"
size={20}
weight="fill"
/>
<div className="absolute z-10 invisible group-hover:visible bg-background-800 text-text-200 text-sm rounded-md shadow-md p-2 right-0 mt-1 w-64">
<p className="font-semibold mb-2">Needs re-indexing due to:</p>
<ul className="list-disc pl-5">
{currentEmbeddingModel != selectedProvider && (
<li>Changed embedding provider</li>
)}
{searchSettings?.multipass_indexing !=
advancedEmbeddingDetails.multipass_indexing && (
<li>Multipass indexing modification</li>
)}
{searchSettings?.embedding_precision !=
advancedEmbeddingDetails.embedding_precision && (
<li>Embedding precision modification</li>
)}
{searchSettings?.reduced_dimension !=
advancedEmbeddingDetails.reduced_dimension && (
<li>Reduced dimension modification</li>
)}
</ul>
</div>
</div>
)}
{!isOverallFormValid &&
Object.keys(combinedFormErrors).length > 0 && (
<div className="relative group">
<Warning
className="text-red-500 cursor-help"
size={20}
weight="fill"
/>
<div className="absolute z-10 invisible group-hover:visible bg-background-800 text-text-200 text-sm rounded-md shadow-md p-2 right-0 mt-1 w-64">
<p className="font-semibold mb-2">Validation Errors:</p>
<ul className="list-disc pl-5">
{Object.entries(combinedFormErrors).map(
([field, error]) => (
<li key={field}>
{field}: {error}
</li>
)
)}
</ul>
</div>
</div>
)}
</div>
) : (
<button
className="enabled:cursor-pointer ml-auto disabled:bg-accent/50 disabled:cursor-not-allowed bg-agent flex mx-auto gap-x-1 items-center text-white py-2.5 px-3.5 text-sm font-regular rounded-sm"
onClick={async () => {
updateSearch();
navigateToEmbeddingPage("search settings");
}}
>
Update Search
</button>
<div className="flex mx-auto gap-x-1 ml-auto items-center">
<button
className="enabled:cursor-pointer ml-auto disabled:bg-accent/50 disabled:cursor-not-allowed bg-agent flex mx-auto gap-x-1 items-center text-white py-2.5 px-3.5 text-sm font-regular rounded-sm"
onClick={() => {
updateSearch();
navigateToEmbeddingPage("search settings");
}}
disabled={!isOverallFormValid}
>
Update Search
</button>
{!isOverallFormValid &&
Object.keys(combinedFormErrors).length > 0 && (
<div className="relative group">
<Warning
className="text-red-500 cursor-help"
size={20}
weight="fill"
/>
<div className="absolute z-10 invisible group-hover:visible bg-background-800 text-text-200 text-sm rounded-md shadow-md p-2 right-0 mt-1 w-64">
<p className="font-semibold mb-2 text-red-400">
Validation Errors:
</p>
<ul className="list-disc pl-5">
{Object.entries(combinedFormErrors).map(
([field, error]) => (
<li key={field}>{error}</li>
)
)}
</ul>
</div>
</div>
)}
</div>
);
};
ReIndexingButtonComponent.displayName = "ReIndexingButton";
return ReIndexingButtonComponent;
}, [needsReIndex, updateSearch]);
}, [needsReIndex, reindexType, isOverallFormValid, combinedFormErrors]);
if (!selectedProvider) {
return <ThreeDotsLoader />;
@@ -246,7 +470,7 @@ export default function EmbeddingForm() {
router.push("/admin/configuration/search?message=search-settings");
};
const onConfirm = async () => {
const handleReIndex = async () => {
if (!selectedProvider) {
return;
}
@@ -260,7 +484,8 @@ export default function EmbeddingForm() {
rerankingDetails,
selectedProvider.provider_type
?.toLowerCase()
.split(" ")[0] as EmbeddingProvider | null
.split(" ")[0] as EmbeddingProvider | null,
reindexType === ReindexType.REINDEX
);
} else {
// This is a locally hosted model
@@ -268,7 +493,8 @@ export default function EmbeddingForm() {
selectedProvider,
advancedEmbeddingDetails,
rerankingDetails,
null
null,
reindexType === ReindexType.REINDEX
);
}
@@ -381,6 +607,17 @@ export default function EmbeddingForm() {
</Modal>
)}
{showInstantSwitchConfirm && (
<InstantSwitchConfirmModal
onClose={() => setShowInstantSwitchConfirm(false)}
onConfirm={() => {
setShowInstantSwitchConfirm(false);
handleReIndex();
navigateToEmbeddingPage("search settings");
}}
/>
)}
{formStep == 1 && (
<>
<h2 className="text-2xl font-bold mb-4 text-text-800">
@@ -395,6 +632,7 @@ export default function EmbeddingForm() {
<CardSection>
<RerankingDetailsForm
ref={rerankFormRef}
setModelTab={setModelTab}
modelTab={
originalRerankingDetails.rerank_model_name
@@ -404,6 +642,7 @@ export default function EmbeddingForm() {
currentRerankingDetails={rerankingDetails}
originalRerankingDetails={originalRerankingDetails}
setRerankingDetails={setRerankingDetails}
onValidationChange={handleRerankValidationChange}
/>
</CardSection>
@@ -444,8 +683,11 @@ export default function EmbeddingForm() {
<CardSection>
<AdvancedEmbeddingFormPage
ref={advancedFormRef}
advancedEmbeddingDetails={advancedEmbeddingDetails}
updateAdvancedEmbeddingDetails={updateAdvancedEmbeddingDetails}
embeddingProviderType={selectedProvider.provider_type}
onValidationChange={handleValidationChange}
/>
</CardSection>

2
web/src/app/admin/embeddings/pages/OpenEmbeddingPage.tsx
View File

@@ -16,7 +16,7 @@ export default function OpenEmbeddingPage({
onSelectOpenSource,
selectedProvider,
}: {
onSelectOpenSource: (model: HostedEmbeddingModel) => Promise<void>;
onSelectOpenSource: (model: HostedEmbeddingModel) => void;
selectedProvider: HostedEmbeddingModel | CloudEmbeddingModel;
}) {
const [configureModel, setConfigureModel] = useState(false);

4
web/src/app/admin/embeddings/pages/utils.ts
View File

@@ -63,12 +63,14 @@ export const combineSearchSettings = (
selectedProvider: CloudEmbeddingProvider | HostedEmbeddingModel,
advancedEmbeddingDetails: AdvancedSearchConfiguration,
rerankingDetails: RerankingDetails,
provider_type: EmbeddingProvider | null
provider_type: EmbeddingProvider | null,
background_reindex_enabled: boolean
): SavedSearchSettings => {
return {
...selectedProvider,
...advancedEmbeddingDetails,
...rerankingDetails,
provider_type: provider_type,
background_reindex_enabled,
};
};

10
web/src/components/admin/connectors/Field.tsx
View File

@@ -51,13 +51,13 @@ export function Label({
className?: string;
}) {
return (
<div
className={`block text-text-darker font-medium base ${className} ${
small ? "text-xs" : "text-sm"
<label
className={`block font-medium text-text-700 dark:text-neutral-100 ${className} ${
small ? "text-sm" : "text-base"
}`}
>
{children}
</div>
</label>
);
}
@@ -686,7 +686,7 @@ export function SelectorFormField({
defaultValue,
tooltip,
includeReset = false,
fontSize = "sm",
fontSize = "md",
small = false,
}: SelectorFormFieldProps) {
const [field] = useField<string>(name);

1
web/src/components/embedding/ReindexingProgressTable.tsx
View File

@@ -29,6 +29,7 @@ export function ReindexingProgressTable({
<TableHead className="w-1/7 sm:w-1/5">Connector Name</TableHead>
<TableHead className="w-3/7 sm:w-1/5">Status</TableHead>
<TableHead className="w-3/7 sm:w-1/5">Docs Re-Indexed</TableHead>
<TableHead className="w-3/7 sm:w-1/5"></TableHead>
</TableRow>
</TableHeader>
<TableBody>

1
web/src/components/embedding/interfaces.tsx
View File

@@ -55,6 +55,7 @@ export interface EmbeddingModelDescriptor {
api_version?: string | null;
deployment_name?: string | null;
index_name: string | null;
background_reindex_enabled?: boolean;
}
export interface CloudEmbeddingModel extends EmbeddingModelDescriptor {