highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-04-07 19:38:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

misc improvements

Browse Source
This commit is contained in:
pablonyx 2025-03-23 12:13:07 -07:00
parent 9e720425c2
commit 63f4c9aea7
8 changed files with 400 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/onyx/server/user_documents/api.py
View File

@ -365,7 +365,7 @@ def create_file_from_link(
soup = BeautifulSoup(content, "html.parser")
parsed_html = web_html_cleanup(soup, mintlify_cleanup_enabled=False)
file_name = f"{parsed_html.title or 'Untitled'}"
file_name = f"{parsed_html.title or 'Untitled'}.txt"
file_content = parsed_html.cleaned_text.encode()
file = UploadFile(filename=file_name, file=io.BytesIO(file_content))

4
backend/onyx/server/user_documents/models.py
View File

@ -35,7 +35,9 @@ class UserFileSnapshot(BaseModel):
def from_model(cls, model: UserFile) -> "UserFileSnapshot":
return cls(
id=model.id,
name=model.name,
name=model.name[:-4]
if model.link_url and model.name.endswith(".txt")
else model.name,
folder_id=model.folder_id,
document_id=model.document_id,
user_id=model.user_id,

3
web/Dockerfile
View File

@ -27,6 +27,9 @@ COPY . .
# Install dependencies
RUN npm ci
# Install Sharp with platform-specific dependencies for Alpine Linux on ARM64
RUN npm install --platform=linuxmusl --arch=arm64 sharp
# needed to get the `standalone` dir we expect later
ENV NEXT_PRIVATE_STANDALONE=true

148
web/src/app/admin/assistants/AssistantEditor.tsx
View File

@ -853,10 +853,7 @@ export function AssistantEditor({
<Separator />
<div className="flex gap-x-2 py-2 flex justify-start">
<div>
<div
className="flex items-start gap-x-2
"
>
<div className="flex items-start gap-x-2">
<p className="block font-medium text-sm">
Knowledge
</p>
@ -895,9 +892,6 @@ export function AssistantEditor({
</TooltipProvider>
</div>
</div>
<p className="text-sm text-neutral-700 dark:text-neutral-400">
Attach additional unique knowledge to this assistant
</p>
</div>
</div>
</>
@ -907,25 +901,51 @@ export function AssistantEditor({
<div>
{canShowKnowledgeSource && (
<>
<TabToggle
options={[
{
id: "user_files",
label: "User Knowledge",
icon: <FileIcon size={16} />,
},
{
id: "team_knowledge",
label: "Team Knowledge",
icon: <BookIcon size={16} />,
},
]}
value={values.knowledge_source}
onChange={(value) => {
setFieldValue("knowledge_source", value);
}}
className="mt-2 mb-4 w-full max-w-sm"
/>
<div className="mt-1.5 mb-2.5">
<div className="flex gap-2.5">
<div
className={`w-[150px] h-[110px] rounded-lg border flex flex-col items-center justify-center cursor-pointer transition-all ${
values.knowledge_source === "user_files"
? "border-2 border-blue-500 bg-blue-50 dark:bg-blue-950/20"
: "border-gray-200 hover:border-gray-300 dark:border-gray-700 dark:hover:border-gray-600"
}`}
onClick={() =>
setFieldValue(
"knowledge_source",
"user_files"
)
}
>
<div className="text-blue-500 mb-2">
<FileIcon size={24} />
</div>
<p className="font-medium text-xs">
User Knowledge
</p>
</div>
<div
className={`w-[150px] h-[110px] rounded-lg border flex flex-col items-center justify-center cursor-pointer transition-all ${
values.knowledge_source === "team_knowledge"
? "border-2 border-blue-500 bg-blue-50 dark:bg-blue-950/20"
: "border-gray-200 hover:border-gray-300 dark:border-gray-700 dark:hover:border-gray-600"
}`}
onClick={() =>
setFieldValue(
"knowledge_source",
"team_knowledge"
)
}
>
<div className="text-blue-500 mb-2">
<BookIcon size={24} />
</div>
<p className="font-medium text-xs">
Team Knowledge
</p>
</div>
</div>
</div>
</>
)}
@ -933,76 +953,22 @@ export function AssistantEditor({
!existingPersona?.is_default_persona &&
!admin && (
<div className="mt-4">
<div className="flex justify-start gap-x-2 items-center">
<Label>User Knowledge</Label>
<span
className="cursor-pointer text-xs text-primary hover:underline"
onClick={() => setFilePickerModalOpen(true)}
>
Attach Files and Groups
</span>
</div>
<SubLabel>
Select which of your user files and groups this
Assistant should use to inform its responses. If
none are specified, the Assistant will not have
access to any user-specific documents.
</SubLabel>
{(selectedFiles.length > 0 ||
selectedFolders.length > 0) && (
<div className="mt-2 mb-4">
<h4 className="text-xs font-normal mb-2">
Selected Files and Folders
</h4>
<div className="flex flex-wrap gap-2">
{selectedFiles.map((file: FileResponse) => (
<SourceChip
key={file.id}
onRemove={() => {
removeSelectedFile(file);
setFieldValue(
"selectedFiles",
values.selectedFiles.filter(
(f: FileResponse) =>
f.id !== file.id
)
);
}}
title={file.name}
icon={<FileIcon size={12} />}
/>
))}
{selectedFolders.map(
(folder: FolderResponse) => (
<SourceChip
key={folder.id}
onRemove={() => {
removeSelectedFolder(folder);
setFieldValue(
"selectedFolders",
values.selectedFolders.filter(
(f: FolderResponse) =>
f.id !== folder.id
)
);
}}
title={folder.name}
icon={<FolderIcon size={12} />}
/>
)
)}
</div>
</div>
)}
<Button
type="button"
variant="outline"
size="sm"
className="text-xs flex justify-start gap-x-2"
onClick={() => setFilePickerModalOpen(true)}
>
<FileIcon size={14} />
Select knowledge
</Button>
</div>
)}
{values.knowledge_source === "team_knowledge" &&
ccPairs.length > 0 && (
<div className="mt-4">
<Label>Team Knowledge</Label>
<div>
<SubLabel>
<>
@ -1013,7 +979,7 @@ export function AssistantEditor({
className="font-semibold underline hover:underline text-text"
target="_blank"
>
Team Document Sets
Document Sets
</Link>
) : (
"Team Document Sets"

164
web/src/app/chat/input/LLMPopover.tsx
View File

@ -3,6 +3,7 @@ import React, {
useEffect,
useCallback,
useLayoutEffect,
useMemo,
} from "react";
import {
Popover,
@ -51,51 +52,66 @@ export default function LLMPopover({
const [isOpen, setIsOpen] = useState(false);
const { user } = useUser();
const llmOptionsByProvider: {
[provider: string]: {
name: string;
value: string;
icon: React.FC<{ size?: number; className?: string }>;
}[];
} = {};
// Memoize the options to prevent unnecessary recalculations
const {
llmOptionsByProvider,
llmOptions,
defaultProvider,
defaultModelDisplayName,
} = useMemo(() => {
const llmOptionsByProvider: {
[provider: string]: {
name: string;
value: string;
icon: React.FC<{ size?: number; className?: string }>;
}[];
} = {};
const uniqueModelNames = new Set<string>();
const uniqueModelNames = new Set<string>();
llmProviders.forEach((llmProvider) => {
if (!llmOptionsByProvider[llmProvider.provider]) {
llmOptionsByProvider[llmProvider.provider] = [];
}
(llmProvider.display_model_names || llmProvider.model_names).forEach(
(modelName) => {
if (!uniqueModelNames.has(modelName)) {
uniqueModelNames.add(modelName);
llmOptionsByProvider[llmProvider.provider].push({
name: modelName,
value: structureValue(
llmProvider.name,
llmProvider.provider,
modelName
),
icon: getProviderIcon(llmProvider.provider, modelName),
});
}
llmProviders.forEach((llmProvider) => {
if (!llmOptionsByProvider[llmProvider.provider]) {
llmOptionsByProvider[llmProvider.provider] = [];
}
(llmProvider.display_model_names || llmProvider.model_names).forEach(
(modelName) => {
if (!uniqueModelNames.has(modelName)) {
uniqueModelNames.add(modelName);
llmOptionsByProvider[llmProvider.provider].push({
name: modelName,
value: structureValue(
llmProvider.name,
llmProvider.provider,
modelName
),
icon: getProviderIcon(llmProvider.provider, modelName),
});
}
}
);
});
const llmOptions = Object.entries(llmOptionsByProvider).flatMap(
([provider, options]) => [...options]
);
});
const llmOptions = Object.entries(llmOptionsByProvider).flatMap(
([provider, options]) => [...options]
);
const defaultProvider = llmProviders.find(
(llmProvider) => llmProvider.is_default_provider
);
const defaultProvider = llmProviders.find(
(llmProvider) => llmProvider.is_default_provider
);
const defaultModelName = defaultProvider?.default_model_name;
const defaultModelDisplayName = defaultModelName
? getDisplayNameForModel(defaultModelName)
: null;
const defaultModelName = defaultProvider?.default_model_name;
const defaultModelDisplayName = defaultModelName
? getDisplayNameForModel(defaultModelName)
: null;
return {
llmOptionsByProvider,
llmOptions,
defaultProvider,
defaultModelDisplayName,
};
}, [llmProviders]);
const [localTemperature, setLocalTemperature] = useState(
llmManager.temperature ?? 0.5
@ -105,42 +121,52 @@ export default function LLMPopover({
setLocalTemperature(llmManager.temperature ?? 0.5);
}, [llmManager.temperature]);
const handleTemperatureChange = (value: number[]) => {
// Use useCallback to prevent function recreation
const handleTemperatureChange = useCallback((value: number[]) => {
setLocalTemperature(value[0]);
};
}, []);
const handleTemperatureChangeComplete = (value: number[]) => {
llmManager.updateTemperature(value[0]);
};
const handleTemperatureChangeComplete = useCallback(
(value: number[]) => {
llmManager.updateTemperature(value[0]);
},
[llmManager]
);
// Memoize trigger content to prevent rerendering
const triggerContent = useMemo(
() => (
<button
className="dark:text-[#fff] text-[#000] focus:outline-none"
data-testid="llm-popover-trigger"
>
<ChatInputOption
minimize
toggle
flexPriority="stiff"
name={getDisplayNameForModel(
llmManager?.currentLlm.modelName ||
defaultModelDisplayName ||
"Models"
)}
Icon={getProviderIcon(
llmManager?.currentLlm.provider ||
defaultProvider?.provider ||
"anthropic",
llmManager?.currentLlm.modelName ||
defaultProvider?.default_model_name ||
"claude-3-5-sonnet-20240620"
)}
tooltipContent="Switch models"
/>
</button>
),
[defaultModelDisplayName, defaultProvider, llmManager?.currentLlm]
);
return (
<Popover open={isOpen} onOpenChange={setIsOpen}>
<PopoverTrigger asChild>
<button
className="dark:text-[#fff] text-[#000] focus:outline-none"
data-testid="llm-popover-trigger"
>
<ChatInputOption
minimize
toggle
flexPriority="stiff"
name={getDisplayNameForModel(
llmManager?.currentLlm.modelName ||
defaultModelDisplayName ||
"Models"
)}
Icon={getProviderIcon(
llmManager?.currentLlm.provider ||
defaultProvider?.provider ||
"anthropic",
llmManager?.currentLlm.modelName ||
defaultProvider?.default_model_name ||
"claude-3-5-sonnet-20240620"
)}
tooltipContent="Switch models"
/>
</button>
</PopoverTrigger>
<PopoverTrigger asChild>{triggerContent}</PopoverTrigger>
<PopoverContent
align="start"
className="w-64 p-1 bg-background border border-background-200 rounded-md shadow-lg flex flex-col"

209
web/src/app/chat/my-documents/[id]/components/DocumentList.tsx
View File

@ -1,4 +1,4 @@
import React, { useState, useEffect } from "react";
import React, { useState, useEffect, useCallback, useMemo } from "react";
import {
FileResponse,
FolderResponse,
@ -9,7 +9,15 @@ import {
SkeletonFileListItem,
} from "../../components/FileListItem";
import { Button } from "@/components/ui/button";
import { Loader2, ArrowUp, ArrowDown, AlertCircle, X } from "lucide-react";
import {
Loader2,
ArrowUp,
ArrowDown,
AlertCircle,
X,
RefreshCw,
Trash2,
} from "lucide-react";
import { MinimalOnyxDocument } from "@/lib/search/interfaces";
import TextView from "@/components/chat/TextView";
import { Input } from "@/components/ui/input";
@ -18,6 +26,11 @@ import { useDocumentSelection } from "@/app/chat/useDocumentSelection";
import { getDisplayNameForModel } from "@/lib/hooks";
import { SortType, SortDirection } from "../UserFolderContent";
import { CircularProgress } from "./upload/CircularProgress";
import {
Popover,
PopoverContent,
PopoverTrigger,
} from "@/components/ui/popover";
// Define a type for uploading files that includes progress
interface UploadingFile {
@ -25,6 +38,13 @@ interface UploadingFile {
progress: number;
}
// Add interface for failed uploads
interface FailedUpload {
name: string;
error: string;
isPopoverOpen: boolean;
}
interface DocumentListProps {
files: FileResponse[];
onRename: (
@ -125,6 +145,8 @@ export const DocumentList: React.FC<DocumentListProps> = ({
};
const [uploadingFiles, setUploadingFiles] = useState<UploadingFile[]>([]);
const [completedFiles, setCompletedFiles] = useState<string[]>([]);
// Add state for failed uploads
const [failedUploads, setFailedUploads] = useState<FailedUpload[]>([]);
const [refreshInterval, setRefreshInterval] = useState<NodeJS.Timeout | null>(
null
);
@ -158,9 +180,54 @@ export const DocumentList: React.FC<DocumentListProps> = ({
startRefreshInterval();
} catch (error) {
console.error("Error creating file from link:", error);
// Remove from uploading files
setUploadingFiles((prev) => prev.filter((file) => file.name !== url));
// Add to failed uploads with isPopoverOpen initialized to false
setFailedUploads((prev) => [
...prev,
{
name: url,
error:
error instanceof Error ? error.message : "Failed to upload file",
isPopoverOpen: false,
},
]);
}
};
// Add handler for retrying failed uploads
const handleRetryUpload = async (url: string) => {
// Remove from failed uploads
setFailedUploads((prev) => prev.filter((file) => file.name !== url));
// Add back to uploading files
setUploadingFiles((prev) => [...prev, { name: url, progress: 0 }]);
try {
await createFileFromLink(url, folderId);
startRefreshInterval();
} catch (error) {
console.error("Error retrying file upload from link:", error);
// Remove from uploading files again
setUploadingFiles((prev) => prev.filter((file) => file.name !== url));
// Add back to failed uploads with isPopoverOpen initialized to false
setFailedUploads((prev) => [
...prev,
{
name: url,
error:
error instanceof Error ? error.message : "Failed to upload file",
isPopoverOpen: false,
},
]);
}
};
// Add handler for deleting failed uploads
const handleDeleteFailedUpload = (url: string) => {
setFailedUploads((prev) => prev.filter((file) => file.name !== url));
};
const handleFileUpload = (files: File[]) => {
const fileObjects = files.map((file) => ({
name: file.name,
@ -286,6 +353,8 @@ export const DocumentList: React.FC<DocumentListProps> = ({
// Get the hostname (domain) from the URL
const url = new URL(uploadingFile.name);
const hostname = url.hostname;
alert("checking for " + hostname);
alert(JSON.stringify(files));
// Look for recently added files that might match this URL
const isUploaded = files.some(
@ -304,7 +373,7 @@ export const DocumentList: React.FC<DocumentListProps> = ({
return isUploaded;
} catch (e) {
console.error("Failed to parse URL:", e);
return false; // Force continued checking
return false;
}
}
@ -349,9 +418,20 @@ export const DocumentList: React.FC<DocumentListProps> = ({
// For URLs, check if any file contains the hostname
const url = new URL(uploadingFile.name);
const hostname = url.hostname;
const fullUrl = uploadingFile.name;
return !files.some((file) =>
file.name.toLowerCase().includes(hostname.toLowerCase())
return (
// !files.some((file) =>
// file.name.toLowerCase().includes(hostname.toLowerCase())
// ) &&
!files.some(
(file) =>
file.link_url &&
// (file.link_url
// .toLowerCase()
// .includes(hostname.toLowerCase()) ||
file.link_url.toLowerCase() === fullUrl.toLowerCase()
)
);
} catch (e) {
console.error("Failed to parse URL:", e);
@ -394,6 +474,18 @@ export const DocumentList: React.FC<DocumentListProps> = ({
startRefreshInterval();
};
// Wrap in useCallback to prevent function recreation on each render
const toggleFailedUploadPopover = useCallback(
(index: number, isOpen: boolean) => {
setFailedUploads((prev) =>
prev.map((item, i) =>
i === index ? { ...item, isPopoverOpen: isOpen } : item
)
);
},
[]
);
return (
<>
<div className="flex flex-col h-full">
@ -550,13 +642,108 @@ export const DocumentList: React.FC<DocumentListProps> = ({
</div>
))}
{sortedFiles.length === 0 && uploadingFiles.length === 0 && (
<div className="text-center py-8 text-neutral-500 dark:text-neutral-400">
{searchQuery
? "No documents match your search."
: "No documents in this folder yet. Upload files or add URLs to get started."}
</div>
{/* Add failed uploads display with popover */}
{useMemo(
() =>
failedUploads.map((failedUpload, index) => (
<div
key={`failed-${index}`}
className="group relative mr-8 flex items-center border-b border-border dark:border-border-200 hover:bg-red-50/30 dark:hover:bg-red-900/10 py-4 px-4 transition-all ease-in-out bg-red-50/20 dark:bg-red-900/5"
>
<div className="flex items-center flex-1 min-w-0">
<div className="flex items-center gap-3 w-[40%] min-w-0">
<Popover
open={failedUpload.isPopoverOpen}
onOpenChange={(open) =>
toggleFailedUploadPopover(index, open)
}
>
<PopoverTrigger
onClick={(e) => e.stopPropagation()}
asChild
>
<div className="text-red-500 cursor-pointer">
<AlertCircle className="h-4 w-4" />
</div>
</PopoverTrigger>
<PopoverContent className="w-56 p-3 shadow-lg rounded-md border border-neutral-200 dark:border-neutral-800">
<div className="flex flex-col gap-3">
<div className="flex items-center gap-2">
<p className="text-xs font-medium text-red-500">
Upload failed.
<br />
You can retry the upload or remove it from
the list.
</p>
</div>
<div className="flex flex-col gap-2">
<Button
variant="outline"
size="sm"
className="w-full justify-start text-sm font-medium hover:bg-neutral-100 dark:hover:bg-neutral-800 transition-colors"
onClick={(e) => {
e.preventDefault();
e.stopPropagation();
toggleFailedUploadPopover(index, false);
handleRetryUpload(failedUpload.name);
}}
>
<RefreshCw className="mr-2 h-3.5 w-3.5" />
Retry Upload
</Button>
<Button
variant="outline"
size="sm"
className="w-full justify-start text-sm font-medium text-red-500 hover:bg-red-50 dark:hover:bg-red-900/20 hover:text-red-600 transition-colors"
onClick={(e) => {
e.stopPropagation();
toggleFailedUploadPopover(index, false);
handleDeleteFailedUpload(
failedUpload.name
);
}}
>
<Trash2 className="mr-2 h-3.5 w-3.5" />
Remove
</Button>
</div>
</div>
</PopoverContent>
</Popover>
<span className="truncate text-sm text-text-dark dark:text-text-dark">
{failedUpload.name.startsWith("http")
? `${failedUpload.name.substring(0, 30)}${
failedUpload.name.length > 30 ? "..." : ""
}`
: failedUpload.name}
</span>
</div>
<div className="w-[30%] text-sm text-red-500 dark:text-red-400">
Upload failed
</div>
<div className="w-[30%] flex items-center gap-2">
{/* Removed inline buttons as we now use the popover */}
</div>
</div>
</div>
)),
[
failedUploads,
toggleFailedUploadPopover,
handleRetryUpload,
handleDeleteFailedUpload,
]
)}
{sortedFiles.length === 0 &&
uploadingFiles.length === 0 &&
failedUploads.length === 0 && (
<div className="text-center py-8 text-neutral-500 dark:text-neutral-400">
{searchQuery
? "No documents match your search."
: "No documents in this folder yet. Upload files or add URLs to get started."}
</div>
)}
</>
)}
</div>

21
web/src/app/chat/my-documents/components/FileListItem.tsx
View File

@ -1,4 +1,4 @@
import React, { useState, useEffect } from "react";
import React, { useState, useEffect, useCallback } from "react";
import { Checkbox } from "@/components/ui/checkbox";
import { File, File as FileIcon, Loader, MoreHorizontal } from "lucide-react";
import { Button } from "@/components/ui/button";
@ -68,6 +68,7 @@ export const FileListItem: React.FC<FileListItemProps> = ({
const { setPopup, popup } = usePopup();
const [showMoveOptions, setShowMoveOptions] = useState(false);
const [indexingStatus, setIndexingStatus] = useState<boolean | null>(null);
const [isPopoverOpen, setIsPopoverOpen] = useState(false);
const { getFilesIndexingStatus, refreshFolderDetails } =
useDocumentsContext();
@ -96,9 +97,9 @@ export const FileListItem: React.FC<FileListItemProps> = ({
onMove(file.id, targetFolderId);
setShowMoveOptions(false);
};
const FailureWithPopover = () => {
const FailureWithPopover = useCallback(() => {
return (
<Popover>
<Popover open={isPopoverOpen} onOpenChange={setIsPopoverOpen}>
<PopoverTrigger onClick={(e) => e.stopPropagation()} asChild>
<div className="text-red-500 cursor-pointer">
<FiAlertTriangle className="h-4 w-4" />
@ -122,6 +123,7 @@ export const FileListItem: React.FC<FileListItemProps> = ({
onClick={(e) => {
e.preventDefault();
e.stopPropagation();
setIsPopoverOpen(false);
fetch(`/api/user/file/reindex`, {
method: "POST",
headers: {
@ -133,7 +135,7 @@ export const FileListItem: React.FC<FileListItemProps> = ({
if (!response.ok) {
throw new Error("Failed to reindex file");
}
setIndexingStatus(false); // Set to false to show indexing status
setIndexingStatus(false);
refreshFolderDetails();
setPopup({
type: "success",
@ -158,6 +160,7 @@ export const FileListItem: React.FC<FileListItemProps> = ({
className="w-full justify-start text-sm font-medium text-red-500 hover:bg-red-50 dark:hover:bg-red-900/20 hover:text-red-600 transition-colors"
onClick={(e) => {
e.stopPropagation();
setIsPopoverOpen(false);
handleDelete();
}}
>
@ -169,7 +172,15 @@ export const FileListItem: React.FC<FileListItemProps> = ({
</PopoverContent>
</Popover>
);
};
}, [
file.id,
handleDelete,
isPopoverOpen,
refreshFolderDetails,
setIndexingStatus,
setIsPopoverOpen,
setPopup,
]);
return (
<div

31
web/src/app/chat/my-documents/components/FilePicker.tsx
View File

@ -1343,7 +1343,7 @@ export const FilePickerModal: React.FC<FilePickerModalProps> = ({
try {
const response: FileResponse[] =
await createFileFromLink(url, currentFolder);
await createFileFromLink(url, -1);
if (response.length > 0) {
// Extract domain from URL to help with detection
@ -1386,9 +1386,32 @@ export const FilePickerModal: React.FC<FilePickerModalProps> = ({
selectedModel={selectedModel}
/>
</div>
<Button onClick={onSave} className="px-8 py-2 w-48">
{buttonContent || "Set Context"}
</Button>
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<div>
<Button
onClick={onSave}
className="px-8 py-2 w-48"
disabled={
isUploadingFile ||
isCreatingFileFromLink ||
uploadingFiles.length > 0
}
>
{buttonContent || "Set Context"}
</Button>
</div>
</TooltipTrigger>
{(isUploadingFile ||
isCreatingFileFromLink ||
uploadingFiles.length > 0) && (
<TooltipContent>
<p>Please wait for all files to finish uploading</p>
</TooltipContent>
)}
</Tooltip>
</TooltipProvider>
</div>
</div>
</div>