highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-04-10 12:59:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

misc improvements

Browse Source
This commit is contained in:
pablonyx 2025-03-22 11:22:09 -07:00
parent 2bad38eb61
commit b733fed89f
25 changed files with 389 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backend/alembic/versions/9aadf32dfeb4_add_user_files.py
View File

@ -1,7 +1,7 @@
"""add user files
Revision ID: 9aadf32dfeb4
Revises: df46c75b714e
Revises: 3781a5eb12cb
Create Date: 2025-01-26 16:08:21.551022
"""
@ -12,7 +12,7 @@ import datetime
# revision identifiers, used by Alembic.
revision = "9aadf32dfeb4"
down_revision = "df46c75b714e"
down_revision = "3781a5eb12cb"
branch_labels = None
depends_on = None
@ -42,6 +42,7 @@ def upgrade() -> None:
sa.ForeignKey("user_folder.id"),
nullable=True,
),
sa.Column("link_url", sa.String(), nullable=True),
sa.Column("token_count", sa.Integer(), nullable=True),
sa.Column("file_type", sa.String(), nullable=True),
sa.Column("file_id", sa.String(length=255), nullable=False),

1
backend/onyx/db/models.py
View File

@ -2430,6 +2430,7 @@ class UserFile(Base):
cc_pair: Mapped["ConnectorCredentialPair"] = relationship(
"ConnectorCredentialPair", back_populates="user_file"
)
link_url: Mapped[str | None] = mapped_column(String, nullable=True)
"""

2
backend/onyx/db/user_documents.py
View File

@ -38,6 +38,7 @@ def create_user_files(
folder_id: int | None,
user: User | None,
db_session: Session,
link_url: str | None = None,
) -> list[UserFile]:
upload_response = upload_files(files, db_session)
user_files = []
@ -50,6 +51,7 @@ def create_user_files(
document_id="USER_FILE_CONNECTOR__" + file_path,
name=file.filename,
token_count=None,
link_url=link_url,
)
db_session.add(new_file)
user_files.append(new_file)

7
backend/onyx/indexing/indexing_pipeline.py
View File

@ -12,9 +12,6 @@ from onyx.configs.app_configs import MAX_DOCUMENT_CHARS
from onyx.configs.constants import DEFAULT_BOOST
from onyx.configs.llm_configs import get_image_extraction_and_analysis_enabled
from onyx.configs.model_configs import USE_INFORMATION_CONTENT_CLASSIFICATION
from onyx.configs.constants import (
DEFAULT_BOOST,
)
from onyx.connectors.cross_connector_utils.miscellaneous_utils import (
get_experts_stores_representations,
)
@ -65,10 +62,10 @@ from onyx.indexing.models import IndexChunk
from onyx.indexing.models import UpdatableChunkData
from onyx.indexing.vector_db_insertion import write_chunks_to_vector_db_with_backoff
from onyx.llm.factory import get_default_llm_with_vision
from onyx.llm.factory import get_default_llms
from onyx.natural_language_processing.search_nlp_models import (
InformationContentClassificationModel,
)
from onyx.llm.factory import get_default_llms
from onyx.natural_language_processing.utils import get_tokenizer
from onyx.utils.logger import setup_logger
from onyx.utils.timing import log_function_time
@ -836,6 +833,8 @@ def index_doc_batch(
chunk_data=updatable_chunk_data, db_session=db_session
)
# Pause user file ccpairs
db_session.commit()
result = IndexingPipelineResult(

4
backend/onyx/main.py
View File

@ -211,7 +211,9 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
logger.notice("Generative AI Q&A disabled")
# fill up Postgres connection pools
print("Warming up connections")
await warm_up_connections()
print("Connections warmed up")
if not MULTI_TENANT:
# We cache this at the beginning so there is no delay in the first telemetry
@ -219,10 +221,12 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
get_or_generate_uuid()
# If we are multi-tenant, we need to only set up initial public tables
print("Setting up onyx")
with Session(engine) as db_session:
setup_onyx(db_session, POSTGRES_DEFAULT_SCHEMA)
else:
setup_multitenant_onyx()
print("onyz set up")
if not MULTI_TENANT:
# don't emit a metric for every pod rollover/restart

62
backend/onyx/server/user_documents/api.py
View File

@ -25,6 +25,7 @@ from onyx.db.connector_credential_pair import add_credential_to_connector
from onyx.db.credentials import create_credential
from onyx.db.engine import get_session
from onyx.db.enums import AccessType
from onyx.db.models import ConnectorCredentialPair
from onyx.db.models import User
from onyx.db.models import UserFile
from onyx.db.models import UserFolder
@ -363,11 +364,13 @@ def create_file_from_link(
soup = BeautifulSoup(content, "html.parser")
parsed_html = web_html_cleanup(soup, mintlify_cleanup_enabled=False)
file_name = f"{parsed_html.title or 'Untitled'}.txt"
file_name = f"{parsed_html.title or 'Untitled'}"
file_content = parsed_html.cleaned_text.encode()
file = UploadFile(filename=file_name, file=io.BytesIO(file_content))
user_files = create_user_files([file], request.folder_id, user, db_session)
user_files = create_user_files(
[file], request.folder_id, user, db_session, link_url=request.url
)
# Create connector and credential (same as in upload_user_files)
for user_file in user_files:
@ -448,6 +451,61 @@ def get_files_token_estimate(
return {"total_tokens": total_tokens}
class ReindexFileRequest(BaseModel):
file_id: int
@router.post("/user/file/reindex")
def reindex_file(
request: ReindexFileRequest,
user: User = Depends(current_user),
db_session: Session = Depends(get_session),
) -> MessageResponse:
user_id = user.id if user else None
user_file_to_reindex = (
db_session.query(UserFile)
.filter(UserFile.id == request.file_id, UserFile.user_id == user_id)
.first()
)
if not user_file_to_reindex:
raise HTTPException(status_code=404, detail="File not found")
if not user_file_to_reindex.cc_pair_id:
raise HTTPException(
status_code=400,
detail="File does not have an associated connector-credential pair",
)
# Get the connector id from the cc_pair
cc_pair = (
db_session.query(ConnectorCredentialPair)
.filter_by(id=user_file_to_reindex.cc_pair_id)
.first()
)
if not cc_pair:
raise HTTPException(
status_code=404, detail="Associated connector-credential pair not found"
)
# Trigger immediate reindexing with highest priority
tenant_id = get_current_tenant_id()
try:
trigger_indexing_for_cc_pair(
[], cc_pair.connector_id, True, tenant_id, db_session, is_user_file=True
)
return MessageResponse(
message="File reindexing has been triggered successfully"
)
except Exception as e:
logger.error(
f"Error triggering reindexing for file {request.file_id}: {str(e)}"
)
raise HTTPException(
status_code=500, detail=f"Failed to trigger reindexing: {str(e)}"
)
class BulkCleanupRequest(BaseModel):
folder_id: int
days_older_than: int | None = None

5
backend/onyx/server/user_documents/models.py
View File

@ -19,6 +19,8 @@ class UserFileSnapshot(BaseModel):
assistant_ids: List[int] = [] # List of assistant IDs
token_count: int | None
indexed: bool
link_url: str | None
failed: bool | None
@classmethod
def from_model(cls, model: UserFile) -> "UserFileSnapshot":
@ -32,9 +34,12 @@ class UserFileSnapshot(BaseModel):
created_at=model.created_at,
assistant_ids=[assistant.id for assistant in model.assistants],
token_count=model.token_count,
failed=len(model.cc_pair.index_attempts) > 0
and model.cc_pair.last_successful_index_time is None,
indexed=model.cc_pair.last_successful_index_time is not None
if model.cc_pair
else False,
link_url=model.link_url,
)

5
backend/onyx/setup.py
View File

@ -74,12 +74,15 @@ def setup_onyx(
The Tenant Service calls the tenants/create endpoint which runs this.
"""
print("checking and performing index swap")
check_and_perform_index_swap(db_session=db_session)
print("getting active search settings")
active_search_settings = get_active_search_settings(db_session)
search_settings = active_search_settings.primary
secondary_search_settings = active_search_settings.secondary
print("getting current search settings")
# search_settings = get_current_search_settings(db_session)
# multipass_config_1 = get_multipass_config(search_settings)
@ -91,6 +94,7 @@ def setup_onyx(
# Break bad state for thrashing indexes
if secondary_search_settings and DISABLE_INDEX_UPDATE_ON_SWAP:
print("expiring index attempts")
expire_index_attempts(
search_settings_id=search_settings.id, db_session=db_session
)
@ -99,6 +103,7 @@ def setup_onyx(
resync_cc_pair(cc_pair, db_session=db_session)
# Expire all old embedding models indexing attempts, technically redundant
print("cancelling indexing attempts past model")
cancel_indexing_attempts_past_model(db_session)
logger.notice(f'Using Embedding model: "{search_settings.model_name}"')

23
web/src/app/chat/ChatPage.tsx
View File

@ -189,6 +189,7 @@ export function ChatPage({
removeSelectedFolder,
clearSelectedItems,
folders: userFolders,
files: allUserFiles,
uploadFile,
removeSelectedFile,
currentMessageFiles,
@ -2769,11 +2770,25 @@ export function ChatPage({
? messageHistory[i + 1]
: undefined;
const userFiles = previousMessage?.files.filter(
(file) =>
file.type == ChatFileType.USER_KNOWLEDGE
const attachedFileDescriptors =
previousMessage?.files.filter(
(file) =>
file.type == ChatFileType.USER_KNOWLEDGE
);
const userFiles = allUserFiles?.filter((file) =>
attachedFileDescriptors?.some(
(descriptor) =>
descriptor.id === file.file_id
)
);
console.log("alluser files");
console.log(allUserFiles);
console.log(
"current attached file descriptors"
);
console.log(attachedFileDescriptors);
console.log("user files");
console.log(userFiles);
return (
<div
className="text-text"

14
web/src/app/chat/documentSidebar/DocumentResults.tsx
View File

@ -15,6 +15,7 @@ import {
FileSourceCard,
FileSourceCardInResults,
} from "../message/SourcesDisplay";
import { useDocumentsContext } from "../my-documents/DocumentsContext";
interface DocumentResultsProps {
agenticMessage: boolean;
humanMessage: Message | null;
@ -67,10 +68,14 @@ export const DocumentResults = forwardRef<HTMLDivElement, DocumentResultsProps>(
return () => clearTimeout(timer);
}, [selectedDocuments]);
const { files: allUserFiles } = useDocumentsContext();
const userFiles = humanMessage?.files.filter(
const humanFileDescriptors = humanMessage?.files.filter(
(file) => file.type == ChatFileType.USER_KNOWLEDGE
);
const userFiles = allUserFiles?.filter((file) =>
humanFileDescriptors?.some((descriptor) => descriptor.id === file.file_id)
);
const selectedDocumentIds =
selectedDocuments?.map((document) => document.document_id) || [];
@ -127,7 +132,12 @@ export const DocumentResults = forwardRef<HTMLDivElement, DocumentResultsProps>(
<FileSourceCardInResults
key={index}
document={file}
setPresentingDocument={setPresentingDocument}
setPresentingDocument={() =>
setPresentingDocument({
document_id: file.document_id,
semantic_identifier: file.file_id || null,
})
}
/>
))}
</div>

4
web/src/app/chat/interfaces.ts
View File

@ -50,6 +50,10 @@ export interface FileDescriptor {
isUploading?: boolean;
}
export interface FileDescriptorWithHighlights extends FileDescriptor {
match_highlights: string[];
}
export interface LLMRelevanceFilterPacket {
relevant_chunk_indices: number[];
}

13
web/src/app/chat/message/MemoizedTextComponents.tsx
View File

@ -10,12 +10,14 @@ import { SourceIcon } from "@/components/SourceIcon";
import { WebResultIcon } from "@/components/WebResultIcon";
import { SubQuestionDetail } from "../interfaces";
import { ValidSources } from "@/lib/types";
import { FileResponse } from "../my-documents/DocumentsContext";
export const MemoizedAnchor = memo(
({
docs,
subQuestions,
openQuestion,
userFiles,
href,
updatePresentingDocument,
children,
@ -23,6 +25,7 @@ export const MemoizedAnchor = memo(
subQuestions?: SubQuestionDetail[];
openQuestion?: (question: SubQuestionDetail) => void;
docs?: OnyxDocument[] | null;
userFiles?: FileResponse[] | null;
updatePresentingDocument: (doc: OnyxDocument) => void;
href?: string;
children: React.ReactNode;
@ -31,8 +34,14 @@ export const MemoizedAnchor = memo(
if (value?.startsWith("[") && value?.endsWith("]")) {
const match = value.match(/\[(D|Q)?(\d+)\]/);
if (match) {
const isSubQuestion = match[1] === "Q";
if (!isSubQuestion) {
const isUserFileCitation = userFiles?.length && userFiles.length > 0;
if (isUserFileCitation) {
const index = parseInt(match[2], 10) - 1;
const associatedUserFile = userFiles?.[index];
if (!associatedUserFile) {
return <a href={children as string}>{children}</a>;
}
} else if (!isUserFileCitation) {
const index = parseInt(match[2], 10) - 1;
const associatedDoc = docs?.[index];
if (!associatedDoc) {

66
web/src/app/chat/message/Messages.tsx
View File

@ -73,6 +73,7 @@ import rehypeKatex from "rehype-katex";
import "katex/dist/katex.min.css";
import { copyAll, handleCopy } from "./copyingUtils";
import { transformLinkUri } from "@/lib/utils";
import { FileResponse } from "../my-documents/DocumentsContext";
const TOOLS_WITH_CUSTOM_HANDLING = [
SEARCH_TOOL_NAME,
@ -164,6 +165,46 @@ function FileDisplay({
);
}
function FileResponseDisplay({
files,
alignBubble,
setPresentingDocument,
}: {
files: FileResponse[];
alignBubble?: boolean;
setPresentingDocument: (document: MinimalOnyxDocument) => void;
}) {
if (!files || files.length === 0) {
return null;
}
return (
<div
id="onyx-file-response"
className={`${alignBubble && "ml-auto"} mt-2 auto mb-4`}
>
<div className="flex flex-col gap-2">
{files.map((file) => {
return (
<div key={file.id} className="w-fit">
<DocumentPreview
fileName={file.name || file.document_id}
alignBubble={alignBubble}
open={() =>
setPresentingDocument({
document_id: file.document_id,
semantic_identifier: file.name || file.document_id,
})
}
/>
</div>
);
})}
</div>
</div>
);
}
export const AIMessage = ({
userKnowledgeFiles = [],
regenerate,
@ -195,7 +236,7 @@ export const AIMessage = ({
documentSidebarVisible,
removePadding,
}: {
userKnowledgeFiles?: FileDescriptor[];
userKnowledgeFiles?: FileResponse[];
index?: number;
shared?: boolean;
isActive?: boolean;
@ -323,6 +364,7 @@ export const AIMessage = ({
<MemoizedAnchor
updatePresentingDocument={setPresentingDocument!}
docs={docs}
userFiles={userKnowledgeFiles}
href={props.href}
>
{props.children}
@ -522,8 +564,11 @@ export const AIMessage = ({
<SourceCard
document={doc}
key={ind}
setPresentingDocument={
setPresentingDocument
setPresentingDocument={() =>
setPresentingDocument({
document_id: doc.document_id,
semantic_identifier: doc.document_id,
})
}
/>
))}
@ -555,15 +600,19 @@ export const AIMessage = ({
userKnowledgeFiles.length > 0 &&
userKnowledgeFiles
.slice(0, 2)
.map((file: FileDescriptor, ind: number) => (
.map((file: FileResponse, ind: number) => (
<FileSourceCard
key={ind}
document={file}
setPresentingDocument={
setPresentingDocument
setPresentingDocument={() =>
setPresentingDocument({
document_id: file.document_id,
semantic_identifier: file.name,
})
}
/>
))}
{userKnowledgeFiles.length > 2 && (
<FilesSeeMoreBlock
key={10}
@ -578,11 +627,12 @@ export const AIMessage = ({
</div>
</div>
)}
{content || userKnowledgeFiles || files ? (
{content || files ? (
<>
<FileDisplay
setPresentingDocument={setPresentingDocument}
files={userKnowledgeFiles || files || []}
files={files || []}
/>
{typeof content === "string" ? (
<div className="overflow-x-visible max-w-content-max">

3
web/src/app/chat/message/SearchSummary.tsx
View File

@ -16,6 +16,7 @@ import { ValidSources } from "@/lib/types";
import { useEffect, useRef, useState } from "react";
import { FiBook, FiCheck, FiEdit2, FiSearch, FiX } from "react-icons/fi";
import { FileDescriptor } from "../interfaces";
import { FileResponse } from "../my-documents/DocumentsContext";
export function ShowHideDocsButton({
messageId,
@ -250,7 +251,7 @@ export function SearchSummary({
export function UserKnowledgeFiles({
userKnowledgeFiles,
}: {
userKnowledgeFiles: FileDescriptor[];
userKnowledgeFiles: FileResponse[];
}): JSX.Element {
if (!userKnowledgeFiles || userKnowledgeFiles.length === 0) {
return <></>;

46
web/src/app/chat/message/SourcesDisplay.tsx
View File

@ -6,8 +6,9 @@ import { buildDocumentSummaryDisplay } from "@/components/search/DocumentDisplay
import { ValidSources } from "@/lib/types";
import { FiFileText } from "react-icons/fi";
import { FileDescriptor } from "../interfaces";
import { getFileIconFromFileName } from "@/lib/assistantIconUtils";
import { getFileIconFromFileNameAndLink } from "@/lib/assistantIconUtils";
import { truncateString } from "@/lib/utils";
import { FileResponse } from "../my-documents/DocumentsContext";
interface SourcesDisplayProps {
documents: OnyxDocument[];
@ -68,14 +69,21 @@ export const SourceCard: React.FC<{
};
export const FileSourceCard: React.FC<{
document: FileDescriptor;
setPresentingDocument: (document: MinimalOnyxDocument) => void;
document: FileResponse;
setPresentingDocument: (document: FileResponse) => void;
}> = ({ document, setPresentingDocument }) => {
const openDocument = () => {
if (document.link_url) {
window.open(document.link_url, "_blank");
} else {
setPresentingDocument(document as any);
}
};
const fileName = document.name || document.id;
return (
<button
onClick={() => setPresentingDocument(document as any)}
onClick={openDocument}
className="w-full max-w-[260px] h-[80px] p-3
text-left bg-accent-background hover:bg-accent-background-hovered dark:bg-accent-background-hovered dark:hover:bg-neutral-700/80
cursor-pointer rounded-lg
@ -91,14 +99,13 @@ export const FileSourceCard: React.FC<{
text-ellipsis
"
>
{truncateString(fileName, 45)}
Content from {fileName}
</div>
<div className="flex items-center gap-1 mt-1">
{getFileIconFromFileName(fileName)}
{getFileIconFromFileNameAndLink(document.name, document.link_url)}
<div className="text-text-700 text-xs leading-tight truncate flex-1 min-w-0">
Document
{truncateString(document.name, 45)}
</div>
</div>
</button>
@ -106,14 +113,19 @@ export const FileSourceCard: React.FC<{
};
export const FileSourceCardInResults: React.FC<{
document: FileDescriptor;
setPresentingDocument: (document: MinimalOnyxDocument) => void;
document: FileResponse;
setPresentingDocument: (document: FileResponse) => void;
}> = ({ document, setPresentingDocument }) => {
const fileName = document.name || document.id;
const openDocument = () => {
if (document.link_url) {
window.open(document.link_url, "_blank");
} else {
setPresentingDocument(document as any);
}
};
return (
<button
onClick={() => setPresentingDocument(document as any)}
onClick={openDocument}
className="w-full h-[80px] p-4
text-left bg-background hover:bg-neutral-100 dark:bg-neutral-800 dark:hover:bg-neutral-700
cursor-pointer rounded-lg
@ -130,13 +142,15 @@ export const FileSourceCardInResults: React.FC<{
text-ellipsis
"
>
{truncateString(fileName, 45)}
Content from {document.name}
</div>
<div className="flex items-center gap-2 mt-2">
<div className="flex-shrink-0">{getFileIconFromFileName(fileName)}</div>
<div className="flex-shrink-0">
{getFileIconFromFileNameAndLink(document.name, document.link_url)}
</div>
<div className="text-text-700 text-xs leading-tight truncate flex-1 min-w-0 font-medium">
Document
{truncateString(document.name, 45)}
</div>
</div>
</button>

5
web/src/app/chat/my-documents/DocumentsContext.tsx
View File

@ -34,7 +34,10 @@ export type FileResponse = {
assistant_ids?: number[];
indexed?: boolean;
created_at?: string;
file_id?: string;
file_type?: string;
link_url?: string | null;
failed?: boolean;
};
export interface FileUploadResponse {
@ -43,6 +46,7 @@ export interface FileUploadResponse {
export interface DocumentsContextType {
folders: FolderResponse[];
files: FileResponse[];
currentFolder: number | null;
presentingDocument: MinimalOnyxDocument | null;
searchQuery: string;
@ -486,6 +490,7 @@ export const DocumentsProvider: React.FC<DocumentsProviderProps> = ({
);
const value: DocumentsContextType = {
files: folders.map((folder) => folder.files).flat(),
folders,
currentFolder,
presentingDocument,

10
web/src/app/chat/my-documents/[id]/components/DocumentList.tsx
View File

@ -116,6 +116,13 @@ export const DocumentList: React.FC<DocumentListProps> = ({
}) => {
const [presentingDocument, setPresentingDocument] =
useState<FileResponse | null>(null);
const openDocument = (file: FileResponse) => {
if (file.link_url) {
window.open(file.link_url, "_blank");
} else {
setPresentingDocument(file);
}
};
const [uploadingFiles, setUploadingFiles] = useState<UploadingFile[]>([]);
const [completedFiles, setCompletedFiles] = useState<string[]>([]);
const [refreshInterval, setRefreshInterval] = useState<NodeJS.Timeout | null>(
@ -499,8 +506,9 @@ export const DocumentList: React.FC<DocumentListProps> = ({
onDownload={onDownload}
onMove={onMove}
folders={folders}
onSelect={() => setPresentingDocument(file)}
onSelect={() => openDocument(file)}
isIndexed={file.indexed || false}
failed={file.failed || false}
/>
)}
</div>

2
web/src/app/chat/my-documents/[id]/components/upload/FileUploadSection.tsx
View File

@ -535,7 +535,7 @@ export const FileUploadSection: React.FC<FileUploadSectionProps> = ({
{/* Content area - different for each mode but with consistent spacing */}
<div className="flex-1 w-full flex flex-col items-center justify-center mt-2">
<div className="flex items-center gap-2 w-full mx-4 max-w-md">
<div className="flex items-center gap-2 w-full px-4 max-w-md">
<input
ref={urlInputRef}
type="text"

87
web/src/app/chat/my-documents/components/FileListItem.tsx
View File

@ -18,12 +18,21 @@ import {
TooltipProvider,
TooltipTrigger,
} from "@/components/ui/tooltip";
import { FiDownload, FiEdit, FiTrash } from "react-icons/fi";
import {
FiAlertCircle,
FiAlertTriangle,
FiDownload,
FiEdit,
FiRefreshCw,
FiTrash,
FiTrash2,
} from "react-icons/fi";
import { getFormattedDateTime } from "@/lib/dateUtils";
import { getFileIconFromFileName } from "@/lib/assistantIconUtils";
import { getFileIconFromFileNameAndLink } from "@/lib/assistantIconUtils";
import { AnimatedDots } from "../[id]/components/DocumentList";
import { FolderMoveIcon } from "@/components/icons/icons";
import { truncateString } from "@/lib/utils";
import { triggerIndexing } from "@/app/admin/connector/[ccPairId]/lib";
interface FileListItemProps {
file: FileResponse;
@ -40,6 +49,7 @@ interface FileListItemProps {
onMove: (fileId: number, targetFolderId: number) => Promise<void>;
folders: FolderResponse[];
isIndexed: boolean;
failed: boolean;
}
export const FileListItem: React.FC<FileListItemProps> = ({
@ -52,6 +62,7 @@ export const FileListItem: React.FC<FileListItemProps> = ({
onMove,
folders,
isIndexed,
failed,
}) => {
const [showMoveOptions, setShowMoveOptions] = useState(false);
const [indexingStatus, setIndexingStatus] = useState<boolean | null>(null);
@ -82,6 +93,72 @@ export const FileListItem: React.FC<FileListItemProps> = ({
onMove(file.id, targetFolderId);
setShowMoveOptions(false);
};
const FailureWithPopover = () => {
return (
<Popover>
<PopoverTrigger onClick={(e) => e.stopPropagation()} asChild>
<div className="text-red-500 cursor-pointer">
<FiAlertTriangle className="h-4 w-4" />
</div>
</PopoverTrigger>
<PopoverContent className="w-56 p-3 shadow-lg rounded-md border border-neutral-200 dark:border-neutral-800">
<div className="flex flex-col gap-3">
<div className="flex items-center gap-2">
<p className="text-xs font-medium text-red-500">
Indexing failed.
<br />
You can attempt a reindex to continue using this file, or delete
the file.
</p>
</div>
<div className="flex flex-col gap-2">
<Button
variant="outline"
size="sm"
className="w-full justify-start text-sm font-medium hover:bg-neutral-100 dark:hover:bg-neutral-800 transition-colors"
onClick={(e) => {
e.preventDefault();
e.stopPropagation();
fetch(`/api/user/file/reindex`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({ file_id: file.id }),
})
.then((response) => {
if (!response.ok) {
throw new Error("Failed to reindex file");
}
setIndexingStatus(false); // Set to false to show indexing status
refreshFolders(); // Refresh the folder list
})
.catch((error) => {
console.error("Error reindexing file:", error);
});
}}
>
<FiRefreshCw className="mr-2 h-3.5 w-3.5" />
Reindex
</Button>
<Button
variant="outline"
size="sm"
className="w-full justify-start text-sm font-medium text-red-500 hover:bg-red-50 dark:hover:bg-red-900/20 hover:text-red-600 transition-colors"
onClick={(e) => {
e.stopPropagation();
handleDelete();
}}
>
<FiTrash2 className="mr-2 h-3.5 w-3.5" />
Delete
</Button>
</div>
</div>
</PopoverContent>
</Popover>
);
};
return (
<div
@ -97,7 +174,11 @@ export const FileListItem: React.FC<FileListItemProps> = ({
{isSelected !== undefined && (
<Checkbox checked={isSelected} className="mr-2 shrink-0" />
)}
{getFileIconFromFileName(file.name)}
{file.failed ? (
<FailureWithPopover />
) : (
getFileIconFromFileNameAndLink(file.name, file.link_url)
)}
{file.name.length > 50 ? (
<TooltipProvider>
<Tooltip>

18
web/src/app/chat/my-documents/components/FilePicker.tsx
View File

@ -53,7 +53,7 @@ import { getFormattedDateTime } from "@/lib/dateUtils";
import { FileUploadSection } from "../[id]/components/upload/FileUploadSection";
import { truncateString } from "@/lib/utils";
import { MinimalOnyxDocument } from "@/lib/search/interfaces";
import { getFileIconFromFileName } from "@/lib/assistantIconUtils";
import { getFileIconFromFileNameAndLink } from "@/lib/assistantIconUtils";
import { TokenDisplay } from "@/components/TokenDisplay";
// Define a type for uploading files that includes progress
@ -114,8 +114,6 @@ const DraggableItem: React.FC<{
{...listeners}
className="flex group w-full items-center"
>
{/* // className="flex group items-center" */}
<div className="w-6 flex items-center justify-center shrink-0">
<div
className="opacity-0 group-hover:opacity-100 transition-opacity duration-150"
@ -157,7 +155,7 @@ const DraggableItem: React.FC<{
>
<div className="flex items-center flex-1 min-w-0" onClick={onClick}>
<div className="flex text-sm items-center gap-2 w-[65%] min-w-0">
{getFileIconFromFileName(file.name)}
{getFileIconFromFileNameAndLink(file.name, file.link_url)}
{file.name.length > 34 ? (
<TooltipProvider>
<Tooltip>
@ -460,10 +458,14 @@ export const FilePickerModal: React.FC<FilePickerModalProps> = ({
}
};
const handleFileClick = (file: FileResponse) => {
setPresentingDocument({
document_id: file.document_id,
semantic_identifier: file.name,
});
if (file.link_url) {
window.open(file.link_url, "_blank");
} else {
setPresentingDocument({
document_id: file.document_id,
semantic_identifier: file.name,
});
}
};
const handleFileSelect = (

17
web/src/app/chat/my-documents/components/SelectedItemsList.tsx
View File

@ -11,7 +11,7 @@ import {
useDocumentsContext,
} from "../DocumentsContext";
import { useDocumentSelection } from "../../useDocumentSelection";
import { getFileIconFromFileName } from "@/lib/assistantIconUtils";
import { getFileIconFromFileNameAndLink } from "@/lib/assistantIconUtils";
import { MinimalOnyxDocument } from "@/lib/search/interfaces";
import { UploadingFile } from "./FilePicker";
import { CircularProgress } from "../[id]/components/upload/CircularProgress";
@ -35,10 +35,14 @@ export const SelectedItemsList: React.FC<SelectedItemsListProps> = ({
}) => {
const hasItems = folders.length > 0 || files.length > 0;
const openFile = (file: FileResponse) => {
setPresentingDocument({
semantic_identifier: file.name,
document_id: file.document_id,
});
if (file.link_url) {
window.open(file.link_url, "_blank");
} else {
setPresentingDocument({
semantic_identifier: file.name,
document_id: file.document_id,
});
}
};
return (
@ -116,13 +120,12 @@ export const SelectedItemsList: React.FC<SelectedItemsListProps> = ({
onClick={() => openFile(file)}
>
<div className="flex items-center min-w-0 flex-1">
{getFileIconFromFileName(file.name)}
{getFileIconFromFileNameAndLink(file.name, file.link_url)}
<span className="text-sm truncate text-neutral-700 dark:text-neutral-200 ml-2.5">
{truncateString(file.name, 34)}
</span>
</div>
</div>
<Button
variant="ghost"
size="sm"

38
web/src/app/chat/my-documents/components/SharedFolderItem.tsx
View File

@ -16,6 +16,8 @@ import {
import { FiArrowDown, FiEdit, FiTrash } from "react-icons/fi";
import { DeleteEntityModal } from "@/components/DeleteEntityModal";
import { useDocumentsContext } from "../DocumentsContext";
import { TruncatedText } from "@/components/ui/truncatedText";
import { truncateString } from "@/lib/utils";
interface SharedFolderItemProps {
folder: {
@ -60,21 +62,27 @@ export const SharedFolderItem: React.FC<SharedFolderItemProps> = ({
<div className="flex items-center flex-1 min-w-0">
<div className="flex items-center gap-3 w-[40%]">
<FolderIcon className="h-5 w-5 text-black dark:text-black shrink-0 fill-black dark:fill-black" />
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<span className="truncate text-text-dark dark:text-text-dark">
{folder.name}
</span>
</TooltipTrigger>
<TooltipContent side="bottom">
<p>{folder.name}</p>
{description && (
<p className="text-xs text-neutral-500">{description}</p>
)}
</TooltipContent>
</Tooltip>
</TooltipProvider>
{folder.name.length > 50 ? (
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<span className="truncate text-text-dark dark:text-text-dark">
{truncateString(folder.name, 60)}
</span>
</TooltipTrigger>
<TooltipContent side="bottom">
<p>{folder.name}</p>
{description && (
<p className="text-xs text-neutral-500">{description}</p>
)}
</TooltipContent>
</Tooltip>
</TooltipProvider>
) : (
<span className="truncate text-text-dark dark:text-text-dark">
{folder.name}
</span>
)}
</div>
<div className="w-[30%] text-sm text-text-400 dark:text-neutral-400">

1
web/src/app/chat/my-documents/components/types.ts
View File

@ -12,6 +12,7 @@ export interface UserFile {
name: string;
parent_folder_id: number | null;
token_count: number | null;
link_url: string | null;
}
export interface FolderNode extends UserFolder {

51
web/src/components/chat/sources/SourceCard.tsx
View File

@ -8,7 +8,8 @@ import React, { useEffect, useState } from "react";
import { SearchResultIcon } from "@/components/SearchResultIcon";
import { FileDescriptor } from "@/app/chat/interfaces";
import { FiFileText } from "react-icons/fi";
import { getFileIconFromFileName } from "@/lib/assistantIconUtils";
import { getFileIconFromFileNameAndLink } from "@/lib/assistantIconUtils";
import { FileResponse } from "@/app/chat/my-documents/DocumentsContext";
export const ResultIcon = ({
doc,
@ -140,48 +141,34 @@ export function SeeMoreBlock({
);
}
export function getUniqueFileIcons(files: FileDescriptor[]): JSX.Element[] {
export function getUniqueFileIcons(files: FileResponse[]): JSX.Element[] {
const uniqueIcons: JSX.Element[] = [];
const seenExtensions = new Set<string>();
// Helper function to extract filename from id when name is missing
const getFileName = (file: FileDescriptor): string => {
if (file.name) return file.name;
// Extract filename from the id if possible
if (file.id) {
const idParts = file.id.split("/");
if (idParts.length > 1) {
return idParts[idParts.length - 1]; // Get the last part after the last slash
}
}
return "unknown.txt"; // Fallback filename
};
// Helper function to get a styled icon
const getStyledIcon = (fileName: string, fileId: string) => {
return React.cloneElement(getFileIconFromFileName(fileName), {
key: `file-${fileId}`,
});
const getStyledIcon = (
fileName: string,
fileId: number,
link_url?: string | null
) => {
return React.cloneElement(
getFileIconFromFileNameAndLink(fileName, link_url),
{
key: `file-${fileId}`,
}
);
};
for (const file of files) {
const fileName = getFileName(file);
const extension = fileName.split(".").pop()?.toLowerCase() || "";
if (!seenExtensions.has(extension)) {
seenExtensions.add(extension);
uniqueIcons.push(getStyledIcon(fileName, file.id));
}
uniqueIcons.push(getStyledIcon(file.name, file.id, file.link_url));
}
// If we have zero icons, use a fallback
if (uniqueIcons.length === 0) {
return [
getFileIconFromFileName("fallback1.txt"),
getFileIconFromFileName("fallback2.txt"),
getFileIconFromFileName("fallback3.txt"),
getFileIconFromFileNameAndLink("fallback1.txt"),
getFileIconFromFileNameAndLink("fallback2.txt"),
getFileIconFromFileNameAndLink("fallback3.txt"),
];
}
@ -208,7 +195,7 @@ export function FilesSeeMoreBlock({
fullWidth = false,
}: {
toggleDocumentSelection: () => void;
files: FileDescriptor[];
files: FileResponse[];
toggled: boolean;
fullWidth?: boolean;
}) {

9
web/src/lib/assistantIconUtils.tsx
View File

@ -9,6 +9,7 @@ import {
ImagesIcon,
XMLIcon,
} from "@/components/icons/icons";
import { SearchResultIcon } from "@/components/SearchResultIcon";
export interface GridShape {
encodedGrid: number;
@ -180,7 +181,13 @@ export const constructMiniFiedPersona = (
};
};
export const getFileIconFromFileName = (fileName: string) => {
export const getFileIconFromFileNameAndLink = (
fileName: string,
linkUrl?: string | null
) => {
if (linkUrl) {
return <SearchResultIcon url={linkUrl} />;
}
const extension = fileName.split(".").pop()?.toLowerCase();
if (extension === "pdf") {
return <PDFIcon className="h-4 w-4 shrink-0" />;