highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-06-06 21:19:54 +02:00
Code Issues Packages Projects Releases Wiki Activity

process files with null title (#1989)

Browse Source
This commit is contained in:
pablodanswer 2024-07-31 08:18:50 -07:00 committed by GitHub
parent 036d5c737e
commit 89b6da36a6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/danswer/indexing/indexing_pipeline.py
View File

@ -129,10 +129,8 @@ def index_doc_batch(
# already be no documents that are empty. # already be no documents that are empty.
documents_to_process = [] documents_to_process = []
for document in documents: for document in documents:
if ( if (not document.title or not document.title.strip()) and not any(
not document.title section.text.strip() for section in document.sections
or not document.title.strip()
and not any(section.text.strip() for section in document.sections)
): ):
logger.warning( logger.warning(
f"Skipping document with ID {document.id} as it has neither title nor content" f"Skipping document with ID {document.id} as it has neither title nor content"