highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-09-27 20:38:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add illegal char stripping to title field

Browse Source
This commit is contained in:
Weves
2023-12-29 00:07:39 -08:00
committed by Chris Weaver
parent cdf260b277
commit b5947a1c74
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/danswer/document_index/vespa/index.py
View File

@@ -304,6 +304,9 @@ def _index_vespa_chunk(
vespa_document_fields[SEMANTIC_IDENTIFIER] = remove_invalid_unicode_chars( vespa_document_fields[SEMANTIC_IDENTIFIER] = remove_invalid_unicode_chars(
cast(str, vespa_document_fields[SEMANTIC_IDENTIFIER]) cast(str, vespa_document_fields[SEMANTIC_IDENTIFIER])
) )
vespa_document_fields[TITLE] = remove_invalid_unicode_chars(
cast(str, vespa_document_fields[TITLE])
)
vespa_document_fields[CONTENT] = remove_invalid_unicode_chars( vespa_document_fields[CONTENT] = remove_invalid_unicode_chars(
cast(str, vespa_document_fields[CONTENT]) cast(str, vespa_document_fields[CONTENT])
) )