highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-09-27 20:38:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Title Truncation Logic (#1828)

Browse Source
This commit is contained in:
Yuhong Sun
2024-07-14 13:54:36 -07:00
committed by GitHub
parent da31da33e7
commit f63d0ca3ad
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/danswer/indexing/chunker.py
View File

@@ -110,7 +110,7 @@ def chunk_document(
tokenizer = get_default_tokenizer() tokenizer = get_default_tokenizer()
title = document.get_title_for_document_index() title = document.get_title_for_document_index()
title_prefix = f"{title}{RETURN_SEPARATOR}"[:MAX_CHUNK_TITLE_LEN] if title else "" title_prefix = f"{title[:MAX_CHUNK_TITLE_LEN]}{RETURN_SEPARATOR}" if title else ""
title_tokens = len(tokenizer.tokenize(title_prefix)) title_tokens = len(tokenizer.tokenize(title_prefix))
metadata_suffix = "" metadata_suffix = ""