highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-04-08 11:58:34 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix rebase issue

Browse Source
This commit is contained in:
Evan Lohn 2025-03-13 13:42:06 -07:00
parent 1ed3ac7802
commit d8abb26d03
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/onyx/connectors/models.py
View File

@ -164,8 +164,8 @@ class DocumentBase(BaseModel):
attributes.append(k + INDEX_SEPARATOR + v)
return attributes
def get_content(self) -> str:
return " ".join([section.text for section in self.sections])
def get_text_content(self) -> str:
return " ".join([section.text for section in self.sections if section.text])
class Document(DocumentBase):

2
backend/onyx/indexing/chunker.py
View File

@ -477,7 +477,7 @@ class Chunker:
single_chunk_fits = True
doc_token_count = 0
if self.enable_contextual_rag:
doc_content = document.get_content()
doc_content = document.get_text_content()
tokenized_doc = self.tokenizer.tokenize(doc_content)
doc_token_count = len(tokenized_doc)