highperfocused/open-webui
1
0
Fork 0
mirror of https://github.com/open-webui/open-webui.git synced 2025-10-04 19:02:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

refac

Browse Source
This commit is contained in:
Timothy J. Baek
2024-10-25 22:23:21 -07:00
parent 925a903e38
commit f2c78ac0fb
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/open_webui/apps/retrieval/main.py
View File

@@ -667,7 +667,11 @@ def save_docs_to_vector_db(
add_start_index=True, add_start_index=True,
) )
elif app.state.config.TEXT_SPLITTER == "token": elif app.state.config.TEXT_SPLITTER == "token":
tiktoken.get_encoding(app.state.config.TIKTOKEN_ENCODING_NAME) log.info(
f"Using token text splitter: {app.state.config.TIKTOKEN_ENCODING_NAME}"
)
tiktoken.get_encoding(str(app.state.config.TIKTOKEN_ENCODING_NAME))
text_splitter = TokenTextSplitter( text_splitter = TokenTextSplitter(
encoding_name=str(app.state.config.TIKTOKEN_ENCODING_NAME), encoding_name=str(app.state.config.TIKTOKEN_ENCODING_NAME),
chunk_size=app.state.config.CHUNK_SIZE, chunk_size=app.state.config.CHUNK_SIZE,