highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-07-12 14:12:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

Improve Search (#2105)

Browse Source
This commit is contained in:
pablodanswer
2024-08-16 21:29:15 -07:00
committed by GitHub
parent efae24acd0
commit 22573aba2a
70 changed files with 2670 additions and 1030 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/model_server/encoders.py
View File

@ -23,6 +23,7 @@ from model_server.constants import DEFAULT_VOYAGE_MODEL
from model_server.constants import EmbeddingModelTextType
from model_server.constants import EmbeddingProvider
from model_server.utils import simple_log_function_time
from shared_configs.configs import ALT_INDEX_SUFFIX
from shared_configs.configs import INDEXING_ONLY
from shared_configs.enums import EmbedTextType
from shared_configs.enums import RerankerProvider
@ -283,8 +284,11 @@ def embed_text(
elif model_name is not None:
prefixed_texts = [f"{prefix}{text}" for text in texts] if prefix else texts
# strip additional metadata from model name right before constructing from Huggingface
stripped_model_name = model_name.removesuffix(ALT_INDEX_SUFFIX)
local_model = get_embedding_model(
model_name=model_name, max_context_length=max_context_length
model_name=stripped_model_name, max_context_length=max_context_length
)
embeddings_vectors = local_model.encode(
prefixed_texts, normalize_embeddings=normalize_embeddings