mirror of
https://github.com/danswer-ai/danswer.git
synced 2025-03-17 21:32:36 +01:00
463340b8a1
* remove title for slack * initial working code * simplification * improvements * name change to information_content_model * avoid boost_score > 1.0 * nit * EL comments and improvements Improvements: - proper import of information content model from cache or HF - warm up for information content model Other: - EL PR review comments * nit * requirements version update * fixed docker file * new home for model_server configs * default off * small updates * YS comments - pt 1 * renaming to chunk_boost & chunk table def * saving and deleting chunk stats in new table * saving and updating chunk stats * improved dict score update * create columns for individual boost factors * RK comments * Update migration * manual import reordering
38 lines
1.1 KiB
Python
38 lines
1.1 KiB
Python
from shared_configs.enums import EmbeddingProvider
|
|
from shared_configs.enums import EmbedTextType
|
|
|
|
|
|
MODEL_WARM_UP_STRING = "hi " * 512
|
|
INFORMATION_CONTENT_MODEL_WARM_UP_STRING = "hi " * 16
|
|
DEFAULT_OPENAI_MODEL = "text-embedding-3-small"
|
|
DEFAULT_COHERE_MODEL = "embed-english-light-v3.0"
|
|
DEFAULT_VOYAGE_MODEL = "voyage-large-2-instruct"
|
|
DEFAULT_VERTEX_MODEL = "text-embedding-005"
|
|
|
|
|
|
class EmbeddingModelTextType:
|
|
PROVIDER_TEXT_TYPE_MAP = {
|
|
EmbeddingProvider.COHERE: {
|
|
EmbedTextType.QUERY: "search_query",
|
|
EmbedTextType.PASSAGE: "search_document",
|
|
},
|
|
EmbeddingProvider.VOYAGE: {
|
|
EmbedTextType.QUERY: "query",
|
|
EmbedTextType.PASSAGE: "document",
|
|
},
|
|
EmbeddingProvider.GOOGLE: {
|
|
EmbedTextType.QUERY: "RETRIEVAL_QUERY",
|
|
EmbedTextType.PASSAGE: "RETRIEVAL_DOCUMENT",
|
|
},
|
|
}
|
|
|
|
@staticmethod
|
|
def get_type(provider: EmbeddingProvider, text_type: EmbedTextType) -> str:
|
|
return EmbeddingModelTextType.PROVIDER_TEXT_TYPE_MAP[provider][text_type]
|
|
|
|
|
|
class GPUStatus:
|
|
CUDA = "cuda"
|
|
MAC_MPS = "mps"
|
|
NONE = "none"
|