highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-09-22 17:16:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix icons + add retries for indexing

Browse Source
This commit is contained in:
Weves
2023-05-05 14:14:36 -07:00
committed by Chris Weaver
parent 63f93594a3
commit 4f4c65acac
2 changed files with 21 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
backend/danswer/datastores/qdrant/indexing.py
View File

@@ -13,6 +13,7 @@ from danswer.configs.model_configs import DOC_EMBEDDING_DIM
from danswer.utils.clients import get_qdrant_client from danswer.utils.clients import get_qdrant_client
from danswer.utils.logging import setup_logger from danswer.utils.logging import setup_logger
from qdrant_client import QdrantClient from qdrant_client import QdrantClient
from qdrant_client.http.exceptions import ResponseHandlingException
from qdrant_client.http.models.models import UpdateStatus from qdrant_client.http.models.models import UpdateStatus
from qdrant_client.models import Distance from qdrant_client.models import Distance
from qdrant_client.models import PointStruct from qdrant_client.models import PointStruct
@@ -42,7 +43,7 @@ def index_chunks(
chunks: list[EmbeddedIndexChunk], chunks: list[EmbeddedIndexChunk],
collection: str, collection: str,
client: QdrantClient | None = None, client: QdrantClient | None = None,
batch_upsert: bool = False, batch_upsert: bool = True,
) -> bool: ) -> bool:
if client is None: if client is None:
client = get_qdrant_client() client = get_qdrant_client()
@@ -57,7 +58,7 @@ def index_chunks(
DOCUMENT_ID: document.id, DOCUMENT_ID: document.id,
CHUNK_ID: chunk.chunk_id, CHUNK_ID: chunk.chunk_id,
CONTENT: chunk.content, CONTENT: chunk.content,
SOURCE_TYPE: str(document.source), SOURCE_TYPE: str(document.source.value),
SOURCE_LINKS: chunk.source_links, SOURCE_LINKS: chunk.source_links,
SECTION_CONTINUATION: chunk.section_continuation, SECTION_CONTINUATION: chunk.section_continuation,
ALLOWED_USERS: [], # TODO ALLOWED_USERS: [], # TODO
@@ -74,11 +75,22 @@ def index_chunks(
for x in range(0, len(point_structs), DEFAULT_BATCH_SIZE) for x in range(0, len(point_structs), DEFAULT_BATCH_SIZE)
] ]
for point_struct_batch in point_struct_batches: for point_struct_batch in point_struct_batches:
index_results = client.upsert(
collection_name=collection, points=point_struct_batch def upsert():
) for _ in range(5):
try:
index_results = client.upsert(
collection_name=collection, points=point_struct_batch
)
return index_results
except ResponseHandlingException as e:
logger.warning(
f"Failed to upsert batch into qdrant due to error: {e}"
)
index_results = upsert()
logger.info( logger.info(
f"Indexing {len(point_struct_batch)} chunks into collection '{collection}', " f"Indexed {len(point_struct_batch)} chunks into collection '{collection}', "
f"status: {index_results.status}" f"status: {index_results.status}"
) )
else: else:

6
web/src/components/SearchResultsDisplay.tsx
View File

@@ -14,11 +14,11 @@ const ICON_STYLE = "text-blue-600 my-auto mr-1 flex flex-shrink-0";
const getSourceIcon = (sourceType: string) => { const getSourceIcon = (sourceType: string) => {
switch (sourceType) { switch (sourceType) {
case "Web": case "web":
return <Globe size={ICON_SIZE} className={ICON_STYLE} />; return <Globe size={ICON_SIZE} className={ICON_STYLE} />;
case "Slack": case "slack":
return <SlackLogo size={ICON_SIZE} className={ICON_STYLE} />; return <SlackLogo size={ICON_SIZE} className={ICON_STYLE} />;
case "Google Drive": case "google_drive":
return <GoogleDriveLogo size={ICON_SIZE} className={ICON_STYLE} />; return <GoogleDriveLogo size={ICON_SIZE} className={ICON_STYLE} />;
default: default:
return null; return null;