highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-10-09 12:47:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

prevent trying to submit to jobclient when it can't take any more work (reduces log spam) (#2482)

Browse Source
This commit is contained in:
rkuo-danswer
2024-09-18 21:01:15 -07:00
committed by GitHub
parent bb279a8580
commit bc9d5fece7
1 changed files with 26 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
backend/danswer/background/update.py
View File

@@ -211,7 +211,6 @@ def cleanup_indexing_jobs(
timeout_hours: int = CLEANUP_INDEXING_JOBS_TIMEOUT, timeout_hours: int = CLEANUP_INDEXING_JOBS_TIMEOUT,
) -> dict[int, Future | SimpleJob]: ) -> dict[int, Future | SimpleJob]:
existing_jobs_copy = existing_jobs.copy() existing_jobs_copy = existing_jobs.copy()
# clean up completed jobs # clean up completed jobs
with Session(get_sqlalchemy_engine()) as db_session: with Session(get_sqlalchemy_engine()) as db_session:
for attempt_id, job in existing_jobs.items(): for attempt_id, job in existing_jobs.items():
@@ -312,7 +311,12 @@ def kickoff_indexing_jobs(
indexing_attempt_count = 0 indexing_attempt_count = 0
primary_client_full = False
secondary_client_full = False
for attempt, search_settings in new_indexing_attempts: for attempt, search_settings in new_indexing_attempts:
if primary_client_full and secondary_client_full:
break
use_secondary_index = ( use_secondary_index = (
search_settings.status == IndexModelStatus.FUTURE search_settings.status == IndexModelStatus.FUTURE
if search_settings is not None if search_settings is not None
@@ -337,22 +341,28 @@ def kickoff_indexing_jobs(
) )
continue continue
if use_secondary_index: if not use_secondary_index:
run = secondary_client.submit( if not primary_client_full:
run_indexing_entrypoint, run = client.submit(
attempt.id, run_indexing_entrypoint,
attempt.connector_credential_pair_id, attempt.id,
global_version.get_is_ee_version(), attempt.connector_credential_pair_id,
pure=False, global_version.get_is_ee_version(),
) pure=False,
)
if not run:
primary_client_full = True
else: else:
run = client.submit( if not secondary_client_full:
run_indexing_entrypoint, run = secondary_client.submit(
attempt.id, run_indexing_entrypoint,
attempt.connector_credential_pair_id, attempt.id,
global_version.get_is_ee_version(), attempt.connector_credential_pair_id,
pure=False, global_version.get_is_ee_version(),
) pure=False,
)
if not run:
secondary_client_full = True
if run: if run:
if indexing_attempt_count == 0: if indexing_attempt_count == 0: