highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-06-10 17:20:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add more logging on existing jobs

Browse Source
This commit is contained in:
Weves 2023-11-30 02:44:31 -08:00 committed by Chris Weaver
parent 7ea774f35b
commit e983aaeca7
1 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
backend/danswer/background/update.py
View File

@ -313,10 +313,14 @@ def update_loop(delay: int = 10, num_workers: int = NUM_INDEXING_WORKERS) -> Non
start = time.time() start = time.time()
start_time_utc = datetime.utcfromtimestamp(start).strftime("%Y-%m-%d %H:%M:%S") start_time_utc = datetime.utcfromtimestamp(start).strftime("%Y-%m-%d %H:%M:%S")
logger.info(f"Running update, current UTC time: {start_time_utc}") logger.info(f"Running update, current UTC time: {start_time_utc}")
logger.debug(
if existing_jobs:
# TODO: make this debug level once the "no jobs are being scheduled" issue is resolved
logger.info(
"Found existing indexing jobs: " "Found existing indexing jobs: "
f"{[(attempt_id, job.status) for attempt_id, job in existing_jobs.items()]}" f"{[(attempt_id, job.status) for attempt_id, job in existing_jobs.items()]}"
) )
try: try:
existing_jobs = cleanup_indexing_jobs(existing_jobs=existing_jobs) existing_jobs = cleanup_indexing_jobs(existing_jobs=existing_jobs)
create_indexing_jobs(existing_jobs=existing_jobs) create_indexing_jobs(existing_jobs=existing_jobs)
@ -330,7 +334,7 @@ def update_loop(delay: int = 10, num_workers: int = NUM_INDEXING_WORKERS) -> Non
time.sleep(sleep_time) time.sleep(sleep_time)
if __name__ == "__main__": def update__main() -> None:
# needed for CUDA to work with multiprocessing # needed for CUDA to work with multiprocessing
# NOTE: needs to be done on application startup # NOTE: needs to be done on application startup
# before any other torch code has been run # before any other torch code has been run
@ -342,3 +346,7 @@ if __name__ == "__main__":
warm_up_models(indexer_only=True, skip_cross_encoders=True) warm_up_models(indexer_only=True, skip_cross_encoders=True)
logger.info("Starting Indexing Loop") logger.info("Starting Indexing Loop")
update_loop() update_loop()
if __name__ == "__main__":
update__main()