highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-07-08 13:40:46 +02:00
Code Issues Packages Projects Releases Wiki Activity

autoscale with pool=thread crashes celery. remove and use concurrency… (#1929)

Browse Source 
* autoscale with pool=thread crashes celery. remove and use concurrency instead (to be improved later)

* update dev background script as well
This commit is contained in:
rkuo-danswer
2024-07-24 17:15:27 -07:00
committed by GitHub
parent 27824d6cc6
commit 546bfbd24b
2 changed files with 2 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/scripts/dev_run_background_jobs.py
View File

@ -24,9 +24,8 @@ def run_jobs(exclude_indexing: bool) -> None:
"ee.danswer.background.celery.celery_app", "ee.danswer.background.celery.celery_app",
"worker", "worker",
"--pool=threads", "--pool=threads",
"--autoscale=3,10", "--concurrency=16",
"--loglevel=INFO", "--loglevel=INFO",
"--concurrency=1",
] ]
cmd_beat = [ cmd_beat = [

2
backend/supervisord.conf
View File

@ -25,7 +25,7 @@ autorestart=true
# relatively compute-light (e.g. they tend to just make a bunch of requests to # relatively compute-light (e.g. they tend to just make a bunch of requests to
# Vespa / Postgres) # Vespa / Postgres)
[program:celery_worker] [program:celery_worker]
command=celery -A danswer.background.celery.celery_run:celery_app worker --pool=threads --autoscale=3,10 --loglevel=INFO --logfile=/var/log/celery_worker.log command=celery -A danswer.background.celery.celery_run:celery_app worker --pool=threads --concurrency=16 --loglevel=INFO --logfile=/var/log/celery_worker.log
stdout_logfile=/var/log/celery_worker_supervisor.log stdout_logfile=/var/log/celery_worker_supervisor.log
stdout_logfile_maxbytes=52428800 stdout_logfile_maxbytes=52428800
redirect_stderr=true redirect_stderr=true