Allow config of background concurrency (#2648)

* Allow config of background concurrency

* Add comment

* Fix light worker

* use backslashes to continue lines in supervisord with bash

---------

Co-authored-by: Richard Kuo (Danswer) <rkuo@danswer.ai>

backend/supervisord.conf

@@ -39,15 +39,18 @@ autorestart=true
 startsecs=10
 stopasgroup=true
 
+# NOTE: only allowing configuration here and not in the other celery workers, 
+# since this is often the bottleneck for "sync" jobs (e.g. document set syncing, 
+# user group syncing, deletion, etc.) 
 [program:celery_worker_light]
-command=celery -A danswer.background.celery.celery_run:celery_app worker 
+command=bash -c "celery -A danswer.background.celery.celery_run:celery_app worker \
-    --pool=threads
+    --pool=threads \
-    --concurrency=16
+    --concurrency=${CELERY_WORKER_LIGHT_CONCURRENCY:-24} \
-    --prefetch-multiplier=8 
+    --prefetch-multiplier=${CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER:-8} \
-    --loglevel=INFO 
+    --loglevel=INFO \
-    --logfile=/var/log/celery_worker_light_supervisor.log 
+    --logfile=/var/log/celery_worker_light_supervisor.log \
-    --hostname=light@%%n 
+    --hostname=light@%%n \
-    -Q vespa_metadata_sync,connector_deletion
+    -Q vespa_metadata_sync,connector_deletion"
 environment=LOG_FILE_NAME=celery_worker_light
 redirect_stderr=true
 autorestart=true

deployment/docker_compose/docker-compose.dev.yml

@@ -169,6 +169,10 @@ services:
       - GONG_CONNECTOR_START_TIME=${GONG_CONNECTOR_START_TIME:-}
       - NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP=${NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP:-}
       - GITHUB_CONNECTOR_BASE_URL=${GITHUB_CONNECTOR_BASE_URL:-}
+      # Celery Configs (defaults are set in the supervisord.conf file, prefer doing that to have on source
+      # of defaults)
+      - CELERY_WORKER_LIGHT_CONCURRENCY=${CELERY_WORKER_LIGHT_CONCURRENCY:-}
+      - CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER=${CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER:-}
 
       # Danswer SlackBot Configs
       - DANSWER_BOT_SLACK_APP_TOKEN=${DANSWER_BOT_SLACK_APP_TOKEN:-}
@@ -287,7 +291,7 @@ services:
 
   relational_db:
     image: postgres:15.2-alpine
-    command: -c 'max_connections=150'
+    command: -c 'max_connections=250'
     restart: always
     environment:
       - POSTGRES_USER=${POSTGRES_USER:-postgres}

deployment/docker_compose/docker-compose.gpu-dev.yml

@@ -182,6 +182,10 @@ services:
       # Log all of Danswer prompts and interactions with the LLM
       - LOG_DANSWER_MODEL_INTERACTIONS=${LOG_DANSWER_MODEL_INTERACTIONS:-}
       - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-}
+      # Celery Configs (defaults are set in the supervisord.conf file, prefer doing that to have on source
+      # of defaults)
+      - CELERY_WORKER_LIGHT_CONCURRENCY=${CELERY_WORKER_LIGHT_CONCURRENCY:-}
+      - CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER=${CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER:-}
 
       # Enterprise Edition only
       - API_KEY_HASH_ROUNDS=${API_KEY_HASH_ROUNDS:-}
@@ -297,7 +301,7 @@ services:
 
   relational_db:
     image: postgres:15.2-alpine
-    command: -c 'max_connections=150'
+    command: -c 'max_connections=250'
     restart: always
     environment:
       - POSTGRES_USER=${POSTGRES_USER:-postgres}

deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml

@@ -147,7 +147,7 @@ services:
 
   relational_db:
     image: postgres:15.2-alpine
-    command: -c 'max_connections=150'
+    command: -c 'max_connections=250'
     restart: always
     # POSTGRES_USER and POSTGRES_PASSWORD should be set in .env file
     env_file:

deployment/docker_compose/docker-compose.prod.yml

@@ -89,7 +89,7 @@ services:
 
   relational_db:
     image: postgres:15.2-alpine
-    command: -c 'max_connections=150'
+    command: -c 'max_connections=250'
     restart: always
     # POSTGRES_USER and POSTGRES_PASSWORD should be set in .env file
     env_file:

deployment/docker_compose/docker-compose.search-testing.yml

@@ -148,7 +148,7 @@ services:
 
   relational_db:
     image: postgres:15.2-alpine
-    command: -c 'max_connections=150'
+    command: -c 'max_connections=250'
     restart: always
     environment:
       - POSTGRES_USER=${POSTGRES_USER:-postgres}

deployment/kubernetes/postgres-service-deployment.yaml

@@ -40,7 +40,7 @@ spec:
             secretKeyRef:
               name: danswer-secrets
               key: postgres_password
-        args: ["-c", "max_connections=150"]
+        args: ["-c", "max_connections=250"]
         ports:
         - containerPort: 5432
         volumeMounts: