highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-09-19 03:58:30 +02:00
Code Issues Packages Projects Releases Wiki Activity

more debugging of start method

Browse Source
This commit is contained in:
Richard Kuo (Danswer)
2025-01-10 12:16:13 -08:00
parent ac182c74b3
commit b6c2ecfecb
4 changed files with 25 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backend/onyx/background/celery/apps/heavy.py
View File

@@ -56,8 +56,14 @@ def on_celeryd_init(sender: Any = None, conf: Any = None, **kwargs: Any) -> None
@worker_init.connect
def on_worker_init(sender: Any, **kwargs: Any) -> None:
logger.info("worker_init signal received.")
all_start_methods: list[str] = multiprocessing.get_all_start_methods()
logger.info(f"Multiprocessing all start methods: {all_start_methods}")
multiprocessing.set_start_method("spawn") # fork is unsafe, set to spawn
logger.info(f"Multiprocessing start method: {multiprocessing.get_start_method()}")
logger.info(
f"Multiprocessing selected start method: {multiprocessing.get_start_method()}"
)
SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_HEAVY_APP_NAME)
SqlEngine.init_engine(pool_size=4, max_overflow=12)

4
backend/onyx/background/celery/apps/indexing.py
View File

@@ -62,7 +62,9 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
logger.info(f"Multiprocessing all start methods: {all_start_methods}")
multiprocessing.set_start_method("spawn") # fork is unsafe, set to spawn
logger.info(f"Multiprocessing start method: {multiprocessing.get_start_method()}")
logger.info(
f"Multiprocessing selected start method: {multiprocessing.get_start_method()}"
)
SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_INDEXING_APP_NAME)

9
backend/onyx/background/celery/apps/light.py
View File

@@ -56,8 +56,15 @@ def on_celeryd_init(sender: Any = None, conf: Any = None, **kwargs: Any) -> None
@worker_init.connect
def on_worker_init(sender: Any, **kwargs: Any) -> None:
logger.info("worker_init signal received.")
all_start_methods: list[str] = multiprocessing.get_all_start_methods()
logger.info(f"Multiprocessing all start methods: {all_start_methods}")
multiprocessing.set_start_method("spawn") # fork is unsafe, set to spawn
logger.info(f"Multiprocessing start method: {multiprocessing.get_start_method()}")
logger.info(
f"Multiprocessing selected start method: {multiprocessing.get_start_method()}"
)
logger.info(f"Concurrency: {sender.concurrency}")
SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_LIGHT_APP_NAME)

8
backend/onyx/background/celery/apps/primary.py
View File

@@ -80,8 +80,14 @@ def on_celeryd_init(sender: Any = None, conf: Any = None, **kwargs: Any) -> None
@worker_init.connect
def on_worker_init(sender: Any, **kwargs: Any) -> None:
logger.info("worker_init signal received.")
all_start_methods: list[str] = multiprocessing.get_all_start_methods()
logger.info(f"Multiprocessing all start methods: {all_start_methods}")
multiprocessing.set_start_method("spawn") # fork is unsafe, set to spawn
logger.info(f"Multiprocessing start method: {multiprocessing.get_start_method()}")
logger.info(
f"Multiprocessing selected start method: {multiprocessing.get_start_method()}"
)
SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME)
SqlEngine.init_engine(pool_size=8, max_overflow=0)