highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-06-19 12:30:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

try using spawn specifically

Browse Source
This commit is contained in:
Richard Kuo (Danswer) 2025-01-10 14:19:31 -08:00
parent 384a38418b
commit ccef350287
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/onyx/background/indexing/job_client.py
View File

@ -4,9 +4,10 @@ not follow the expected behavior, etc.
NOTE: cannot use Celery directly due to NOTE: cannot use Celery directly due to
https://github.com/celery/celery/issues/7007#issuecomment-1740139367""" https://github.com/celery/celery/issues/7007#issuecomment-1740139367"""
import multiprocessing as mp
from collections.abc import Callable from collections.abc import Callable
from dataclasses import dataclass from dataclasses import dataclass
from multiprocessing import Process from multiprocessing.context import SpawnProcess
from typing import Any from typing import Any
from typing import Literal from typing import Literal
from typing import Optional from typing import Optional
@ -63,7 +64,7 @@ class SimpleJob:
"""Drop in replacement for `dask.distributed.Future`""" """Drop in replacement for `dask.distributed.Future`"""
id: int id: int
process: Optional["Process"] = None process: Optional["SpawnProcess"] = None
def cancel(self) -> bool: def cancel(self) -> bool:
return self.release() return self.release()
@ -131,7 +132,8 @@ class SimpleJobClient:
job_id = self.job_id_counter job_id = self.job_id_counter
self.job_id_counter += 1 self.job_id_counter += 1
process = Process(target=_run_in_process, args=(func, args), daemon=True) ctx = mp.get_context("spawn")
process = ctx.Process(target=_run_in_process, args=(func, args), daemon=True)
job = SimpleJob(id=job_id, process=process) job = SimpleJob(id=job_id, process=process)
process.start() process.start()