mirror of
https://github.com/danswer-ai/danswer.git
synced 2025-05-08 19:00:38 +02:00
bec0f9fb23
* try fixing exception in cloud * raise beat expiry ... 60 seconds might be starving certain tasks completely * adjust expiry down to 10 min * raise concurrency overflow for indexing worker. * parent pid check * fix comment * fix parent pid check, also actually raise an exception from the task if the spawned task exit status is bad * fix pid check * some cleanup and task wait fixes * review fixes * comment some code so we don't change too many things at once --------- Co-authored-by: Richard Kuo (Danswer) <rkuo@onyx.app> Co-authored-by: Richard Kuo <rkuo@rkuo.com>
93 lines
2.8 KiB
Python
93 lines
2.8 KiB
Python
from datetime import timedelta
|
|
from typing import Any
|
|
|
|
from onyx.configs.constants import OnyxCeleryPriority
|
|
from onyx.configs.constants import OnyxCeleryTask
|
|
|
|
# choosing 15 minutes because it roughly gives us enough time to process many tasks
|
|
# we might be able to reduce this greatly if we can run a unified
|
|
# loop across all tenants rather than tasks per tenant
|
|
|
|
BEAT_EXPIRES_DEFAULT = 15 * 60 # 15 minutes (in seconds)
|
|
|
|
# we set expires because it isn't necessary to queue up these tasks
|
|
# it's only important that they run relatively regularly
|
|
tasks_to_schedule = [
|
|
{
|
|
"name": "check-for-vespa-sync",
|
|
"task": OnyxCeleryTask.CHECK_FOR_VESPA_SYNC_TASK,
|
|
"schedule": timedelta(seconds=20),
|
|
"options": {
|
|
"priority": OnyxCeleryPriority.HIGH,
|
|
"expires": BEAT_EXPIRES_DEFAULT,
|
|
},
|
|
},
|
|
{
|
|
"name": "check-for-connector-deletion",
|
|
"task": OnyxCeleryTask.CHECK_FOR_CONNECTOR_DELETION,
|
|
"schedule": timedelta(seconds=20),
|
|
"options": {
|
|
"priority": OnyxCeleryPriority.HIGH,
|
|
"expires": BEAT_EXPIRES_DEFAULT,
|
|
},
|
|
},
|
|
{
|
|
"name": "check-for-indexing",
|
|
"task": OnyxCeleryTask.CHECK_FOR_INDEXING,
|
|
"schedule": timedelta(seconds=15),
|
|
"options": {
|
|
"priority": OnyxCeleryPriority.HIGH,
|
|
"expires": BEAT_EXPIRES_DEFAULT,
|
|
},
|
|
},
|
|
{
|
|
"name": "check-for-prune",
|
|
"task": OnyxCeleryTask.CHECK_FOR_PRUNING,
|
|
"schedule": timedelta(seconds=15),
|
|
"options": {
|
|
"priority": OnyxCeleryPriority.HIGH,
|
|
"expires": BEAT_EXPIRES_DEFAULT,
|
|
},
|
|
},
|
|
{
|
|
"name": "kombu-message-cleanup",
|
|
"task": OnyxCeleryTask.KOMBU_MESSAGE_CLEANUP_TASK,
|
|
"schedule": timedelta(seconds=3600),
|
|
"options": {
|
|
"priority": OnyxCeleryPriority.LOWEST,
|
|
"expires": BEAT_EXPIRES_DEFAULT,
|
|
},
|
|
},
|
|
{
|
|
"name": "monitor-vespa-sync",
|
|
"task": OnyxCeleryTask.MONITOR_VESPA_SYNC,
|
|
"schedule": timedelta(seconds=5),
|
|
"options": {
|
|
"priority": OnyxCeleryPriority.HIGH,
|
|
"expires": BEAT_EXPIRES_DEFAULT,
|
|
},
|
|
},
|
|
{
|
|
"name": "check-for-doc-permissions-sync",
|
|
"task": OnyxCeleryTask.CHECK_FOR_DOC_PERMISSIONS_SYNC,
|
|
"schedule": timedelta(seconds=30),
|
|
"options": {
|
|
"priority": OnyxCeleryPriority.HIGH,
|
|
"expires": BEAT_EXPIRES_DEFAULT,
|
|
},
|
|
},
|
|
{
|
|
"name": "check-for-external-group-sync",
|
|
"task": OnyxCeleryTask.CHECK_FOR_EXTERNAL_GROUP_SYNC,
|
|
"schedule": timedelta(seconds=20),
|
|
"options": {
|
|
"priority": OnyxCeleryPriority.HIGH,
|
|
"expires": BEAT_EXPIRES_DEFAULT,
|
|
},
|
|
},
|
|
]
|
|
|
|
|
|
def get_tasks_to_schedule() -> list[dict[str, Any]]:
|
|
return tasks_to_schedule
|