diff --git a/backend/ee/onyx/background/celery/apps/primary.py b/backend/ee/onyx/background/celery/apps/primary.py index 79361efad4..4563ce49aa 100644 --- a/backend/ee/onyx/background/celery/apps/primary.py +++ b/backend/ee/onyx/background/celery/apps/primary.py @@ -32,6 +32,7 @@ def perform_ttl_management_task( @celery_app.task( name="check_ttl_management_task", + ignore_result=True, soft_time_limit=JOB_TIMEOUT, ) def check_ttl_management_task(*, tenant_id: str | None) -> None: @@ -56,6 +57,7 @@ def check_ttl_management_task(*, tenant_id: str | None) -> None: @celery_app.task( name="autogenerate_usage_report_task", + ignore_result=True, soft_time_limit=JOB_TIMEOUT, ) def autogenerate_usage_report_task(*, tenant_id: str | None) -> None: diff --git a/backend/onyx/background/celery/tasks/connector_deletion/tasks.py b/backend/onyx/background/celery/tasks/connector_deletion/tasks.py index 6f54ce50fe..2b8b3f1607 100644 --- a/backend/onyx/background/celery/tasks/connector_deletion/tasks.py +++ b/backend/onyx/background/celery/tasks/connector_deletion/tasks.py @@ -33,6 +33,7 @@ class TaskDependencyError(RuntimeError): @shared_task( name=OnyxCeleryTask.CHECK_FOR_CONNECTOR_DELETION, + ignore_result=True, soft_time_limit=JOB_TIMEOUT, trail=False, bind=True, diff --git a/backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py b/backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py index 5db7a41917..6ca9530942 100644 --- a/backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py +++ b/backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py @@ -91,6 +91,7 @@ def _is_external_doc_permissions_sync_due(cc_pair: ConnectorCredentialPair) -> b @shared_task( name=OnyxCeleryTask.CHECK_FOR_DOC_PERMISSIONS_SYNC, + ignore_result=True, soft_time_limit=JOB_TIMEOUT, bind=True, ) diff --git a/backend/onyx/background/celery/tasks/external_group_syncing/tasks.py b/backend/onyx/background/celery/tasks/external_group_syncing/tasks.py index 5fed241c42..ce85441466 100644 --- a/backend/onyx/background/celery/tasks/external_group_syncing/tasks.py +++ b/backend/onyx/background/celery/tasks/external_group_syncing/tasks.py @@ -91,6 +91,7 @@ def _is_external_group_sync_due(cc_pair: ConnectorCredentialPair) -> bool: @shared_task( name=OnyxCeleryTask.CHECK_FOR_EXTERNAL_GROUP_SYNC, + ignore_result=True, soft_time_limit=JOB_TIMEOUT, bind=True, ) diff --git a/backend/onyx/background/celery/tasks/indexing/utils.py b/backend/onyx/background/celery/tasks/indexing/utils.py index 8651139146..e14e79b5ff 100644 --- a/backend/onyx/background/celery/tasks/indexing/utils.py +++ b/backend/onyx/background/celery/tasks/indexing/utils.py @@ -295,11 +295,13 @@ def validate_indexing_fences( r_celery: Redis, lock_beat: RedisLock, ) -> None: + """Validates all indexing fences for this tenant ... aka makes sure + indexing tasks sent to celery are still in flight. + """ reserved_indexing_tasks = celery_get_unacked_task_ids( OnyxCeleryQueues.CONNECTOR_INDEXING, r_celery ) - # validate all existing indexing jobs # Use replica for this because the worst thing that happens # is that we don't run the validation on this pass for key_bytes in r_replica.scan_iter( diff --git a/backend/onyx/background/celery/tasks/llm_model_update/tasks.py b/backend/onyx/background/celery/tasks/llm_model_update/tasks.py index 7bd597661c..72eb22cfb8 100644 --- a/backend/onyx/background/celery/tasks/llm_model_update/tasks.py +++ b/backend/onyx/background/celery/tasks/llm_model_update/tasks.py @@ -54,6 +54,7 @@ def _process_model_list_response(model_list_json: Any) -> list[str]: @shared_task( name=OnyxCeleryTask.CHECK_FOR_LLM_MODEL_UPDATE, + ignore_result=True, soft_time_limit=JOB_TIMEOUT, trail=False, bind=True, diff --git a/backend/onyx/background/celery/tasks/monitoring/tasks.py b/backend/onyx/background/celery/tasks/monitoring/tasks.py index 64ac2e3985..80c29d3e25 100644 --- a/backend/onyx/background/celery/tasks/monitoring/tasks.py +++ b/backend/onyx/background/celery/tasks/monitoring/tasks.py @@ -563,6 +563,7 @@ def build_job_id( @shared_task( name=OnyxCeleryTask.MONITOR_BACKGROUND_PROCESSES, + ignore_result=True, soft_time_limit=_MONITORING_SOFT_TIME_LIMIT, time_limit=_MONITORING_TIME_LIMIT, queue=OnyxCeleryQueues.MONITORING, diff --git a/backend/onyx/background/celery/tasks/pruning/tasks.py b/backend/onyx/background/celery/tasks/pruning/tasks.py index 122dfd580b..3003af1960 100644 --- a/backend/onyx/background/celery/tasks/pruning/tasks.py +++ b/backend/onyx/background/celery/tasks/pruning/tasks.py @@ -78,6 +78,7 @@ def _is_pruning_due(cc_pair: ConnectorCredentialPair) -> bool: @shared_task( name=OnyxCeleryTask.CHECK_FOR_PRUNING, + ignore_result=True, soft_time_limit=JOB_TIMEOUT, bind=True, ) diff --git a/backend/onyx/background/celery/tasks/shared/tasks.py b/backend/onyx/background/celery/tasks/shared/tasks.py index 46082492b8..214987fa86 100644 --- a/backend/onyx/background/celery/tasks/shared/tasks.py +++ b/backend/onyx/background/celery/tasks/shared/tasks.py @@ -214,6 +214,7 @@ def document_by_cc_pair_cleanup_task( @shared_task( name=OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR, + ignore_result=True, trail=False, bind=True, ) diff --git a/backend/onyx/background/celery/tasks/vespa/tasks.py b/backend/onyx/background/celery/tasks/vespa/tasks.py index 58df78ee43..fa82d21fca 100644 --- a/backend/onyx/background/celery/tasks/vespa/tasks.py +++ b/backend/onyx/background/celery/tasks/vespa/tasks.py @@ -98,6 +98,7 @@ logger = setup_logger() # which bloats the result metadata considerably. trail=False prevents this. @shared_task( name=OnyxCeleryTask.CHECK_FOR_VESPA_SYNC_TASK, + ignore_result=True, soft_time_limit=JOB_TIMEOUT, trail=False, bind=True, @@ -872,7 +873,12 @@ def monitor_ccpair_indexing_taskset( redis_connector_index.reset() -@shared_task(name=OnyxCeleryTask.MONITOR_VESPA_SYNC, soft_time_limit=300, bind=True) +@shared_task( + name=OnyxCeleryTask.MONITOR_VESPA_SYNC, + ignore_result=True, + soft_time_limit=300, + bind=True, +) def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None: """This is a celery beat task that monitors and finalizes various long running tasks.