highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-06-15 02:20:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

switched default pruning behavior and removed some logging (#1898)

Browse Source
This commit is contained in:
hagen-danswer 2024-07-22 10:36:26 -07:00 committed by GitHub
parent 48a7fe80b1
commit 17005fb705
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 4 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/danswer/background/celery/celery_utils.py
View File

@ -6,8 +6,8 @@ from sqlalchemy.orm import Session
from danswer.background.task_utils import name_cc_cleanup_task from danswer.background.task_utils import name_cc_cleanup_task
from danswer.background.task_utils import name_cc_prune_task from danswer.background.task_utils import name_cc_prune_task
from danswer.background.task_utils import name_document_set_sync_task from danswer.background.task_utils import name_document_set_sync_task
from danswer.configs.app_configs import ALLOW_SIMULTANEOUS_PRUNING
from danswer.configs.app_configs import MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE from danswer.configs.app_configs import MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE
from danswer.configs.app_configs import PREVENT_SIMULTANEOUS_PRUNING
from danswer.connectors.cross_connector_utils.rate_limit_wrapper import ( from danswer.connectors.cross_connector_utils.rate_limit_wrapper import (
rate_limit_builder, rate_limit_builder,
) )
@ -80,7 +80,7 @@ def should_prune_cc_pair(
return True return True
return False return False
if PREVENT_SIMULTANEOUS_PRUNING: if not ALLOW_SIMULTANEOUS_PRUNING:
pruning_type_task_name = name_cc_prune_task() pruning_type_task_name = name_cc_prune_task()
last_pruning_type_task = get_latest_task_by_type( last_pruning_type_task = get_latest_task_by_type(
pruning_type_task_name, db_session pruning_type_task_name, db_session
@ -89,11 +89,9 @@ def should_prune_cc_pair(
if last_pruning_type_task and check_task_is_live_and_not_timed_out( if last_pruning_type_task and check_task_is_live_and_not_timed_out(
last_pruning_type_task, db_session last_pruning_type_task, db_session
): ):
logger.info("Another Connector is already pruning. Skipping.")
return False return False
if check_task_is_live_and_not_timed_out(last_pruning_task, db_session): if check_task_is_live_and_not_timed_out(last_pruning_task, db_session):
logger.info(f"Connector '{connector.name}' is already pruning. Skipping.")
return False return False
if not last_pruning_task.start_time: if not last_pruning_task.start_time:

4
backend/danswer/configs/app_configs.py
View File

@ -214,8 +214,8 @@ EXPERIMENTAL_CHECKPOINTING_ENABLED = (
DEFAULT_PRUNING_FREQ = 60 * 60 * 24 # Once a day DEFAULT_PRUNING_FREQ = 60 * 60 * 24 # Once a day
PREVENT_SIMULTANEOUS_PRUNING = ( ALLOW_SIMULTANEOUS_PRUNING = (
os.environ.get("PREVENT_SIMULTANEOUS_PRUNING", "").lower() == "true" os.environ.get("ALLOW_SIMULTANEOUS_PRUNING", "").lower() == "true"
) )
# This is the maxiumum rate at which documents are queried for a pruning job. 0 disables the limitation. # This is the maxiumum rate at which documents are queried for a pruning job. 0 disables the limitation.