highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-04-09 04:18:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Updated pruning defaults (#2186)

Browse Source 
* Updated pruning defaults

* changed minutes to days
This commit is contained in:
hagen-danswer 2024-08-20 16:29:19 -07:00 committed by GitHub
parent 1ddb4b2025
commit f77b1ebd87
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 57 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
backend/alembic/versions/4b08d97e175a_change_default_prune_freq.py Normal file
View File

@ -0,0 +1,34 @@
"""change default prune_freq
Revision ID: 4b08d97e175a
Revises: da4c21c69164
Create Date: 2024-08-20 15:28:52.993827
"""
from alembic import op
# revision identifiers, used by Alembic.
revision = "4b08d97e175a"
down_revision = "da4c21c69164"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.execute(
"""
UPDATE connector
SET prune_freq = 2592000
WHERE prune_freq = 86400
"""
)
def downgrade() -> None:
op.execute(
"""
UPDATE connector
SET prune_freq = 86400
WHERE prune_freq = 2592000
"""
)

6
backend/danswer/background/celery/celery_utils.py
View File

@ -33,7 +33,7 @@ from danswer.utils.logger import setup_logger
logger = setup_logger()
def get_deletion_status(
def _get_deletion_status(
connector_id: int, credential_id: int, db_session: Session
) -> TaskQueueState | None:
cleanup_task_name = name_cc_cleanup_task(
@ -45,7 +45,7 @@ def get_deletion_status(
def get_deletion_attempt_snapshot(
connector_id: int, credential_id: int, db_session: Session
) -> DeletionAttemptSnapshot | None:
deletion_task = get_deletion_status(connector_id, credential_id, db_session)
deletion_task = _get_deletion_status(connector_id, credential_id, db_session)
if not deletion_task:
return None
@ -65,7 +65,7 @@ def should_kick_off_deletion_of_cc_pair(
if check_deletion_attempt_is_allowed(cc_pair, db_session):
return False
deletion_task = get_deletion_status(
deletion_task = _get_deletion_status(
connector_id=cc_pair.connector_id,
credential_id=cc_pair.credential_id,
db_session=db_session,

11
web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx
View File

@ -47,11 +47,16 @@ export function AdvancedConfigDisplay({
refreshFreq: number | null;
indexingStart: Date | null;
}) {
const formatFrequency = (seconds: number | null): string => {
const formatRefreshFrequency = (seconds: number | null): string => {
if (seconds === null) return "-";
const minutes = Math.round(seconds / 60);
return `${minutes} minute${minutes !== 1 ? "s" : ""}`;
};
const formatPruneFrequency = (seconds: number | null): string => {
if (seconds === null) return "-";
const days = Math.round(seconds / (60 * 60 * 24));
return `${days} day${days !== 1 ? "s" : ""}`;
};
const formatDate = (date: Date | null): string => {
if (date === null) return "-";
@ -73,13 +78,13 @@ export function AdvancedConfigDisplay({
{pruneFreq && (
<ListItem key={0}>
<span>Pruning Frequency</span>
<span>{formatFrequency(pruneFreq)}</span>
<span>{formatPruneFrequency(pruneFreq)}</span>
</ListItem>
)}
{refreshFreq && (
<ListItem key={1}>
<span>Refresh Frequency</span>
<span>{formatFrequency(refreshFreq)}</span>
<span>{formatRefreshFrequency(refreshFreq)}</span>
</ListItem>
)}
{indexingStart && (

8
web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
View File

@ -88,8 +88,8 @@ export default function AddConnector({
// Default to 10 minutes unless otherwise specified
const defaultRefresh = configuration.overrideDefaultFreq || 10;
// default is 1 day (in minutes)
const defaultPrune = 24 * 60;
// Default is 30 days
const defaultPrune = 30;
const [refreshFreq, setRefreshFreq] = useState<number>(defaultRefresh || 0);
const [pruneFreq, setPruneFreq] = useState<number>(defaultPrune);
@ -140,7 +140,7 @@ export default function AddConnector({
const createConnector = async () => {
const AdvancedConfig: AdvancedConfig = {
pruneFreq: pruneFreq * 60,
pruneFreq: pruneFreq * 60 * 60 * 24,
indexingStart,
refreshFreq: refreshFreq * 60,
};
@ -187,7 +187,7 @@ export default function AddConnector({
name: name,
source: connector,
refresh_freq: refreshFreq * 60 || null,
prune_freq: pruneFreq * 60 || null,
prune_freq: pruneFreq * 60 * 60 * 24 || null,
indexing_start: indexingStart,
},
undefined,

13
web/src/app/admin/connectors/[connector]/pages/Advanced.tsx
View File

@ -51,8 +51,12 @@ const AdvancedFormPage = forwardRef<FormikProps<any>, AdvancedFormPageProps>(
<div key="prune_freq">
<EditingValue
showNever
description="Checking all documents against the source to see if any no longer exist. Documents are deleted based on this. Note: To do this, we must check every document with the source so careful turning up the frequency of this (in minutes). This defaults to 1 day. If you input 0, we will never prune this connector."
optional
description={`
Checks all documents against the source to delete those that no longer exist.
Note: This process checks every document, so be cautious when increasing frequency.
Default is 30 days.
Enter 0 to disable pruning for this connector.
`}
currentValue={values.pruneFreq}
onChangeNumber={(value: number) => {
setPruneFreq(value);
@ -60,7 +64,7 @@ const AdvancedFormPage = forwardRef<FormikProps<any>, AdvancedFormPageProps>(
}}
setFieldValue={setFieldValue}
type="number"
label="Prune Frequency"
label="Prune Frequency (days)"
name="pruneFreq"
/>
</div>
@ -68,7 +72,6 @@ const AdvancedFormPage = forwardRef<FormikProps<any>, AdvancedFormPageProps>(
<EditingValue
showNever
description="This is how frequently we pull new documents from the source (in minutes). If you input 0, we will never pull new documents for this connector."
optional
currentValue={values.refreshFreq}
onChangeNumber={(value: number) => {
setRefreshFreq(value);
@ -76,7 +79,7 @@ const AdvancedFormPage = forwardRef<FormikProps<any>, AdvancedFormPageProps>(
}}
setFieldValue={setFieldValue}
type="number"
label="Refresh Frequency"
label="Refresh Frequency (minutes)"
name="refreshFreq"
/>
</div>