From f77b1ebd87c54bdb6047c6a2e27d5a7a239be10d Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Tue, 20 Aug 2024 16:29:19 -0700
Subject: [PATCH] Updated pruning defaults (#2186)

* Updated pruning defaults

* changed minutes to days
---
 .../4b08d97e175a_change_default_prune_freq.py | 34 +++++++++++++++++++
 .../danswer/background/celery/celery_utils.py |  6 ++--
 .../connector/[ccPairId]/ConfigDisplay.tsx    | 11 ++++--
 .../[connector]/AddConnectorPage.tsx          |  8 ++---
 .../connectors/[connector]/pages/Advanced.tsx | 13 ++++---
 5 files changed, 57 insertions(+), 15 deletions(-)
 create mode 100644 backend/alembic/versions/4b08d97e175a_change_default_prune_freq.py

diff --git a/backend/alembic/versions/4b08d97e175a_change_default_prune_freq.py b/backend/alembic/versions/4b08d97e175a_change_default_prune_freq.py
new file mode 100644
index 000000000..0e20b0720
--- /dev/null
+++ b/backend/alembic/versions/4b08d97e175a_change_default_prune_freq.py
@@ -0,0 +1,34 @@
+"""change default prune_freq
+
+Revision ID: 4b08d97e175a
+Revises: da4c21c69164
+Create Date: 2024-08-20 15:28:52.993827
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "4b08d97e175a"
+down_revision = "da4c21c69164"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        UPDATE connector
+        SET prune_freq = 2592000
+        WHERE prune_freq = 86400
+        """
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        """
+        UPDATE connector
+        SET prune_freq = 86400
+        WHERE prune_freq = 2592000
+        """
+    )
diff --git a/backend/danswer/background/celery/celery_utils.py b/backend/danswer/background/celery/celery_utils.py
index acc4ae8f6..e4d4d13bb 100644
--- a/backend/danswer/background/celery/celery_utils.py
+++ b/backend/danswer/background/celery/celery_utils.py
@@ -33,7 +33,7 @@ from danswer.utils.logger import setup_logger
 logger = setup_logger()
 
 
-def get_deletion_status(
+def _get_deletion_status(
     connector_id: int, credential_id: int, db_session: Session
 ) -> TaskQueueState | None:
     cleanup_task_name = name_cc_cleanup_task(
@@ -45,7 +45,7 @@ def get_deletion_status(
 def get_deletion_attempt_snapshot(
     connector_id: int, credential_id: int, db_session: Session
 ) -> DeletionAttemptSnapshot | None:
-    deletion_task = get_deletion_status(connector_id, credential_id, db_session)
+    deletion_task = _get_deletion_status(connector_id, credential_id, db_session)
     if not deletion_task:
         return None
 
@@ -65,7 +65,7 @@ def should_kick_off_deletion_of_cc_pair(
     if check_deletion_attempt_is_allowed(cc_pair, db_session):
         return False
 
-    deletion_task = get_deletion_status(
+    deletion_task = _get_deletion_status(
         connector_id=cc_pair.connector_id,
         credential_id=cc_pair.credential_id,
         db_session=db_session,
diff --git a/web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx b/web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx
index d7cdaef51..7bd429471 100644
--- a/web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx
@@ -47,11 +47,16 @@ export function AdvancedConfigDisplay({
   refreshFreq: number | null;
   indexingStart: Date | null;
 }) {
-  const formatFrequency = (seconds: number | null): string => {
+  const formatRefreshFrequency = (seconds: number | null): string => {
     if (seconds === null) return "-";
     const minutes = Math.round(seconds / 60);
     return `${minutes} minute${minutes !== 1 ? "s" : ""}`;
   };
+  const formatPruneFrequency = (seconds: number | null): string => {
+    if (seconds === null) return "-";
+    const days = Math.round(seconds / (60 * 60 * 24));
+    return `${days} day${days !== 1 ? "s" : ""}`;
+  };
 
   const formatDate = (date: Date | null): string => {
     if (date === null) return "-";
@@ -73,13 +78,13 @@ export function AdvancedConfigDisplay({
           {pruneFreq && (
             <ListItem key={0}>
               <span>Pruning Frequency</span>
-              <span>{formatFrequency(pruneFreq)}</span>
+              <span>{formatPruneFrequency(pruneFreq)}</span>
             </ListItem>
           )}
           {refreshFreq && (
             <ListItem key={1}>
               <span>Refresh Frequency</span>
-              <span>{formatFrequency(refreshFreq)}</span>
+              <span>{formatRefreshFrequency(refreshFreq)}</span>
             </ListItem>
           )}
           {indexingStart && (
diff --git a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
index bcf9fd051..d0bc07da1 100644
--- a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
@@ -88,8 +88,8 @@ export default function AddConnector({
 
   // Default to 10 minutes unless otherwise specified
   const defaultRefresh = configuration.overrideDefaultFreq || 10;
-  // default is 1 day (in minutes)
-  const defaultPrune = 24 * 60;
+  // Default is 30 days
+  const defaultPrune = 30;
 
   const [refreshFreq, setRefreshFreq] = useState<number>(defaultRefresh || 0);
   const [pruneFreq, setPruneFreq] = useState<number>(defaultPrune);
@@ -140,7 +140,7 @@ export default function AddConnector({
 
   const createConnector = async () => {
     const AdvancedConfig: AdvancedConfig = {
-      pruneFreq: pruneFreq * 60,
+      pruneFreq: pruneFreq * 60 * 60 * 24,
       indexingStart,
       refreshFreq: refreshFreq * 60,
     };
@@ -187,7 +187,7 @@ export default function AddConnector({
         name: name,
         source: connector,
         refresh_freq: refreshFreq * 60 || null,
-        prune_freq: pruneFreq * 60 || null,
+        prune_freq: pruneFreq * 60 * 60 * 24 || null,
         indexing_start: indexingStart,
       },
       undefined,
diff --git a/web/src/app/admin/connectors/[connector]/pages/Advanced.tsx b/web/src/app/admin/connectors/[connector]/pages/Advanced.tsx
index d3c40ad80..e4f802ea5 100644
--- a/web/src/app/admin/connectors/[connector]/pages/Advanced.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/Advanced.tsx
@@ -51,8 +51,12 @@ const AdvancedFormPage = forwardRef<FormikProps<any>, AdvancedFormPageProps>(
               <div key="prune_freq">
                 <EditingValue
                   showNever
-                  description="Checking all documents against the source to see if any no longer exist. Documents are deleted based on this. Note: To do this, we must check every document with the source so careful turning up the frequency of this (in minutes). This defaults to 1 day. If you input 0, we will never prune this connector."
-                  optional
+                  description={`
+                    Checks all documents against the source to delete those that no longer exist.
+                    Note: This process checks every document, so be cautious when increasing frequency.
+                    Default is 30 days.
+                    Enter 0 to disable pruning for this connector.
+                  `}
                   currentValue={values.pruneFreq}
                   onChangeNumber={(value: number) => {
                     setPruneFreq(value);
@@ -60,7 +64,7 @@ const AdvancedFormPage = forwardRef<FormikProps<any>, AdvancedFormPageProps>(
                   }}
                   setFieldValue={setFieldValue}
                   type="number"
-                  label="Prune Frequency"
+                  label="Prune Frequency (days)"
                   name="pruneFreq"
                 />
               </div>
@@ -68,7 +72,6 @@ const AdvancedFormPage = forwardRef<FormikProps<any>, AdvancedFormPageProps>(
                 <EditingValue
                   showNever
                   description="This is how frequently we pull new documents from the source (in minutes). If you input 0, we will never pull new documents for this connector."
-                  optional
                   currentValue={values.refreshFreq}
                   onChangeNumber={(value: number) => {
                     setRefreshFreq(value);
@@ -76,7 +79,7 @@ const AdvancedFormPage = forwardRef<FormikProps<any>, AdvancedFormPageProps>(
                   }}
                   setFieldValue={setFieldValue}
                   type="number"
-                  label="Refresh Frequency"
+                  label="Refresh Frequency (minutes)"
                   name="refreshFreq"
                 />
               </div>