quick hack to prevent resyncing the same doc

backend/onyx/background/celery/tasks/vespa/tasks.py

@@ -898,6 +898,9 @@ def vespa_metadata_sync_task(
             # the sync might repeat again later
             mark_document_as_synced(document_id, db_session)
 
+            r = get_redis_client(tenant_id=tenant_id)
+            r.hdel(RedisConnectorCredentialPair.SYNCING_HASH, document_id)
+
             task_logger.info(f"doc={document_id} action=sync chunks={chunks_affected}")
     except SoftTimeLimitExceeded:
         task_logger.info(f"SoftTimeLimitExceeded exception. doc={document_id}")

backend/onyx/redis/redis_connector_credential_pair.py

@@ -30,6 +30,8 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
     FENCE_PREFIX = PREFIX + "_fence"
     TASKSET_PREFIX = PREFIX + "_taskset"
 
+    SYNCING_HASH = PREFIX + ":vespa_syncing"
+
     def __init__(self, tenant_id: str | None, id: int) -> None:
         super().__init__(tenant_id, str(id))
 
@@ -64,6 +66,9 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
         lock: RedisLock,
         tenant_id: str | None,
     ) -> tuple[int, int] | None:
+        # an arbitrary number in seconds to prevent the same doc from syncing repeatedly
+        SYNC_EXPIRATION = 24 * 60 * 60
+
         last_lock_time = time.monotonic()
 
         async_results = []
@@ -92,6 +97,10 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
             if doc.id in self.skip_docs:
                 continue
 
+            # is the document sync already queued?
+            if redis_client.hexists(doc.id):
+                continue
+
             # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
             # the key for the result is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
             # we prefix the task id so it's easier to keep track of who created the task
@@ -104,6 +113,11 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
                 RedisConnectorCredentialPair.get_taskset_key(), custom_task_id
             )
 
+            # track the doc.id in redis so that we don't resubmit it repeatedly
+            redis_client.hset(
+                self.SYNCING_HASH, doc.id, custom_task_id, ex=SYNC_EXPIRATION
+            )
+
             # Priority on sync's triggered by new indexing should be medium
             result = celery_app.send_task(
                 OnyxCeleryTask.VESPA_METADATA_SYNC_TASK,

backend/onyx/redis/redis_pool.py

@@ -110,6 +110,9 @@ class TenantRedis(redis.Redis):
             "sadd",
             "srem",
             "scard",
+            "hexists",
+            "hset",
+            "hdel",
         ]  # Regular methods that need simple prefixing
 
         if item == "scan_iter":