diff --git a/backend/onyx/auth/email_utils.py b/backend/onyx/auth/email_utils.py index 239daab98..9c8fba65d 100644 --- a/backend/onyx/auth/email_utils.py +++ b/backend/onyx/auth/email_utils.py @@ -23,6 +23,7 @@ from onyx.utils.url import add_url_params from onyx.utils.variable_functionality import fetch_versioned_implementation from shared_configs.configs import MULTI_TENANT + HTML_EMAIL_TEMPLATE = """\ diff --git a/backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py b/backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py index 43885db2c..5f3e68e6e 100644 --- a/backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py +++ b/backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py @@ -886,11 +886,8 @@ def monitor_ccpair_permissions_taskset( record_type=RecordType.PERMISSION_SYNC_PROGRESS, data={ "cc_pair_id": cc_pair_id, - "id": payload.id if payload else None, - "total_docs": initial if initial is not None else 0, - "remaining_docs": remaining, - "synced_docs": (initial - remaining) if initial is not None else 0, - "is_complete": remaining == 0, + "total_docs_synced": initial if initial is not None else 0, + "remaining_docs_to_sync": remaining, }, tenant_id=tenant_id, ) @@ -906,6 +903,13 @@ def monitor_ccpair_permissions_taskset( f"num_synced={initial}" ) + # Add telemetry for permission syncing complete + optional_telemetry( + record_type=RecordType.PERMISSION_SYNC_COMPLETE, + data={"cc_pair_id": cc_pair_id}, + tenant_id=tenant_id, + ) + update_sync_record_status( db_session=db_session, entity_id=cc_pair_id, diff --git a/backend/onyx/background/indexing/run_indexing.py b/backend/onyx/background/indexing/run_indexing.py index aab03640e..850e611c6 100644 --- a/backend/onyx/background/indexing/run_indexing.py +++ b/backend/onyx/background/indexing/run_indexing.py @@ -56,7 +56,6 @@ from onyx.indexing.indexing_pipeline import build_indexing_pipeline from onyx.natural_language_processing.search_nlp_models import ( InformationContentClassificationModel, ) -from onyx.redis.redis_connector import RedisConnector from onyx.utils.logger import setup_logger from onyx.utils.logger import TaskAttemptSingleton from onyx.utils.telemetry import create_milestone_and_report @@ -578,11 +577,8 @@ def _run_indexing( data={ "index_attempt_id": index_attempt_id, "cc_pair_id": ctx.cc_pair_id, - "connector_id": ctx.connector_id, - "credential_id": ctx.credential_id, - "total_docs_indexed": document_count, - "total_chunks": chunk_count, - "batch_num": batch_num, + "current_docs_indexed": document_count, + "current_chunks_indexed": chunk_count, "source": ctx.source.value, }, tenant_id=tenant_id, @@ -603,26 +599,15 @@ def _run_indexing( checkpoint=checkpoint, ) - # Add telemetry for completed indexing - redis_connector = RedisConnector(tenant_id, ctx.cc_pair_id) - redis_connector_index = redis_connector.new_index( - index_attempt_start.search_settings_id - ) - final_progress = redis_connector_index.get_progress() or 0 - optional_telemetry( record_type=RecordType.INDEXING_COMPLETE, data={ "index_attempt_id": index_attempt_id, "cc_pair_id": ctx.cc_pair_id, - "connector_id": ctx.connector_id, - "credential_id": ctx.credential_id, "total_docs_indexed": document_count, "total_chunks": chunk_count, - "batch_count": batch_num, "time_elapsed_seconds": time.monotonic() - start_time, "source": ctx.source.value, - "redis_progress": final_progress, }, tenant_id=tenant_id, ) diff --git a/backend/onyx/db/index_attempt.py b/backend/onyx/db/index_attempt.py index 98e2e46b4..6034343cf 100644 --- a/backend/onyx/db/index_attempt.py +++ b/backend/onyx/db/index_attempt.py @@ -217,7 +217,6 @@ def mark_attempt_in_progress( "index_attempt_id": index_attempt.id, "status": IndexingStatus.IN_PROGRESS.value, "cc_pair_id": index_attempt.connector_credential_pair_id, - "search_settings_id": index_attempt.search_settings_id, }, ) except Exception: @@ -246,9 +245,6 @@ def mark_attempt_succeeded( "index_attempt_id": index_attempt_id, "status": IndexingStatus.SUCCESS.value, "cc_pair_id": attempt.connector_credential_pair_id, - "search_settings_id": attempt.search_settings_id, - "total_docs_indexed": attempt.total_docs_indexed, - "new_docs_indexed": attempt.new_docs_indexed, }, ) except Exception: @@ -277,9 +273,6 @@ def mark_attempt_partially_succeeded( "index_attempt_id": index_attempt_id, "status": IndexingStatus.COMPLETED_WITH_ERRORS.value, "cc_pair_id": attempt.connector_credential_pair_id, - "search_settings_id": attempt.search_settings_id, - "total_docs_indexed": attempt.total_docs_indexed, - "new_docs_indexed": attempt.new_docs_indexed, }, ) except Exception: @@ -312,10 +305,6 @@ def mark_attempt_canceled( "index_attempt_id": index_attempt_id, "status": IndexingStatus.CANCELED.value, "cc_pair_id": attempt.connector_credential_pair_id, - "search_settings_id": attempt.search_settings_id, - "reason": reason, - "total_docs_indexed": attempt.total_docs_indexed, - "new_docs_indexed": attempt.new_docs_indexed, }, ) except Exception: @@ -350,10 +339,6 @@ def mark_attempt_failed( "index_attempt_id": index_attempt_id, "status": IndexingStatus.FAILED.value, "cc_pair_id": attempt.connector_credential_pair_id, - "search_settings_id": attempt.search_settings_id, - "reason": failure_reason, - "total_docs_indexed": attempt.total_docs_indexed, - "new_docs_indexed": attempt.new_docs_indexed, }, ) except Exception: diff --git a/backend/onyx/utils/telemetry.py b/backend/onyx/utils/telemetry.py index 7e5e7f575..656141f58 100644 --- a/backend/onyx/utils/telemetry.py +++ b/backend/onyx/utils/telemetry.py @@ -39,6 +39,7 @@ class RecordType(str, Enum): INDEXING_PROGRESS = "indexing_progress" INDEXING_COMPLETE = "indexing_complete" PERMISSION_SYNC_PROGRESS = "permission_sync_progress" + PERMISSION_SYNC_COMPLETE = "permission_sync_complete" INDEX_ATTEMPT_STATUS = "index_attempt_status"