highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-06-24 23:11:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

Tracking things (#4352)

Browse Source
This commit is contained in:
pablonyx 2025-04-01 14:46:41 -07:00 committed by Weves
parent 2e524816a0
commit ee073f6d30
5 changed files with 13 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/onyx/auth/email_utils.py
View File

@ -23,6 +23,7 @@ from onyx.utils.url import add_url_params
from onyx.utils.variable_functionality import fetch_versioned_implementation from onyx.utils.variable_functionality import fetch_versioned_implementation
from shared_configs.configs import MULTI_TENANT from shared_configs.configs import MULTI_TENANT
HTML_EMAIL_TEMPLATE = """\ HTML_EMAIL_TEMPLATE = """\
<!DOCTYPE html> <!DOCTYPE html>
<html lang="en"> <html lang="en">

14
backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py
View File

@ -886,11 +886,8 @@ def monitor_ccpair_permissions_taskset(
record_type=RecordType.PERMISSION_SYNC_PROGRESS, record_type=RecordType.PERMISSION_SYNC_PROGRESS,
data={ data={
"cc_pair_id": cc_pair_id, "cc_pair_id": cc_pair_id,
"id": payload.id if payload else None, "total_docs_synced": initial if initial is not None else 0,
"total_docs": initial if initial is not None else 0, "remaining_docs_to_sync": remaining,
"remaining_docs": remaining,
"synced_docs": (initial - remaining) if initial is not None else 0,
"is_complete": remaining == 0,
}, },
tenant_id=tenant_id, tenant_id=tenant_id,
) )
@ -906,6 +903,13 @@ def monitor_ccpair_permissions_taskset(
f"num_synced={initial}" f"num_synced={initial}"
) )
# Add telemetry for permission syncing complete
optional_telemetry(
record_type=RecordType.PERMISSION_SYNC_COMPLETE,
data={"cc_pair_id": cc_pair_id},
tenant_id=tenant_id,
)
update_sync_record_status( update_sync_record_status(
db_session=db_session, db_session=db_session,
entity_id=cc_pair_id, entity_id=cc_pair_id,

19
backend/onyx/background/indexing/run_indexing.py
View File

@ -56,7 +56,6 @@ from onyx.indexing.indexing_pipeline import build_indexing_pipeline
from onyx.natural_language_processing.search_nlp_models import ( from onyx.natural_language_processing.search_nlp_models import (
InformationContentClassificationModel, InformationContentClassificationModel,
) )
from onyx.redis.redis_connector import RedisConnector
from onyx.utils.logger import setup_logger from onyx.utils.logger import setup_logger
from onyx.utils.logger import TaskAttemptSingleton from onyx.utils.logger import TaskAttemptSingleton
from onyx.utils.telemetry import create_milestone_and_report from onyx.utils.telemetry import create_milestone_and_report
@ -578,11 +577,8 @@ def _run_indexing(
data={ data={
"index_attempt_id": index_attempt_id, "index_attempt_id": index_attempt_id,
"cc_pair_id": ctx.cc_pair_id, "cc_pair_id": ctx.cc_pair_id,
"connector_id": ctx.connector_id, "current_docs_indexed": document_count,
"credential_id": ctx.credential_id, "current_chunks_indexed": chunk_count,
"total_docs_indexed": document_count,
"total_chunks": chunk_count,
"batch_num": batch_num,
"source": ctx.source.value, "source": ctx.source.value,
}, },
tenant_id=tenant_id, tenant_id=tenant_id,
@ -603,26 +599,15 @@ def _run_indexing(
checkpoint=checkpoint, checkpoint=checkpoint,
) )
# Add telemetry for completed indexing
redis_connector = RedisConnector(tenant_id, ctx.cc_pair_id)
redis_connector_index = redis_connector.new_index(
index_attempt_start.search_settings_id
)
final_progress = redis_connector_index.get_progress() or 0
optional_telemetry( optional_telemetry(
record_type=RecordType.INDEXING_COMPLETE, record_type=RecordType.INDEXING_COMPLETE,
data={ data={
"index_attempt_id": index_attempt_id, "index_attempt_id": index_attempt_id,
"cc_pair_id": ctx.cc_pair_id, "cc_pair_id": ctx.cc_pair_id,
"connector_id": ctx.connector_id,
"credential_id": ctx.credential_id,
"total_docs_indexed": document_count, "total_docs_indexed": document_count,
"total_chunks": chunk_count, "total_chunks": chunk_count,
"batch_count": batch_num,
"time_elapsed_seconds": time.monotonic() - start_time, "time_elapsed_seconds": time.monotonic() - start_time,
"source": ctx.source.value, "source": ctx.source.value,
"redis_progress": final_progress,
}, },
tenant_id=tenant_id, tenant_id=tenant_id,
) )

15
backend/onyx/db/index_attempt.py
View File

@ -217,7 +217,6 @@ def mark_attempt_in_progress(
"index_attempt_id": index_attempt.id, "index_attempt_id": index_attempt.id,
"status": IndexingStatus.IN_PROGRESS.value, "status": IndexingStatus.IN_PROGRESS.value,
"cc_pair_id": index_attempt.connector_credential_pair_id, "cc_pair_id": index_attempt.connector_credential_pair_id,
"search_settings_id": index_attempt.search_settings_id,
}, },
) )
except Exception: except Exception:
@ -246,9 +245,6 @@ def mark_attempt_succeeded(
"index_attempt_id": index_attempt_id, "index_attempt_id": index_attempt_id,
"status": IndexingStatus.SUCCESS.value, "status": IndexingStatus.SUCCESS.value,
"cc_pair_id": attempt.connector_credential_pair_id, "cc_pair_id": attempt.connector_credential_pair_id,
"search_settings_id": attempt.search_settings_id,
"total_docs_indexed": attempt.total_docs_indexed,
"new_docs_indexed": attempt.new_docs_indexed,
}, },
) )
except Exception: except Exception:
@ -277,9 +273,6 @@ def mark_attempt_partially_succeeded(
"index_attempt_id": index_attempt_id, "index_attempt_id": index_attempt_id,
"status": IndexingStatus.COMPLETED_WITH_ERRORS.value, "status": IndexingStatus.COMPLETED_WITH_ERRORS.value,
"cc_pair_id": attempt.connector_credential_pair_id, "cc_pair_id": attempt.connector_credential_pair_id,
"search_settings_id": attempt.search_settings_id,
"total_docs_indexed": attempt.total_docs_indexed,
"new_docs_indexed": attempt.new_docs_indexed,
}, },
) )
except Exception: except Exception:
@ -312,10 +305,6 @@ def mark_attempt_canceled(
"index_attempt_id": index_attempt_id, "index_attempt_id": index_attempt_id,
"status": IndexingStatus.CANCELED.value, "status": IndexingStatus.CANCELED.value,
"cc_pair_id": attempt.connector_credential_pair_id, "cc_pair_id": attempt.connector_credential_pair_id,
"search_settings_id": attempt.search_settings_id,
"reason": reason,
"total_docs_indexed": attempt.total_docs_indexed,
"new_docs_indexed": attempt.new_docs_indexed,
}, },
) )
except Exception: except Exception:
@ -350,10 +339,6 @@ def mark_attempt_failed(
"index_attempt_id": index_attempt_id, "index_attempt_id": index_attempt_id,
"status": IndexingStatus.FAILED.value, "status": IndexingStatus.FAILED.value,
"cc_pair_id": attempt.connector_credential_pair_id, "cc_pair_id": attempt.connector_credential_pair_id,
"search_settings_id": attempt.search_settings_id,
"reason": failure_reason,
"total_docs_indexed": attempt.total_docs_indexed,
"new_docs_indexed": attempt.new_docs_indexed,
}, },
) )
except Exception: except Exception:

1
backend/onyx/utils/telemetry.py
View File

@ -39,6 +39,7 @@ class RecordType(str, Enum):
INDEXING_PROGRESS = "indexing_progress" INDEXING_PROGRESS = "indexing_progress"
INDEXING_COMPLETE = "indexing_complete" INDEXING_COMPLETE = "indexing_complete"
PERMISSION_SYNC_PROGRESS = "permission_sync_progress" PERMISSION_SYNC_PROGRESS = "permission_sync_progress"
PERMISSION_SYNC_COMPLETE = "permission_sync_complete"
INDEX_ATTEMPT_STATUS = "index_attempt_status" INDEX_ATTEMPT_STATUS = "index_attempt_status"