highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-09-21 14:12:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix airtable connector w/ mt cloud + move telem logic to match new st… (#3868)

Browse Source 
* Fix airtable connector w/ mt cloud + move telem logic to match new standard

* Address Greptile comment

* Small fixes/improvements

* Revert back monitoring frequency

* Small monitoring fix
This commit is contained in:
Chris Weaver
2025-01-31 16:29:04 -08:00
committed by GitHub
parent 7bc8554e01
commit a5d2f0d9ac
7 changed files with 55 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
backend/onyx/connectors/airtable/airtable_connector.py
View File

@@ -1,4 +1,6 @@
import contextvars
from concurrent.futures import as_completed
from concurrent.futures import Future
from concurrent.futures import ThreadPoolExecutor
from io import BytesIO
from typing import Any
@@ -347,15 +349,19 @@ class AirtableConnector(LoadConnector):
with ThreadPoolExecutor(max_workers=max_workers) as executor:
# Submit batch tasks
future_to_record = {
executor.submit(
self._process_record,
record=record,
table_schema=table_schema,
primary_field_name=primary_field_name,
): record
for record in batch_records
}
future_to_record: dict[Future, RecordDict] = {}
for record in batch_records:
# Capture the current context so that the thread gets the current tenant ID
current_context = contextvars.copy_context()
future_to_record[
executor.submit(
current_context.run,
self._process_record,
record=record,
table_schema=table_schema,
primary_field_name=primary_field_name,
)
] = record
# Wait for all tasks in this batch to complete
for future in as_completed(future_to_record):