highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-06-23 14:30:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

Airtable improvement

Browse Source
This commit is contained in:
Weves 2025-01-02 17:49:07 -08:00 committed by Chris Weaver
parent 2232702e99
commit 3b214133a8
1 changed files with 43 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/onyx/connectors/airtable/airtable_connector.py
View File

@ -200,7 +200,10 @@ class AirtableConnector(LoadConnector):
table = self.airtable_client.table(self.base_id, self.table_name_or_id)
table_id = table.id
record_pages = table.iterate()
# due to https://community.airtable.com/t5/development-apis/pagination-returns-422-error/td-p/54778,
# we can't user the `iterate()` method - we need to get everything up front
# this also means we can't handle tables that won't fit in memory
records = table.all()
table_schema = table.schema()
# have to get the name from the schema, since the table object will
@ -216,8 +219,7 @@ class AirtableConnector(LoadConnector):
break
record_documents: list[Document] = []
for page in record_pages:
for record in page:
for record in records:
record_id = record["id"]
fields = record["fields"]
sections: list[Section] = []