mirror of
https://github.com/danswer-ai/danswer.git
synced 2025-07-29 14:24:06 +02:00
909403a648
* first cut at slack oauth flow
* fix usage of hooks
* fix button spacing
* add additional error logging
* no dev redirect
* early cut at google drive oauth
* second pass
* switch to production uri's
* try handling oauth_interactive differently
* pass through client id and secret if uploaded
* fix call
* fix test
* temporarily disable check for testing
* Revert "temporarily disable check for testing"
This reverts commit 4b5a022a5f.
* support visibility in test
* missed file
* first cut at confluence oauth
* work in progress
* work in progress
* work in progress
* work in progress
* work in progress
* first cut at distributed locking
* WIP to make test work
* add some dev mode affordances and gate usage of redis behind dynamic credentials
* mypy and credentials provider fixes
* WIP
* fix created at
* fix setting initialValue on everything
* remove debugging, fix ??? some TextFormField issues
* npm fixes
* comment cleanup
* fix comments
* pin the size of the card section
* more review fixes
* more fixes
---------
Co-authored-by: Richard Kuo <rkuo@rkuo.com>
Co-authored-by: Richard Kuo (Danswer) <rkuo@onyx.app>
77 lines
2.7 KiB
Python
77 lines
2.7 KiB
Python
from datetime import datetime
|
|
from datetime import timezone
|
|
|
|
from onyx.access.models import DocExternalAccess
|
|
from onyx.access.models import ExternalAccess
|
|
from onyx.connectors.gmail.connector import GmailConnector
|
|
from onyx.connectors.interfaces import GenerateSlimDocumentOutput
|
|
from onyx.db.models import ConnectorCredentialPair
|
|
from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
|
|
from onyx.utils.logger import setup_logger
|
|
|
|
logger = setup_logger()
|
|
|
|
|
|
def _get_slim_doc_generator(
|
|
cc_pair: ConnectorCredentialPair,
|
|
gmail_connector: GmailConnector,
|
|
callback: IndexingHeartbeatInterface | None = None,
|
|
) -> GenerateSlimDocumentOutput:
|
|
current_time = datetime.now(timezone.utc)
|
|
start_time = (
|
|
cc_pair.last_time_perm_sync.replace(tzinfo=timezone.utc).timestamp()
|
|
if cc_pair.last_time_perm_sync
|
|
else 0.0
|
|
)
|
|
|
|
return gmail_connector.retrieve_all_slim_documents(
|
|
start=start_time,
|
|
end=current_time.timestamp(),
|
|
callback=callback,
|
|
)
|
|
|
|
|
|
def gmail_doc_sync(
|
|
cc_pair: ConnectorCredentialPair,
|
|
callback: IndexingHeartbeatInterface | None,
|
|
) -> list[DocExternalAccess]:
|
|
"""
|
|
Adds the external permissions to the documents in postgres
|
|
if the document doesn't already exists in postgres, we create
|
|
it in postgres so that when it gets created later, the permissions are
|
|
already populated
|
|
"""
|
|
gmail_connector = GmailConnector(**cc_pair.connector.connector_specific_config)
|
|
gmail_connector.load_credentials(cc_pair.credential.credential_json)
|
|
|
|
slim_doc_generator = _get_slim_doc_generator(
|
|
cc_pair, gmail_connector, callback=callback
|
|
)
|
|
|
|
document_external_access: list[DocExternalAccess] = []
|
|
for slim_doc_batch in slim_doc_generator:
|
|
for slim_doc in slim_doc_batch:
|
|
if callback:
|
|
if callback.should_stop():
|
|
raise RuntimeError("gmail_doc_sync: Stop signal detected")
|
|
|
|
callback.progress("gmail_doc_sync", 1)
|
|
|
|
if slim_doc.perm_sync_data is None:
|
|
logger.warning(f"No permissions found for document {slim_doc.id}")
|
|
continue
|
|
if user_email := slim_doc.perm_sync_data.get("user_email"):
|
|
ext_access = ExternalAccess(
|
|
external_user_emails=set([user_email]),
|
|
external_user_group_ids=set(),
|
|
is_public=False,
|
|
)
|
|
document_external_access.append(
|
|
DocExternalAccess(
|
|
doc_id=slim_doc.id,
|
|
external_access=ext_access,
|
|
)
|
|
)
|
|
|
|
return document_external_access
|