diff --git a/backend/danswer/connectors/google_drive/connector_auth.py b/backend/danswer/connectors/google_drive/connector_auth.py index e360a240f70c..87962cde29d3 100644 --- a/backend/danswer/connectors/google_drive/connector_auth.py +++ b/backend/danswer/connectors/google_drive/connector_auth.py @@ -61,9 +61,7 @@ def verify_csrf(credential_id: int, state: str) -> None: ) -def get_auth_url( - credential_id: int, -) -> str: +def get_auth_url(credential_id: int) -> str: creds_str = str(get_dynamic_config_store().load(GOOGLE_DRIVE_CRED_KEY)) credential_json = json.loads(creds_str) flow = InstalledAppFlow.from_client_config( diff --git a/backend/danswer/datastores/qdrant/indexing.py b/backend/danswer/datastores/qdrant/indexing.py index c0f8e1da9d86..b4414be3bd85 100644 --- a/backend/danswer/datastores/qdrant/indexing.py +++ b/backend/danswer/datastores/qdrant/indexing.py @@ -47,7 +47,6 @@ def create_collection( raise RuntimeError("Could not create Qdrant collection") -@log_function_time() def get_document_whitelists( doc_chunk_id: str, collection_name: str, q_client: QdrantClient ) -> tuple[int, list[str], list[str]]: @@ -66,7 +65,6 @@ def get_document_whitelists( return len(results), payload[ALLOWED_USERS], payload[ALLOWED_GROUPS] -@log_function_time() def delete_doc_chunks( document_id: str, collection_name: str, q_client: QdrantClient ) -> None: diff --git a/backend/danswer/db/connector.py b/backend/danswer/db/connector.py index 21a9187c7d08..a2e91436ddf8 100644 --- a/backend/danswer/db/connector.py +++ b/backend/danswer/db/connector.py @@ -272,10 +272,12 @@ def fetch_latest_index_attempts_by_status( subquery = ( db_session.query( IndexAttempt.connector_id, + IndexAttempt.credential_id, IndexAttempt.status, func.max(IndexAttempt.time_updated).label("time_updated"), ) .group_by(IndexAttempt.connector_id) + .group_by(IndexAttempt.credential_id) .group_by(IndexAttempt.status) .subquery() ) @@ -286,6 +288,7 @@ def fetch_latest_index_attempts_by_status( alias, and_( IndexAttempt.connector_id == alias.connector_id, + IndexAttempt.credential_id == alias.credential_id, IndexAttempt.status == alias.status, IndexAttempt.time_updated == alias.time_updated, ), diff --git a/backend/danswer/db/credentials.py b/backend/danswer/db/credentials.py index 872a312ae147..8aadf7c2ab3d 100644 --- a/backend/danswer/db/credentials.py +++ b/backend/danswer/db/credentials.py @@ -96,9 +96,6 @@ def update_credential_json( user: User, db_session: Session, ) -> Credential | None: - logger.info("HIIII") - logger.info(credential_id) - logger.info(credential_json) credential = fetch_credential_by_id(credential_id, user, db_session) if credential is None: return None diff --git a/backend/danswer/main.py b/backend/danswer/main.py index 89cc309c10e4..32b7cac8ca93 100644 --- a/backend/danswer/main.py +++ b/backend/danswer/main.py @@ -12,9 +12,9 @@ from danswer.configs.app_configs import SECRET from danswer.configs.app_configs import WEB_DOMAIN from danswer.datastores.qdrant.indexing import list_collections from danswer.db.credentials import create_initial_public_credential -from danswer.server.admin import router as admin_router from danswer.server.event_loading import router as event_processing_router from danswer.server.health import router as health_router +from danswer.server.manage import router as admin_router from danswer.server.search_backend import router as backend_router from danswer.utils.logging import setup_logger from fastapi import FastAPI diff --git a/backend/danswer/server/admin.py b/backend/danswer/server/manage.py similarity index 82% rename from backend/danswer/server/admin.py rename to backend/danswer/server/manage.py index cdaf2a8bad46..3d6a6c079b94 100644 --- a/backend/danswer/server/admin.py +++ b/backend/danswer/server/manage.py @@ -2,6 +2,7 @@ from collections import defaultdict from typing import cast from danswer.auth.users import current_admin_user +from danswer.auth.users import current_user from danswer.configs.app_configs import MASK_CREDENTIAL_PREFIX from danswer.configs.constants import DocumentSource from danswer.configs.constants import OPENAI_API_KEY_STORAGE_KEY @@ -32,6 +33,7 @@ from danswer.db.credentials import mask_credential_dict from danswer.db.credentials import update_credential from danswer.db.engine import get_session from danswer.db.index_attempt import create_index_attempt +from danswer.db.models import Connector from danswer.db.models import IndexAttempt from danswer.db.models import IndexingStatus from danswer.db.models import User @@ -61,12 +63,16 @@ from fastapi import Request from fastapi import Response from sqlalchemy.orm import Session -router = APIRouter(prefix="/admin") +router = APIRouter(prefix="/manage") logger = setup_logger() +_GOOGLE_DRIVE_CREDENTIAL_ID_COOKIE_NAME = "google_drive_credential_id" -@router.get("/connector/google-drive/app-credential") +"""Admin only API endpoints""" + + +@router.get("/admin/connector/google-drive/app-credential") def check_google_app_credentials_exist( _: User = Depends(current_admin_user), ) -> dict[str, str]: @@ -76,7 +82,7 @@ def check_google_app_credentials_exist( raise HTTPException(status_code=404, detail="Google App Credentials not found") -@router.put("/connector/google-drive/app-credential") +@router.put("/admin/connector/google-drive/app-credential") def update_google_app_credentials( app_credentials: GoogleAppCredentials, _: User = Depends(current_admin_user) ) -> StatusResponse: @@ -90,7 +96,7 @@ def update_google_app_credentials( ) -@router.get("/connector/google-drive/check-auth/{credential_id}") +@router.get("/admin/connector/google-drive/check-auth/{credential_id}") def check_drive_tokens( credential_id: int, user: User = Depends(current_admin_user), @@ -109,11 +115,8 @@ def check_drive_tokens( return AuthStatus(authenticated=True) -_GOOGLE_DRIVE_CREDENTIAL_ID_COOKIE_NAME = "google_drive_credential_id" - - -@router.get("/connector/google-drive/authorize/{credential_id}", response_model=AuthUrl) -def google_drive_auth( +@router.get("/admin/connector/google-drive/authorize/{credential_id}") +def admin_google_drive_auth( response: Response, credential_id: str, _: User = Depends(current_admin_user) ) -> AuthUrl: # set a cookie that we can read in the callback (used for `verify_csrf`) @@ -123,35 +126,10 @@ def google_drive_auth( httponly=True, max_age=600, ) - return AuthUrl(auth_url=get_auth_url(int(credential_id))) + return AuthUrl(auth_url=get_auth_url(credential_id=int(credential_id))) -@router.get("/connector/google-drive/callback") -def google_drive_callback( - request: Request, - callback: GDriveCallback = Depends(), - user: User = Depends(current_admin_user), - db_session: Session = Depends(get_session), -) -> StatusResponse: - credential_id_cookie = request.cookies.get(_GOOGLE_DRIVE_CREDENTIAL_ID_COOKIE_NAME) - if credential_id_cookie is None or not credential_id_cookie.isdigit(): - raise HTTPException( - status_code=401, detail="Request did not pass CSRF verification." - ) - credential_id = int(credential_id_cookie) - verify_csrf(credential_id, callback.state) - if ( - update_credential_access_tokens(callback.code, credential_id, user, db_session) - is None - ): - raise HTTPException( - status_code=500, detail="Unable to fetch Google Drive access tokens" - ) - - return StatusResponse(success=True, message="Updated Google Drive access tokens") - - -@router.get("/latest-index-attempt", response_model=list[IndexAttemptSnapshot]) +@router.get("/admin/latest-index-attempt") def list_all_index_attempts( _: User = Depends(current_admin_user), db_session: Session = Depends(get_session), @@ -173,7 +151,7 @@ def list_all_index_attempts( ] -@router.get("/latest-index-attempt/{source}", response_model=list[IndexAttemptSnapshot]) +@router.get("/admin/latest-index-attempt/{source}") def list_index_attempts( source: DocumentSource, _: User = Depends(current_admin_user), @@ -196,38 +174,39 @@ def list_index_attempts( ] -@router.get("/connector", response_model=list[ConnectorSnapshot]) -def get_connectors( - _: User = Depends(current_admin_user), - db_session: Session = Depends(get_session), -) -> list[ConnectorSnapshot]: - connectors = fetch_connectors(db_session) - return [ - ConnectorSnapshot.from_connector_db_model(connector) for connector in connectors - ] - - -@router.get("/connector/indexing-status") +@router.get("/admin/connector/indexing-status") def get_connector_indexing_status( _: User = Depends(current_admin_user), db_session: Session = Depends(get_session), ) -> list[ConnectorIndexingStatus]: - connector_id_to_connector = { + connector_id_to_connector: dict[int, Connector] = { connector.id: connector for connector in fetch_connectors(db_session) } index_attempts = fetch_latest_index_attempts_by_status(db_session) - connector_to_index_attempts: dict[int, list[IndexAttempt]] = defaultdict(list) + connector_credential_pair_to_index_attempts: dict[ + tuple[int, int], list[IndexAttempt] + ] = defaultdict(list) for index_attempt in index_attempts: # don't consider index attempts where the connector has been deleted - if index_attempt.connector_id: - connector_to_index_attempts[index_attempt.connector_id].append( - index_attempt - ) + # or the credential has been deleted + if index_attempt.connector_id and index_attempt.credential_id: + connector_credential_pair_to_index_attempts[ + (index_attempt.connector_id, index_attempt.credential_id) + ].append(index_attempt) indexing_statuses: list[ConnectorIndexingStatus] = [] - for connector_id, index_attempts in connector_to_index_attempts.items(): + for ( + connector_id, + credential_id, + ), index_attempts in connector_credential_pair_to_index_attempts.items(): # NOTE: index_attempts is guaranteed to be length > 0 connector = connector_id_to_connector[connector_id] + credential = [ + credential_association.credential + for credential_association in connector.credentials + if credential_association.credential_id == credential_id + ][0] + index_attempts_sorted = sorted( index_attempts, key=lambda x: x.time_updated, reverse=True ) @@ -239,6 +218,8 @@ def get_connector_indexing_status( indexing_statuses.append( ConnectorIndexingStatus( connector=ConnectorSnapshot.from_connector_db_model(connector), + public_doc=credential.public_doc, + owner=credential.user.email if credential.user else "", last_status=index_attempts_sorted[0].status, last_success=successful_index_attempts_sorted[0].time_updated if successful_index_attempts_sorted @@ -250,53 +231,28 @@ def get_connector_indexing_status( ), ) - # add in the connector that haven't started indexing yet + # add in the connectors that haven't started indexing yet for connector in connector_id_to_connector.values(): - if connector.id not in connector_to_index_attempts: - indexing_statuses.append( - ConnectorIndexingStatus( - connector=ConnectorSnapshot.from_connector_db_model(connector), - last_status=IndexingStatus.NOT_STARTED, - last_success=None, - docs_indexed=0, - ), - ) + for credential_association in connector.credentials: + if ( + connector.id, + credential_association.credential_id, + ) not in connector_credential_pair_to_index_attempts: + indexing_statuses.append( + ConnectorIndexingStatus( + connector=ConnectorSnapshot.from_connector_db_model(connector), + public_doc=credential_association.credential.public_doc, + owner=credential.user.email if credential.user else "", + last_status=IndexingStatus.NOT_STARTED, + last_success=None, + docs_indexed=0, + ), + ) return indexing_statuses -@router.get( - "/connector/{connector_id}", - response_model=ConnectorSnapshot | StatusResponse[int], -) -def get_connector_by_id( - connector_id: int, - _: User = Depends(current_admin_user), - db_session: Session = Depends(get_session), -) -> ConnectorSnapshot | StatusResponse[int]: - connector = fetch_connector_by_id(connector_id, db_session) - if connector is None: - raise HTTPException( - status_code=404, detail=f"Connector {connector_id} does not exist" - ) - - return ConnectorSnapshot( - id=connector.id, - name=connector.name, - source=connector.source, - input_type=connector.input_type, - connector_specific_config=connector.connector_specific_config, - refresh_freq=connector.refresh_freq, - credential_ids=[ - association.credential.id for association in connector.credentials - ], - time_created=connector.time_created, - time_updated=connector.time_updated, - disabled=connector.disabled, - ) - - -@router.post("/connector", response_model=ObjectCreationIdResponse) +@router.post("/admin/connector") def create_connector_from_model( connector_info: ConnectorBase, _: User = Depends(current_admin_user), @@ -308,10 +264,7 @@ def create_connector_from_model( raise HTTPException(status_code=400, detail=str(e)) -@router.patch( - "/connector/{connector_id}", - response_model=ConnectorSnapshot | StatusResponse[int], -) +@router.patch("/admin/connector/{connector_id}") def update_connector_from_model( connector_id: int, connector_data: ConnectorBase, @@ -340,7 +293,7 @@ def update_connector_from_model( ) -@router.delete("/connector/{connector_id}", response_model=StatusResponse[int]) +@router.delete("/admin/connector/{connector_id}", response_model=StatusResponse[int]) def delete_connector_by_id( connector_id: int, _: User = Depends(current_admin_user), @@ -349,128 +302,7 @@ def delete_connector_by_id( return delete_connector(connector_id, db_session) -@router.get("/credential", response_model=list[CredentialSnapshot]) -def get_credentials( - user: User = Depends(current_admin_user), - db_session: Session = Depends(get_session), -) -> list[CredentialSnapshot]: - credentials = fetch_credentials(user, db_session) - return [ - CredentialSnapshot( - id=credential.id, - credential_json=mask_credential_dict(credential.credential_json) - if MASK_CREDENTIAL_PREFIX - else credential.credential_json, - user_id=credential.user_id, - public_doc=credential.public_doc, - time_created=credential.time_created, - time_updated=credential.time_updated, - ) - for credential in credentials - ] - - -@router.get( - "/credential/{credential_id}", - response_model=CredentialSnapshot | StatusResponse[int], -) -def get_credential_by_id( - credential_id: int, - user: User = Depends(current_admin_user), - db_session: Session = Depends(get_session), -) -> CredentialSnapshot | StatusResponse[int]: - credential = fetch_credential_by_id(credential_id, user, db_session) - if credential is None: - raise HTTPException( - status_code=401, - detail=f"Credential {credential_id} does not exist or does not belong to user", - ) - - return CredentialSnapshot( - id=credential.id, - credential_json=mask_credential_dict(credential.credential_json) - if MASK_CREDENTIAL_PREFIX - else credential.credential_json, - user_id=credential.user_id, - public_doc=credential.public_doc, - time_created=credential.time_created, - time_updated=credential.time_updated, - ) - - -@router.post("/credential", response_model=ObjectCreationIdResponse) -def create_credential_from_model( - connector_info: CredentialBase, - user: User = Depends(current_admin_user), - db_session: Session = Depends(get_session), -) -> ObjectCreationIdResponse: - return create_credential(connector_info, user, db_session) - - -@router.patch( - "/credential/{credential_id}", - response_model=CredentialSnapshot | StatusResponse[int], -) -def update_credential_from_model( - credential_id: int, - credential_data: CredentialBase, - user: User = Depends(current_admin_user), - db_session: Session = Depends(get_session), -) -> CredentialSnapshot | StatusResponse[int]: - updated_credential = update_credential( - credential_id, credential_data, user, db_session - ) - if updated_credential is None: - raise HTTPException( - status_code=401, - detail=f"Credential {credential_id} does not exist or does not belong to user", - ) - - return CredentialSnapshot( - id=updated_credential.id, - credential_json=updated_credential.credential_json, - user_id=updated_credential.user_id, - public_doc=updated_credential.public_doc, - time_created=updated_credential.time_created, - time_updated=updated_credential.time_updated, - ) - - -@router.delete("/credential/{credential_id}", response_model=StatusResponse[int]) -def delete_credential_by_id( - credential_id: int, - user: User = Depends(current_admin_user), - db_session: Session = Depends(get_session), -) -> StatusResponse: - delete_credential(credential_id, user, db_session) - return StatusResponse( - success=True, message="Credential deleted successfully", data=credential_id - ) - - -@router.put("/connector/{connector_id}/credential/{credential_id}") -def associate_credential_to_connector( - connector_id: int, - credential_id: int, - user: User = Depends(current_admin_user), - db_session: Session = Depends(get_session), -) -> StatusResponse[int]: - return add_credential_to_connector(connector_id, credential_id, user, db_session) - - -@router.delete("/connector/{connector_id}/credential/{credential_id}") -def dissociate_credential_from_connector( - connector_id: int, - credential_id: int, - user: User = Depends(current_admin_user), - db_session: Session = Depends(get_session), -) -> StatusResponse[int]: - return remove_credential_from_connector( - connector_id, credential_id, user, db_session - ) - - -@router.post("/connector/run-once") +@router.post("/admin/connector/run-once") def connector_run_once( run_info: RunConnectorRequest, _: User = Depends(current_admin_user), @@ -516,7 +348,7 @@ def connector_run_once( ) -@router.head("/openai-api-key/validate") +@router.head("/admin/openai-api-key/validate") def validate_existing_openai_api_key( _: User = Depends(current_admin_user), ) -> None: @@ -532,7 +364,7 @@ def validate_existing_openai_api_key( raise HTTPException(status_code=400, detail="Invalid API key provided") -@router.get("/openai-api-key", response_model=ApiKey) +@router.get("/admin/openai-api-key", response_model=ApiKey) def get_openai_api_key_from_dynamic_config_store( _: User = Depends(current_admin_user), ) -> ApiKey: @@ -550,7 +382,7 @@ def get_openai_api_key_from_dynamic_config_store( raise HTTPException(status_code=404, detail="Key not found") -@router.post("/openai-api-key") +@router.put("/admin/openai-api-key") def store_openai_api_key( request: ApiKey, _: User = Depends(current_admin_user), @@ -564,8 +396,204 @@ def store_openai_api_key( raise HTTPException(400, str(e)) -@router.delete("/openai-api-key") +@router.delete("/admin/openai-api-key") def delete_openai_api_key( _: User = Depends(current_admin_user), ) -> None: get_dynamic_config_store().delete(OPENAI_API_KEY_STORAGE_KEY) + + +"""Endpoints for all!""" + + +@router.get("/connector/google-drive/authorize/{credential_id}") +def google_drive_auth( + response: Response, credential_id: str, _: User = Depends(current_user) +) -> AuthUrl: + # set a cookie that we can read in the callback (used for `verify_csrf`) + response.set_cookie( + key=_GOOGLE_DRIVE_CREDENTIAL_ID_COOKIE_NAME, + value=credential_id, + httponly=True, + max_age=600, + ) + return AuthUrl(auth_url=get_auth_url(int(credential_id))) + + +@router.get("/connector/google-drive/callback") +def google_drive_callback( + request: Request, + callback: GDriveCallback = Depends(), + user: User = Depends(current_user), + db_session: Session = Depends(get_session), +) -> StatusResponse: + credential_id_cookie = request.cookies.get(_GOOGLE_DRIVE_CREDENTIAL_ID_COOKIE_NAME) + if credential_id_cookie is None or not credential_id_cookie.isdigit(): + raise HTTPException( + status_code=401, detail="Request did not pass CSRF verification." + ) + credential_id = int(credential_id_cookie) + verify_csrf(credential_id, callback.state) + if ( + update_credential_access_tokens(callback.code, credential_id, user, db_session) + is None + ): + raise HTTPException( + status_code=500, detail="Unable to fetch Google Drive access tokens" + ) + + return StatusResponse(success=True, message="Updated Google Drive access tokens") + + +@router.get("/connector") +def get_connectors( + _: User = Depends(current_user), + db_session: Session = Depends(get_session), +) -> list[ConnectorSnapshot]: + connectors = fetch_connectors(db_session) + return [ + ConnectorSnapshot.from_connector_db_model(connector) for connector in connectors + ] + + +@router.get("/connector/{connector_id}") +def get_connector_by_id( + connector_id: int, + _: User = Depends(current_user), + db_session: Session = Depends(get_session), +) -> ConnectorSnapshot | StatusResponse[int]: + connector = fetch_connector_by_id(connector_id, db_session) + if connector is None: + raise HTTPException( + status_code=404, detail=f"Connector {connector_id} does not exist" + ) + + return ConnectorSnapshot( + id=connector.id, + name=connector.name, + source=connector.source, + input_type=connector.input_type, + connector_specific_config=connector.connector_specific_config, + refresh_freq=connector.refresh_freq, + credential_ids=[ + association.credential.id for association in connector.credentials + ], + time_created=connector.time_created, + time_updated=connector.time_updated, + disabled=connector.disabled, + ) + + +@router.get("/credential") +def get_credentials( + user: User = Depends(current_user), + db_session: Session = Depends(get_session), +) -> list[CredentialSnapshot]: + credentials = fetch_credentials(user, db_session) + return [ + CredentialSnapshot( + id=credential.id, + credential_json=mask_credential_dict(credential.credential_json) + if MASK_CREDENTIAL_PREFIX + else credential.credential_json, + user_id=credential.user_id, + public_doc=credential.public_doc, + time_created=credential.time_created, + time_updated=credential.time_updated, + ) + for credential in credentials + ] + + +@router.get("/credential/{credential_id}") +def get_credential_by_id( + credential_id: int, + user: User = Depends(current_user), + db_session: Session = Depends(get_session), +) -> CredentialSnapshot | StatusResponse[int]: + credential = fetch_credential_by_id(credential_id, user, db_session) + if credential is None: + raise HTTPException( + status_code=401, + detail=f"Credential {credential_id} does not exist or does not belong to user", + ) + + return CredentialSnapshot( + id=credential.id, + credential_json=mask_credential_dict(credential.credential_json) + if MASK_CREDENTIAL_PREFIX + else credential.credential_json, + user_id=credential.user_id, + public_doc=credential.public_doc, + time_created=credential.time_created, + time_updated=credential.time_updated, + ) + + +@router.post("/credential") +def create_credential_from_model( + connector_info: CredentialBase, + user: User = Depends(current_user), + db_session: Session = Depends(get_session), +) -> ObjectCreationIdResponse: + return create_credential(connector_info, user, db_session) + + +@router.patch("/credential/{credential_id}") +def update_credential_from_model( + credential_id: int, + credential_data: CredentialBase, + user: User = Depends(current_user), + db_session: Session = Depends(get_session), +) -> CredentialSnapshot | StatusResponse[int]: + updated_credential = update_credential( + credential_id, credential_data, user, db_session + ) + if updated_credential is None: + raise HTTPException( + status_code=401, + detail=f"Credential {credential_id} does not exist or does not belong to user", + ) + + return CredentialSnapshot( + id=updated_credential.id, + credential_json=updated_credential.credential_json, + user_id=updated_credential.user_id, + public_doc=updated_credential.public_doc, + time_created=updated_credential.time_created, + time_updated=updated_credential.time_updated, + ) + + +@router.delete("/credential/{credential_id}") +def delete_credential_by_id( + credential_id: int, + user: User = Depends(current_user), + db_session: Session = Depends(get_session), +) -> StatusResponse: + delete_credential(credential_id, user, db_session) + return StatusResponse( + success=True, message="Credential deleted successfully", data=credential_id + ) + + +@router.put("/connector/{connector_id}/credential/{credential_id}") +def associate_credential_to_connector( + connector_id: int, + credential_id: int, + user: User = Depends(current_user), + db_session: Session = Depends(get_session), +) -> StatusResponse[int]: + return add_credential_to_connector(connector_id, credential_id, user, db_session) + + +@router.delete("/connector/{connector_id}/credential/{credential_id}") +def dissociate_credential_from_connector( + connector_id: int, + credential_id: int, + user: User = Depends(current_user), + db_session: Session = Depends(get_session), +) -> StatusResponse[int]: + return remove_credential_from_connector( + connector_id, credential_id, user, db_session + ) diff --git a/backend/danswer/server/models.py b/backend/danswer/server/models.py index ee294dcd9a5e..6b4667715de1 100644 --- a/backend/danswer/server/models.py +++ b/backend/danswer/server/models.py @@ -137,6 +137,8 @@ class ConnectorIndexingStatus(BaseModel): """Represents the latest indexing status of a connector""" connector: ConnectorSnapshot + owner: str + public_doc: bool last_status: IndexingStatus last_success: datetime | None docs_indexed: int diff --git a/web/package-lock.json b/web/package-lock.json index 2c2b060a7bd5..f3e18d02695b 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "dependencies": { "@phosphor-icons/react": "^2.0.8", + "@types/js-cookie": "^3.0.3", "@types/node": "18.15.11", "@types/react": "18.0.32", "@types/react-dom": "18.0.11", @@ -16,6 +17,7 @@ "eslint": "8.37.0", "eslint-config-next": "13.2.4", "formik": "^2.2.9", + "js-cookie": "^3.0.5", "next": "^13.2.4", "postcss": "^8.4.23", "react": "^18.2.0", @@ -392,6 +394,11 @@ "tslib": "^2.4.0" } }, + "node_modules/@types/js-cookie": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/js-cookie/-/js-cookie-3.0.3.tgz", + "integrity": "sha512-Xe7IImK09HP1sv2M/aI+48a20VX+TdRJucfq4vfRVy6nWN8PYPOEnlMRSgxJAgYQIXJVL8dZ4/ilAM7dWNaOww==" + }, "node_modules/@types/json5": { "version": "0.0.29", "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", @@ -2483,6 +2490,14 @@ "jiti": "bin/jiti.js" } }, + "node_modules/js-cookie": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.5.tgz", + "integrity": "sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==", + "engines": { + "node": ">=14" + } + }, "node_modules/js-sdsl": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.4.0.tgz", @@ -4324,6 +4339,11 @@ "tslib": "^2.4.0" } }, + "@types/js-cookie": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/js-cookie/-/js-cookie-3.0.3.tgz", + "integrity": "sha512-Xe7IImK09HP1sv2M/aI+48a20VX+TdRJucfq4vfRVy6nWN8PYPOEnlMRSgxJAgYQIXJVL8dZ4/ilAM7dWNaOww==" + }, "@types/json5": { "version": "0.0.29", "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", @@ -5791,6 +5811,11 @@ "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.18.2.tgz", "integrity": "sha512-QAdOptna2NYiSSpv0O/BwoHBSmz4YhpzJHyi+fnMRTXFjp7B8i/YG5Z8IfusxB1ufjcD2Sre1F3R+nX3fvy7gg==" }, + "js-cookie": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/js-cookie/-/js-cookie-3.0.5.tgz", + "integrity": "sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==" + }, "js-sdsl": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.4.0.tgz", diff --git a/web/package.json b/web/package.json index d6f3a3bff876..d215334a1991 100644 --- a/web/package.json +++ b/web/package.json @@ -10,6 +10,7 @@ }, "dependencies": { "@phosphor-icons/react": "^2.0.8", + "@types/js-cookie": "^3.0.3", "@types/node": "18.15.11", "@types/react": "18.0.32", "@types/react-dom": "18.0.11", @@ -17,6 +18,7 @@ "eslint": "8.37.0", "eslint-config-next": "13.2.4", "formik": "^2.2.9", + "js-cookie": "^3.0.5", "next": "^13.2.4", "postcss": "^8.4.23", "react": "^18.2.0", diff --git a/web/src/app/admin/connectors/confluence/page.tsx b/web/src/app/admin/connectors/confluence/page.tsx index f574ca957a13..9e8c26f3f5cc 100644 --- a/web/src/app/admin/connectors/confluence/page.tsx +++ b/web/src/app/admin/connectors/confluence/page.tsx @@ -25,7 +25,7 @@ const Main = () => { isLoading: isConnectorIndexingStatusesLoading, error: isConnectorIndexingStatusesError, } = useSWR[]>( - "/api/admin/connector/indexing-status", + "/api/manage/admin/connector/indexing-status", fetcher ); const { @@ -34,7 +34,7 @@ const Main = () => { isValidating: isCredentialsValidating, error: isCredentialsError, } = useSWR[]>( - "/api/admin/credential", + "/api/manage/credential", fetcher ); @@ -85,7 +85,7 @@ const Main = () => { className="ml-1 hover:bg-gray-700 rounded-full p-1" onClick={async () => { await deleteCredential(confluenceCredential.id); - mutate("/api/admin/credential"); + mutate("/api/manage/credential"); }} > @@ -131,7 +131,7 @@ const Main = () => { }} onSubmit={(isSuccess) => { if (isSuccess) { - mutate("/api/admin/credential"); + mutate("/api/manage/credential"); } }} /> @@ -181,7 +181,7 @@ const Main = () => { onCredentialLink={async (connectorId) => { if (confluenceCredential) { await linkCredential(connectorId, confluenceCredential.id); - mutate("/api/admin/connector/indexing-status"); + mutate("/api/manage/admin/connector/indexing-status"); } }} specialColumns={[ @@ -198,7 +198,9 @@ const Main = () => { ), }, ]} - onUpdate={() => mutate("/api/admin/connector/indexing-status")} + onUpdate={() => + mutate("/api/manage/admin/connector/indexing-status") + } /> @@ -229,7 +231,7 @@ const Main = () => { onSubmit={async (isSuccess, responseJson) => { if (isSuccess && responseJson) { await linkCredential(responseJson.id, confluenceCredential.id); - mutate("/api/admin/connector/indexing-status"); + mutate("/api/manage/admin/connector/indexing-status"); } }} /> diff --git a/web/src/app/admin/connectors/github/page.tsx b/web/src/app/admin/connectors/github/page.tsx index 79a96543815f..23a83f7c62c2 100644 --- a/web/src/app/admin/connectors/github/page.tsx +++ b/web/src/app/admin/connectors/github/page.tsx @@ -25,7 +25,7 @@ const Main = () => { isLoading: isConnectorIndexingStatusesLoading, error: isConnectorIndexingStatusesError, } = useSWR[]>( - "/api/admin/connector/indexing-status", + "/api/manage/admin/connector/indexing-status", fetcher ); @@ -35,7 +35,7 @@ const Main = () => { isValidating: isCredentialsValidating, error: isCredentialsError, } = useSWR[]>( - "/api/admin/credential", + "/api/manage/credential", fetcher ); @@ -81,7 +81,7 @@ const Main = () => { className="ml-1 hover:bg-gray-700 rounded-full p-1" onClick={async () => { await deleteCredential(githubCredential.id); - mutate("/api/admin/credential"); + mutate("/api/manage/credential"); }} > @@ -121,7 +121,7 @@ const Main = () => { }} onSubmit={(isSuccess) => { if (isSuccess) { - mutate("/api/admin/credential"); + mutate("/api/manage/credential"); } }} /> @@ -149,7 +149,7 @@ const Main = () => { onCredentialLink={async (connectorId) => { if (githubCredential) { await linkCredential(connectorId, githubCredential.id); - mutate("/api/admin/connector/indexing-status"); + mutate("/api/manage/admin/connector/indexing-status"); } }} specialColumns={[ @@ -160,7 +160,9 @@ const Main = () => { `${connector.connector_specific_config.repo_owner}/${connector.connector_specific_config.repo_name}`, }, ]} - onUpdate={() => mutate("/api/admin/connector/indexing-status")} + onUpdate={() => + mutate("/api/manage/admin/connector/indexing-status") + } /> @@ -196,7 +198,7 @@ const Main = () => { onSubmit={async (isSuccess, responseJson) => { if (isSuccess && responseJson) { await linkCredential(responseJson.id, githubCredential.id); - mutate("/api/admin/connector/indexing-status"); + mutate("/api/manage/admin/connector/indexing-status"); } }} /> diff --git a/web/src/app/admin/connectors/google-drive/auth/callback/route.ts b/web/src/app/admin/connectors/google-drive/auth/callback/route.ts index 829a49d73abd..ba740a72a319 100644 --- a/web/src/app/admin/connectors/google-drive/auth/callback/route.ts +++ b/web/src/app/admin/connectors/google-drive/auth/callback/route.ts @@ -2,11 +2,12 @@ import { getDomain } from "@/lib/redirectSS"; import { buildUrl } from "@/lib/utilsSS"; import { NextRequest, NextResponse } from "next/server"; import { cookies } from "next/headers"; +import { GOOGLE_DRIVE_AUTH_IS_ADMIN_COOKIE_NAME } from "@/lib/constants"; export const GET = async (request: NextRequest) => { // Wrapper around the FastAPI endpoint /connectors/google-drive/callback, // which adds back a redirect to the Google Drive admin page. - const url = new URL(buildUrl("/admin/connector/google-drive/callback")); + const url = new URL(buildUrl("/manage/connector/google-drive/callback")); url.search = request.nextUrl.search; const response = await fetch(url.toString(), { @@ -26,7 +27,14 @@ export const GET = async (request: NextRequest) => { return NextResponse.redirect(new URL("/auth/error", getDomain(request))); } - return NextResponse.redirect( - new URL("/admin/connectors/google-drive", getDomain(request)) - ); + if ( + cookies() + .get(GOOGLE_DRIVE_AUTH_IS_ADMIN_COOKIE_NAME) + ?.value?.toLowerCase() === "true" + ) { + return NextResponse.redirect( + new URL("/admin/connectors/google-drive", getDomain(request)) + ); + } + return NextResponse.redirect(new URL("/user/connectors", getDomain(request))); }; diff --git a/web/src/app/admin/connectors/google-drive/page.tsx b/web/src/app/admin/connectors/google-drive/page.tsx index d9d9221ca7a8..4426f6d0e032 100644 --- a/web/src/app/admin/connectors/google-drive/page.tsx +++ b/web/src/app/admin/connectors/google-drive/page.tsx @@ -18,6 +18,10 @@ import { } from "@/lib/types"; import { deleteConnector } from "@/lib/connector"; import { StatusRow } from "@/components/admin/connectors/table/ConnectorsTable"; +import { setupGoogleDriveOAuth } from "@/lib/googleDrive"; +import Cookies from "js-cookie"; +import { GOOGLE_DRIVE_AUTH_IS_ADMIN_COOKIE_NAME } from "@/lib/constants"; +import { deleteCredential, linkCredential } from "@/lib/credential"; const AppCredentialUpload = ({ setPopup, @@ -61,7 +65,7 @@ const AppCredentialUpload = ({ disabled={!appCredentialJsonStr} onClick={async () => { const response = await fetch( - "/api/admin/connector/google-drive/app-credential", + "/api/manage/admin/connector/google-drive/app-credential", { method: "PUT", headers: { @@ -98,7 +102,7 @@ const Main = () => { isLoading: isAppCredentialLoading, error: isAppCredentialError, } = useSWR<{ client_id: string }>( - "/api/admin/connector/google-drive/app-credential", + "/api/manage/admin/connector/google-drive/app-credential", fetcher ); const { @@ -106,7 +110,7 @@ const Main = () => { isLoading: isConnectorIndexingStatusesLoading, error: isConnectorIndexingStatusesError, } = useSWR[]>( - "/api/admin/connector/indexing-status", + "/api/manage/admin/connector/indexing-status", fetcher ); const { @@ -114,7 +118,7 @@ const Main = () => { isLoading: isCredentialsLoading, error: isCredentialsError, } = useSWR[]>( - "/api/admin/credential", + "/api/manage/credential", fetcher ); @@ -167,6 +171,10 @@ const Main = () => { ); } + const googleDrivePublicCredential = credentialsData.find( + (credential) => + credential.credential_json?.google_drive_tokens && credential.public_doc + ); const googleDriveConnectorIndexingStatuses: ConnectorIndexingStatus<{}>[] = connectorIndexingStatuses.filter( (connectorIndexingStatus) => @@ -174,9 +182,13 @@ const Main = () => { ); const googleDriveConnectorIndexingStatus = googleDriveConnectorIndexingStatuses[0]; - const googleDriveCredential = credentialsData.filter( - (credential) => credential.credential_json?.google_drive_tokens - )[0]; + + const credentialIsLinked = + googleDriveConnectorIndexingStatus !== undefined && + googleDrivePublicCredential !== undefined && + googleDriveConnectorIndexingStatus.connector.credential_ids.includes( + googleDrivePublicCredential.id + ); return ( <> @@ -198,7 +210,9 @@ const Main = () => {
{ - mutate("/api/admin/connector/google-drive/app-credential"); + mutate( + "/api/manage/admin/connector/google-drive/app-credential" + ); setPopupWithExpiration(popup); }} /> @@ -221,7 +235,9 @@ const Main = () => {

{ - mutate("/api/admin/connector/google-drive/app-credential"); + mutate( + "/api/manage/admin/connector/google-drive/app-credential" + ); setPopupWithExpiration(popup); }} /> @@ -233,175 +249,199 @@ const Main = () => { Step 2: Authenticate with Danswer
- {googleDriveCredential ? ( -

- Existing credential already setup! If you want to reset that - credential, click the button below to go through the OAuth flow - again. -

+ {googleDrivePublicCredential ? ( + <> +

+ Existing credential already setup! +

+ + ) : ( <> -

+

Next, you must provide credentials via OAuth. This gives us read access to the docs you have access to in your google drive account.

+ )}
-

Step 3: Start Indexing!

- {googleDriveConnectorIndexingStatus ? ( -
-
-
- The Google Drive connector is setup!{" "} - Status:{" "} - { - mutate("/api/admin/connector/indexing-status"); + {googleDrivePublicCredential ? ( + googleDriveConnectorIndexingStatus ? ( + credentialIsLinked ? ( +
+
+
+ The Google Drive connector is setup!{" "} + Status:{" "} + { + mutate("/api/manage/admin/connector/indexing-status"); + }} + /> +
+

+ Checkout the{" "} + + status page + {" "} + for the latest indexing status. We fetch the latest documents + from Google Drive every 10 minutes. +

+
+
-

- Checkout the{" "} - - status page - {" "} - for the latest indexing status. We fetch the latest documents from - Google Drive every 10 minutes. + ) : ( + <> +

+ Click the button below to link your credentials! Once this is + done, all public documents in your Google Drive will be + searchable. We will refresh the latest documents every 10{" "} + minutes. +

+ + + ) + ) : ( + <> +

+ Click the button below to create a connector. We will refresh the + latest documents from Google Drive every 10 minutes.

-
- -
+ mutate("/api/manage/admin/connector/indexing-status"); + }} + > + Add + + + ) ) : ( - <> -

- Click the button below to create a connector. We will refresh the - latest documents from Google Drive every 10 minutes. -

- - +

+ Please authenticate with Google Drive as described in Step 2! Once + done with that, you can then move on to enable this connector. +

)} ); diff --git a/web/src/app/admin/connectors/slack/page.tsx b/web/src/app/admin/connectors/slack/page.tsx index fc53185de3db..1f16e9071cca 100644 --- a/web/src/app/admin/connectors/slack/page.tsx +++ b/web/src/app/admin/connectors/slack/page.tsx @@ -26,7 +26,7 @@ const MainSection = () => { isLoading: isConnectorIndexingStatusesLoading, error: isConnectorIndexingStatusesError, } = useSWR[]>( - "/api/admin/connector/indexing-status", + "/api/manage/admin/connector/indexing-status", fetcher ); @@ -36,7 +36,7 @@ const MainSection = () => { isValidating: isCredentialsValidating, error: isCredentialsError, } = useSWR[]>( - "/api/admin/credential", + "/api/manage/credential", fetcher ); @@ -81,7 +81,7 @@ const MainSection = () => { className="ml-1 hover:bg-gray-700 rounded-full p-1" onClick={async () => { await deleteCredential(slackCredential.id); - mutate("/api/admin/credential"); + mutate("/api/manage/credential"); }} > @@ -123,7 +123,7 @@ const MainSection = () => { }} onSubmit={(isSuccess) => { if (isSuccess) { - mutate("/api/admin/credential"); + mutate("/api/manage/credential"); } }} /> @@ -156,11 +156,13 @@ const MainSection = () => { connector.connector_specific_config.workspace, }, ]} - onUpdate={() => mutate("/api/admin/connector/indexing-status")} + onUpdate={() => + mutate("/api/manage/admin/connector/indexing-status") + } onCredentialLink={async (connectorId) => { if (slackCredential) { await linkCredential(connectorId, slackCredential.id); - mutate("/api/admin/connector/indexing-status"); + mutate("/api/manage/admin/connector/indexing-status"); } }} /> @@ -191,7 +193,7 @@ const MainSection = () => { onSubmit={async (isSuccess, responseJson) => { if (isSuccess && responseJson) { await linkCredential(responseJson.id, slackCredential.id); - mutate("/api/admin/connector/indexing-status"); + mutate("/api/manage/admin/connector/indexing-status"); } }} /> diff --git a/web/src/app/admin/connectors/web/page.tsx b/web/src/app/admin/connectors/web/page.tsx index 9d24c0039105..16b5dc8ff3f4 100644 --- a/web/src/app/admin/connectors/web/page.tsx +++ b/web/src/app/admin/connectors/web/page.tsx @@ -21,7 +21,7 @@ export default function Web() { isLoading: isConnectorIndexingStatusesLoading, error: isConnectorIndexingStatusesError, } = useSWR[]>( - "/api/admin/connector/indexing-status", + "/api/manage/admin/connector/indexing-status", fetcher ); @@ -69,7 +69,7 @@ export default function Web() { if (isSuccess && responseJson) { // assumes there is a dummy credential with id 0 await linkCredential(responseJson.id, 0); - mutate("/api/admin/connector/indexing-status"); + mutate("/api/manage/admin/connector/indexing-status"); } }} /> @@ -99,7 +99,7 @@ export default function Web() { ), }, ]} - onUpdate={() => mutate("/api/admin/connector/indexing-status")} + onUpdate={() => mutate("/api/manage/admin/connector/indexing-status")} /> ) : (

No indexed websites found

diff --git a/web/src/app/admin/indexing/status/page.tsx b/web/src/app/admin/indexing/status/page.tsx index edce954ae13d..150837c7f9e9 100644 --- a/web/src/app/admin/indexing/status/page.tsx +++ b/web/src/app/admin/indexing/status/page.tsx @@ -9,12 +9,13 @@ import { NotebookIcon, XSquareIcon } from "@/components/icons/icons"; import { fetcher } from "@/lib/fetcher"; import { getSourceMetadata } from "@/components/source"; import { CheckCircle, XCircle } from "@phosphor-icons/react"; -import { useState } from "react"; -import { Popup } from "@/components/admin/connectors/Popup"; import { HealthCheckBanner } from "@/components/health/healthcheck"; -import { Connector, ConnectorIndexingStatus } from "@/lib/types"; +import { ConnectorIndexingStatus } from "@/lib/types"; -const getSourceDisplay = (connector: Connector) => { +const getSourceDisplay = ( + connectorIndexingStatus: ConnectorIndexingStatus +) => { + const connector = connectorIndexingStatus.connector; const sourceMetadata = getSourceMetadata(connector.source); if (connector.source === "web") { return ( @@ -38,28 +39,140 @@ const getSourceDisplay = (connector: Connector) => { ); } + if ( + connector.source === "google_drive" && + !connectorIndexingStatus.public_doc + ) { + if (connectorIndexingStatus.owner) { + return `${sourceMetadata.displayName} [${connectorIndexingStatus.owner}]`; + } + return `${sourceMetadata.displayName} [private]`; + } + return sourceMetadata.displayName; }; -export default function Status() { +function Main() { const { data: indexAttemptData, isLoading: indexAttemptIsLoading, error: indexAttemptIsError, } = useSWR[]>( - "/api/admin/connector/indexing-status", + "/api/manage/admin/connector/indexing-status", fetcher, { refreshInterval: 30000 } // 30 seconds ); - const [popup, setPopup] = useState<{ - message: string; - type: "success" | "error"; - } | null>(null); + if (indexAttemptIsLoading) { + return ; + } + + if (indexAttemptIsError || !indexAttemptData) { + return
Error loading indexing history.
; + } + + // sort by source name + indexAttemptData.sort((a, b) => { + if (a.connector.source < b.connector.source) { + return -1; + } else if (a.connector.source > b.connector.source) { + return 1; + } else { + return 0; + } + }); + return ( + { + const sourceMetadata = getSourceMetadata( + connectorIndexingStatus.connector.source + ); + let statusDisplay =
In Progress...
; + if (connectorIndexingStatus.connector.disabled) { + statusDisplay = ( +
+ + Disabled +
+ ); + } else if (connectorIndexingStatus.last_status === "success") { + statusDisplay = ( +
+ + Enabled +
+ ); + } else if (connectorIndexingStatus.last_status === "failed") { + statusDisplay = ( +
+ + Error +
+ ); + } + return { + indexed_at: timeAgo(connectorIndexingStatus?.last_success) || "-", + docs_indexed: connectorIndexingStatus?.docs_indexed + ? `${connectorIndexingStatus?.docs_indexed} documents` + : "-", + connector: ( + + {sourceMetadata.icon({ size: "20" })} +
+ {getSourceDisplay(connectorIndexingStatus)} +
+ + ), + status: statusDisplay, + // TODO: add the below back in after this is supported in the backend + // reindex: ( + // + // ), + }; + })} + /> + ); +} + +export default function Status() { return (
- {popup && }
@@ -67,99 +180,7 @@ export default function Status() {

Indexing Status

- - {indexAttemptIsLoading ? ( - - ) : indexAttemptIsError || !indexAttemptData ? ( -
Error loading indexing history
- ) : ( - { - const sourceMetadata = getSourceMetadata( - connectorIndexingStatus.connector.source - ); - let statusDisplay = ( -
In Progress...
- ); - if (connectorIndexingStatus.connector.disabled) { - statusDisplay = ( -
- - Disabled -
- ); - } else if (connectorIndexingStatus.last_status === "success") { - statusDisplay = ( -
- - Enabled -
- ); - } else if (connectorIndexingStatus.last_status === "failed") { - statusDisplay = ( -
- - Error -
- ); - } - return { - indexed_at: timeAgo(connectorIndexingStatus?.last_success) || "-", - docs_indexed: connectorIndexingStatus?.docs_indexed - ? `${connectorIndexingStatus?.docs_indexed} documents` - : "-", - connector: ( - - {sourceMetadata.icon({ size: "20" })} -
- {getSourceDisplay(connectorIndexingStatus.connector)} -
- - ), - status: statusDisplay, - // TODO: add the below back in after this is supported in the backend - // reindex: ( - // - // ), - }; - })} - /> - )} +
); } diff --git a/web/src/app/user/connectors/GoogleDriveCard.tsx b/web/src/app/user/connectors/GoogleDriveCard.tsx new file mode 100644 index 000000000000..e0135150c0d9 --- /dev/null +++ b/web/src/app/user/connectors/GoogleDriveCard.tsx @@ -0,0 +1,146 @@ +import { Button } from "@/components/Button"; +import { GoogleDriveIcon } from "@/components/icons/icons"; +import { deleteCredential, linkCredential } from "@/lib/credential"; +import { setupGoogleDriveOAuth } from "@/lib/googleDrive"; +import { GoogleDriveCredentialJson, Credential } from "@/lib/types"; +import { GOOGLE_DRIVE_AUTH_IS_ADMIN_COOKIE_NAME } from "@/lib/constants"; +import Cookies from "js-cookie"; +import { CardProps } from "./interface"; +import { CheckCircle, MinusCircle } from "@phosphor-icons/react"; + +export const GoogleDriveCard = ({ + connector, + userCredentials, + setPopup, + router, + mutate, +}: CardProps) => { + if (!connector) return null; + + const existingCredential: Credential | undefined = + userCredentials?.find( + (credential) => + credential.credential_json?.google_drive_tokens !== undefined && + !credential.public_doc + ); + + const credentialIsLinked = + existingCredential !== undefined && + connector.credential_ids.includes(existingCredential.id); + + return ( +
+
+ {" "} + Google Drive +
+ +
+ {existingCredential && credentialIsLinked ? ( +
+ + Enabled +
+ ) : ( +
+ + Not Setup +
+ )} +
+ +
+ {existingCredential ? ( + credentialIsLinked ? ( + <> +

+ Danswer has access to your Google Drive documents! Don't + worry, only you will be able to see your private + documents. You can revoke this access by clicking the button + below. +

+
+ +
+ + ) : ( + <> +

+ We've recieved your credentials from Google! Click the + button below to activate the connector - we will pull the latest + state of your documents every 10 minutes. +

+
+ +
+ + ) + ) : ( + <> +

+ If you want to make all your Google Drive documents searchable + through Danswer, click the button below! Don't worry, only{" "} + you will be able to see your private documents. Currently, + you'll only be able to search through documents shared with + the whole company. +

+
+ +
+ + )} +
+
+ ); +}; diff --git a/web/src/app/user/connectors/interface.ts b/web/src/app/user/connectors/interface.ts new file mode 100644 index 000000000000..b3e91d7582b4 --- /dev/null +++ b/web/src/app/user/connectors/interface.ts @@ -0,0 +1,12 @@ +import { PopupSpec } from "@/components/admin/connectors/Popup"; +import { Connector, Credential } from "@/lib/types"; +import { AppRouterInstance } from "next/dist/shared/lib/app-router-context"; +import { ScopedMutator } from "swr/_internal"; + +export interface CardProps { + connector: Connector<{}> | null | undefined; + userCredentials: Credential[] | null | undefined; + setPopup: (popup: PopupSpec | null) => void; + router: AppRouterInstance; + mutate: ScopedMutator; +} diff --git a/web/src/app/user/connectors/page.tsx b/web/src/app/user/connectors/page.tsx new file mode 100644 index 000000000000..b91262df9adc --- /dev/null +++ b/web/src/app/user/connectors/page.tsx @@ -0,0 +1,136 @@ +"use client"; + +import { PlugIcon } from "@/components/icons/icons"; +import useSWR, { useSWRConfig } from "swr"; +import { fetcher } from "@/lib/fetcher"; +import { LoadingAnimation } from "@/components/Loading"; +import { useRouter } from "next/navigation"; +import { Popup, PopupSpec } from "@/components/admin/connectors/Popup"; +import { useState } from "react"; +import { HealthCheckBanner } from "@/components/health/healthcheck"; +import { Connector, Credential, ValidSources } from "@/lib/types"; +import { GoogleDriveCard } from "./GoogleDriveCard"; +import { CardProps } from "./interface"; + +const connectorSourceToConnectorCard = ( + source: ValidSources +): React.FC | null => { + switch (source) { + case "google_drive": + return GoogleDriveCard; + default: + return null; + } +}; + +const Main = () => { + const router = useRouter(); + const { mutate } = useSWRConfig(); + + const { + data: appCredentialData, + isLoading: isAppCredentialLoading, + error: isAppCredentialError, + } = useSWR<{ client_id: string }>( + "/api/manage/admin/connector/google-drive/app-credential", + fetcher + ); + const { + data: connectorsData, + isLoading: isConnectorDataLoading, + error: isConnectorDataError, + } = useSWR[]>("/api/manage/connector", fetcher); + const { + data: credentialsData, + isLoading: isCredentialsLoading, + error: isCredentialsError, + } = useSWR[]>("/api/manage/credential", fetcher); + + const [popup, setPopup] = useState<{ + message: string; + type: "success" | "error"; + } | null>(null); + const setPopupWithExpiration = (popupSpec: PopupSpec | null) => { + setPopup(popupSpec); + setTimeout(() => { + setPopup(null); + }, 4000); + }; + + if ( + isCredentialsLoading || + isAppCredentialLoading || + isConnectorDataLoading + ) { + return ( +
+ +
+ ); + } + + if (isCredentialsError || !credentialsData) { + return ( +
+
Failed to load credentials.
+
+ ); + } + + if (isConnectorDataError || !connectorsData) { + return ( +
+
Failed to load connectors.
+
+ ); + } + + if (isAppCredentialError || !appCredentialData) { + return ( +
+
+ Error loading Google Drive app credentials. Contact an administrator. +
+
+ ); + } + + return ( + <> + {popup && } + + {connectorsData.map((connector) => { + const connectorCard = connectorSourceToConnectorCard(connector.source); + if (connectorCard) { + return ( +
+ {connectorCard({ + connector, + userCredentials: credentialsData, + setPopup: setPopupWithExpiration, + router, + mutate, + })} +
+ ); + } + })} + + ); +}; + +export default function Page() { + return ( +
+
+ +
+
+ +

Personal Connectors

+
+ +
+
+ ); +} diff --git a/web/src/app/user/layout.tsx b/web/src/app/user/layout.tsx new file mode 100644 index 000000000000..7991f26be847 --- /dev/null +++ b/web/src/app/user/layout.tsx @@ -0,0 +1,42 @@ +import { Header } from "@/components/Header"; +import { Sidebar } from "@/components/admin/connectors/Sidebar"; +import { + NotebookIcon, + GithubIcon, + GlobeIcon, + GoogleDriveIcon, + SlackIcon, + KeyIcon, + ConfluenceIcon, +} from "@/components/icons/icons"; +import { DISABLE_AUTH } from "@/lib/constants"; +import { getCurrentUserSS } from "@/lib/userSS"; +import { redirect } from "next/navigation"; + +export default async function AdminLayout({ + children, +}: { + children: React.ReactNode; +}) { + let user = null; + if (!DISABLE_AUTH) { + user = await getCurrentUserSS(); + if (!user) { + return redirect("/auth/login"); + } + if (user.role !== "admin") { + return redirect("/"); + } + } + + return ( +
+
+
+
+ {children} +
+
+
+ ); +} diff --git a/web/src/components/Button.tsx b/web/src/components/Button.tsx index 7bb8a0dc9adc..1480d911dbfd 100644 --- a/web/src/components/Button.tsx +++ b/web/src/components/Button.tsx @@ -2,13 +2,20 @@ interface Props { onClick: () => void; children: JSX.Element | string; disabled?: boolean; + fullWidth?: boolean; } -export const Button = ({ onClick, children, disabled = false }: Props) => { +export const Button = ({ + onClick, + children, + disabled = false, + fullWidth = false, +}: Props) => { return ( -
- - )} - - - ); -} diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx index ee5b923830e5..3f367d0e60b9 100644 --- a/web/src/components/icons/icons.tsx +++ b/web/src/components/icons/icons.tsx @@ -8,6 +8,7 @@ import { XSquare, LinkBreak, Link, + Plug, } from "@phosphor-icons/react"; import { SiConfluence, SiGithub, SiGoogledrive, SiSlack } from "react-icons/si"; import { FaGlobe } from "react-icons/fa"; @@ -19,6 +20,13 @@ interface IconProps { const defaultTailwindCSS = "text-blue-400 my-auto flex flex-shrink-0"; +export const PlugIcon = ({ + size = "16", + className = defaultTailwindCSS, +}: IconProps) => { + return ; +}; + export const NotebookIcon = ({ size = "16", className = defaultTailwindCSS, diff --git a/web/src/components/openai/ApiKeyModal.tsx b/web/src/components/openai/ApiKeyModal.tsx index 20897a11fd9b..5fc08501b2b1 100644 --- a/web/src/components/openai/ApiKeyModal.tsx +++ b/web/src/components/openai/ApiKeyModal.tsx @@ -7,7 +7,7 @@ export const ApiKeyModal = () => { const [isOpen, setIsOpen] = useState(false); useEffect(() => { - fetch("/api/admin/openai-api-key/validate", { + fetch("/api/manage/admin/openai-api-key/validate", { method: "HEAD", }).then((res) => { // show popup if either the API key is not set or the API key is invalid diff --git a/web/src/components/openai/constants.ts b/web/src/components/openai/constants.ts index 6562b1fe0189..581ed69d2884 100644 --- a/web/src/components/openai/constants.ts +++ b/web/src/components/openai/constants.ts @@ -1 +1 @@ -export const OPENAI_API_KEY_URL = "/api/admin/openai-api-key"; +export const OPENAI_API_KEY_URL = "/api/manage/admin/openai-api-key"; diff --git a/web/src/lib/connector.ts b/web/src/lib/connector.ts index df5f97b61ee4..a55fbe4b64b2 100644 --- a/web/src/lib/connector.ts +++ b/web/src/lib/connector.ts @@ -3,7 +3,7 @@ import { Connector, ConnectorBase } from "./types"; export async function createConnector( connector: ConnectorBase ): Promise> { - const response = await fetch(`/api/admin/connector`, { + const response = await fetch(`/api/manage/admin/connector`, { method: "POST", headers: { "Content-Type": "application/json", @@ -16,7 +16,7 @@ export async function createConnector( export async function updateConnector( connector: Connector ): Promise> { - const response = await fetch(`/api/admin/connector/${connector.id}`, { + const response = await fetch(`/api/manage/admin/connector/${connector.id}`, { method: "PATCH", headers: { "Content-Type": "application/json", @@ -29,7 +29,7 @@ export async function updateConnector( export async function deleteConnector( connectorId: number ): Promise> { - const response = await fetch(`/api/admin/connector/${connectorId}`, { + const response = await fetch(`/api/manage/admin/connector/${connectorId}`, { method: "DELETE", headers: { "Content-Type": "application/json", diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts index 1c54c72690b7..cdd43d7f264d 100644 --- a/web/src/lib/constants.ts +++ b/web/src/lib/constants.ts @@ -1,2 +1,5 @@ export const DISABLE_AUTH = process.env.DISABLE_AUTH?.toLowerCase() === "true"; export const INTERNAL_URL = process.env.INTERNAL_URL || "http://127.0.0.1:8080"; + +export const GOOGLE_DRIVE_AUTH_IS_ADMIN_COOKIE_NAME = + "google_drive_auth_is_admin"; diff --git a/web/src/lib/credential.ts b/web/src/lib/credential.ts index 3c95b1135757..41a32085a2b4 100644 --- a/web/src/lib/credential.ts +++ b/web/src/lib/credential.ts @@ -1,5 +1,5 @@ export async function deleteCredential(credentialId: number) { - const response = await fetch(`/api/admin/credential/${credentialId}`, { + const response = await fetch(`/api/manage/credential/${credentialId}`, { method: "DELETE", headers: { "Content-Type": "application/json", @@ -13,7 +13,7 @@ export async function linkCredential( credentialId: number ) { const response = await fetch( - `/api/admin/connector/${connectorId}/credential/${credentialId}`, + `/api/manage/connector/${connectorId}/credential/${credentialId}`, { method: "PUT", headers: { diff --git a/web/src/lib/googleDrive.ts b/web/src/lib/googleDrive.ts new file mode 100644 index 000000000000..1ab205ff66b1 --- /dev/null +++ b/web/src/lib/googleDrive.ts @@ -0,0 +1,43 @@ +import { Credential } from "@/lib/types"; + +interface SetupGoogleDriveArgs { + isPublic: boolean; +} + +export const setupGoogleDriveOAuth = async ({ + isPublic, +}: SetupGoogleDriveArgs): Promise<[string | null, string]> => { + const credentialCreationResponse = await fetch("/api/manage/credential", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + public_doc: isPublic, + credential_json: {}, + }), + }); + if (!credentialCreationResponse.ok) { + return [ + null, + `Failed to create credential - ${credentialCreationResponse.status}`, + ]; + } + const credential = + (await credentialCreationResponse.json()) as Credential<{}>; + + const authorizationUrlResponse = await fetch( + `/api/manage/connector/google-drive/authorize/${credential.id}` + ); + if (!authorizationUrlResponse.ok) { + return [ + null, + `Failed to create credential - ${authorizationUrlResponse.status}`, + ]; + } + const authorizationUrlJson = (await authorizationUrlResponse.json()) as { + auth_url: string; + }; + + return [authorizationUrlJson.auth_url, ""]; +}; diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 3fc8208f6547..0a821defba17 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -51,6 +51,8 @@ export interface SlackConfig { export interface ConnectorIndexingStatus { connector: Connector; + public_doc: boolean; + owner: string; last_status: "success" | "failed" | "in_progress" | "not_started"; last_success: string | null; docs_indexed: number;