diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py index b8acd3adff..5f06a4acc0 100644 --- a/backend/danswer/configs/constants.py +++ b/backend/danswer/configs/constants.py @@ -93,6 +93,7 @@ class DocumentSource(str, Enum): GOOGLE_SITES = "google_sites" ZENDESK = "zendesk" LOOPIO = "loopio" + DROPBOX = "dropbox" SHAREPOINT = "sharepoint" DISCOURSE = "discourse" AXERO = "axero" diff --git a/backend/danswer/connectors/dropbox/__init__.py b/backend/danswer/connectors/dropbox/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/danswer/connectors/dropbox/connector.py b/backend/danswer/connectors/dropbox/connector.py new file mode 100644 index 0000000000..2fd39948aa --- /dev/null +++ b/backend/danswer/connectors/dropbox/connector.py @@ -0,0 +1,151 @@ +from datetime import timezone +from io import BytesIO +from typing import Any + +from dropbox import Dropbox # type: ignore +from dropbox.exceptions import ApiError # type:ignore +from dropbox.files import FileMetadata # type:ignore +from dropbox.files import FolderMetadata # type:ignore + +from danswer.configs.app_configs import INDEX_BATCH_SIZE +from danswer.configs.constants import DocumentSource +from danswer.connectors.interfaces import GenerateDocumentsOutput +from danswer.connectors.interfaces import LoadConnector +from danswer.connectors.interfaces import PollConnector +from danswer.connectors.interfaces import SecondsSinceUnixEpoch +from danswer.connectors.models import ConnectorMissingCredentialError +from danswer.connectors.models import Document +from danswer.connectors.models import Section +from danswer.file_processing.extract_file_text import extract_file_text +from danswer.utils.logger import setup_logger + + +logger = setup_logger() + + +class DropboxConnector(LoadConnector, PollConnector): + def __init__(self, batch_size: int = INDEX_BATCH_SIZE) -> None: + self.batch_size = batch_size + self.dropbox_client: Dropbox | None = None + + def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None: + self.dropbox_client = Dropbox(credentials["dropbox_access_token"]) + return None + + def _download_file(self, path: str) -> bytes: + """Download a single file from Dropbox.""" + if self.dropbox_client is None: + raise ConnectorMissingCredentialError("Dropbox") + _, resp = self.dropbox_client.files_download(path) + return resp.content + + def _get_shared_link(self, path: str) -> str: + """Create a shared link for a file in Dropbox.""" + if self.dropbox_client is None: + raise ConnectorMissingCredentialError("Dropbox") + + try: + # Check if a shared link already exists + shared_links = self.dropbox_client.sharing_list_shared_links(path=path) + if shared_links.links: + return shared_links.links[0].url + + link_metadata = ( + self.dropbox_client.sharing_create_shared_link_with_settings(path) + ) + return link_metadata.url + except ApiError as err: + logger.exception(f"Failed to create a shared link for {path}: {err}") + return "" + + def _yield_files_recursive( + self, + path: str, + start: SecondsSinceUnixEpoch | None, + end: SecondsSinceUnixEpoch | None, + ) -> GenerateDocumentsOutput: + """Yield files in batches from a specified Dropbox folder, including subfolders.""" + if self.dropbox_client is None: + raise ConnectorMissingCredentialError("Dropbox") + + result = self.dropbox_client.files_list_folder( + path, + limit=self.batch_size, + recursive=False, + include_non_downloadable_files=False, + ) + + while True: + batch: list[Document] = [] + for entry in result.entries: + if isinstance(entry, FileMetadata): + modified_time = entry.client_modified + if modified_time.tzinfo is None: + # If no timezone info, assume it is UTC + modified_time = modified_time.replace(tzinfo=timezone.utc) + else: + # If not in UTC, translate it + modified_time = modified_time.astimezone(timezone.utc) + + time_as_seconds = int(modified_time.timestamp()) + if start and time_as_seconds < start: + continue + if end and time_as_seconds > end: + continue + + downloaded_file = self._download_file(entry.path_display) + link = self._get_shared_link(entry.path_display) + try: + text = extract_file_text(entry.name, BytesIO(downloaded_file)) + batch.append( + Document( + id=f"doc:{entry.id}", + sections=[Section(link=link, text=text)], + source=DocumentSource.DROPBOX, + semantic_identifier=entry.name, + doc_updated_at=modified_time, + metadata={"type": "article"}, + ) + ) + except Exception as e: + logger.exception( + f"Error decoding file {entry.path_display} as utf-8 error occurred: {e}" + ) + + elif isinstance(entry, FolderMetadata): + yield from self._yield_files_recursive(entry.path_lower, start, end) + + if batch: + yield batch + + if not result.has_more: + break + + result = self.dropbox_client.files_list_folder_continue(result.cursor) + + def load_from_state(self) -> GenerateDocumentsOutput: + return self.poll_source(None, None) + + def poll_source( + self, start: SecondsSinceUnixEpoch | None, end: SecondsSinceUnixEpoch | None + ) -> GenerateDocumentsOutput: + if self.dropbox_client is None: + raise ConnectorMissingCredentialError("Dropbox") + + for batch in self._yield_files_recursive("", start, end): + yield batch + + return None + + +if __name__ == "__main__": + import os + + connector = DropboxConnector() + connector.load_credentials( + { + "dropbox_access_token": os.environ["DROPBOX_ACCESS_TOKEN"], + } + ) + document_batches = connector.load_from_state() + print(next(document_batches)) diff --git a/backend/danswer/connectors/factory.py b/backend/danswer/connectors/factory.py index f68294be1a..4873147e4f 100644 --- a/backend/danswer/connectors/factory.py +++ b/backend/danswer/connectors/factory.py @@ -8,6 +8,7 @@ from danswer.connectors.confluence.connector import ConfluenceConnector from danswer.connectors.danswer_jira.connector import JiraConnector from danswer.connectors.discourse.connector import DiscourseConnector from danswer.connectors.document360.connector import Document360Connector +from danswer.connectors.dropbox.connector import DropboxConnector from danswer.connectors.file.connector import LocalFileConnector from danswer.connectors.github.connector import GithubConnector from danswer.connectors.gitlab.connector import GitlabConnector @@ -73,6 +74,7 @@ def identify_connector_class( DocumentSource.GOOGLE_SITES: GoogleSitesConnector, DocumentSource.ZENDESK: ZendeskConnector, DocumentSource.LOOPIO: LoopioConnector, + DocumentSource.DROPBOX: DropboxConnector, DocumentSource.SHAREPOINT: SharepointConnector, DocumentSource.DISCOURSE: DiscourseConnector, DocumentSource.AXERO: AxeroConnector, diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt index 6052624ad2..e25f02b7a9 100644 --- a/backend/requirements/default.txt +++ b/backend/requirements/default.txt @@ -69,3 +69,4 @@ uvicorn==0.21.1 zulip==0.8.2 hubspot-api-client==8.1.0 zenpy==2.0.41 +dropbox==11.36.2 diff --git a/web/public/Dropbox.png b/web/public/Dropbox.png new file mode 100644 index 0000000000..cd83e09eb6 Binary files /dev/null and b/web/public/Dropbox.png differ diff --git a/web/src/app/admin/connectors/dropbox/page.tsx b/web/src/app/admin/connectors/dropbox/page.tsx new file mode 100644 index 0000000000..a42c8d141f --- /dev/null +++ b/web/src/app/admin/connectors/dropbox/page.tsx @@ -0,0 +1,209 @@ +"use client"; + +import { AdminPageTitle } from "@/components/admin/Title"; +import { HealthCheckBanner } from "@/components/health/healthcheck"; +import { DropboxIcon } from "@/components/icons/icons"; +import { LoadingAnimation } from "@/components/Loading"; +import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; +import { CredentialForm } from "@/components/admin/connectors/CredentialForm"; +import { TextFormField } from "@/components/admin/connectors/Field"; +import { usePopup } from "@/components/admin/connectors/Popup"; +import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable"; +import { TrashIcon } from "@/components/icons/icons"; +import { adminDeleteCredential, linkCredential } from "@/lib/credential"; +import { fetcher } from "@/lib/fetcher"; +import { usePublicCredentials } from "@/lib/hooks"; +import { + ConnectorIndexingStatus, + Credential, + DropboxConfig, + DropboxCredentialJson, +} from "@/lib/types"; +import { Card, Text, Title } from "@tremor/react"; +import useSWR, { useSWRConfig } from "swr"; +import * as Yup from "yup"; + +const Main = () => { + const { popup, setPopup } = usePopup(); + + const { mutate } = useSWRConfig(); + const { + data: connectorIndexingStatuses, + isLoading: isConnectorIndexingStatusesLoading, + error: isConnectorIndexingStatusesError, + } = useSWR[]>( + "/api/manage/admin/connector/indexing-status", + fetcher + ); + const { + data: credentialsData, + isLoading: isCredentialsLoading, + error: isCredentialsError, + refreshCredentials, + } = usePublicCredentials(); + + if ( + (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) || + (!credentialsData && isCredentialsLoading) + ) { + return ; + } + + if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) { + return
Failed to load connectors
; + } + + if (isCredentialsError || !credentialsData) { + return
Failed to load credentials
; + } + + const dropboxConnectorIndexingStatuses: ConnectorIndexingStatus< + DropboxConfig, + DropboxCredentialJson + >[] = connectorIndexingStatuses.filter( + (connectorIndexingStatus) => + connectorIndexingStatus.connector.source === "dropbox" + ); + const dropboxCredential: Credential | undefined = + credentialsData.find( + (credential) => credential.credential_json?.dropbox_access_token + ); + + return ( + <> + {popup} + + Provide your API details + + + {dropboxCredential ? ( + <> +
+

Existing API Token:

+

+ {dropboxCredential.credential_json?.dropbox_access_token} +

+ +
+ + ) : ( + <> + + See the Dropbox connector{" "} + + setup guide + {" "} + on the Danswer docs to obtain a Dropbox token. + + + + formBody={ + <> + + + } + validationSchema={Yup.object().shape({ + dropbox_access_token: Yup.string().required( + "Please enter your Dropbox API token" + ), + })} + initialValues={{ + dropbox_access_token: "", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + refreshCredentials(); + mutate("/api/manage/admin/connector/indexing-status"); + } + }} + /> + + + )} + + {dropboxConnectorIndexingStatuses.length > 0 && ( + <> + + Dropbox indexing status + + + The latest article changes are fetched every 10 minutes. + +
+ + connectorIndexingStatuses={dropboxConnectorIndexingStatuses} + liveCredential={dropboxCredential} + onCredentialLink={async (connectorId) => { + if (dropboxCredential) { + await linkCredential(connectorId, dropboxCredential.id); + mutate("/api/manage/admin/connector/indexing-status"); + } + }} + onUpdate={() => + mutate("/api/manage/admin/connector/indexing-status") + } + /> +
+ + )} + + {dropboxCredential && dropboxConnectorIndexingStatuses.length === 0 && ( + <> + +

Create Connection

+

+ Press connect below to start the connection to your Dropbox + instance. +

+ + nameBuilder={(values) => `Dropbox`} + ccPairNameBuilder={(values) => `Dropbox`} + source="dropbox" + inputType="poll" + formBody={<>} + validationSchema={Yup.object().shape({})} + initialValues={{}} + refreshFreq={10 * 60} // 10 minutes + credentialId={dropboxCredential.id} + /> + + + )} + + ); +}; + +export default function Page() { + return ( +
+
+ +
+ } title="Dropbox" /> +
+
+ ); +} diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx index 3a385c1afd..1fec3bb272 100644 --- a/web/src/app/chat/ChatPage.tsx +++ b/web/src/app/chat/ChatPage.tsx @@ -332,10 +332,10 @@ export function ChatPage({ (persona) => persona.id === existingChatSessionPersonaId ) : defaultSelectedPersonaId !== undefined - ? filteredAssistants.find( - (persona) => persona.id === defaultSelectedPersonaId - ) - : undefined + ? filteredAssistants.find( + (persona) => persona.id === defaultSelectedPersonaId + ) + : undefined ); const livePersona = selectedPersona || filteredAssistants[0] || availablePersonas[0]; diff --git a/web/src/app/chat/message/custom-code-styles.css b/web/src/app/chat/message/custom-code-styles.css index b7d419beba..30dee55400 100644 --- a/web/src/app/chat/message/custom-code-styles.css +++ b/web/src/app/chat/message/custom-code-styles.css @@ -21,9 +21,7 @@ pre[class*="language-"] { ::-webkit-scrollbar-thumb { background: #4b5563; /* Dark handle color */ border-radius: 10px; - transition: - background 0.3s ease, - box-shadow 0.3s ease; /* Smooth transition for hover effect */ + transition: background 0.3s ease, box-shadow 0.3s ease; /* Smooth transition for hover effect */ } ::-webkit-scrollbar-thumb:hover { diff --git a/web/src/app/chat/useDocumentSelection.ts b/web/src/app/chat/useDocumentSelection.ts index df33f13c3f..9e5fa3d2c4 100644 --- a/web/src/app/chat/useDocumentSelection.ts +++ b/web/src/app/chat/useDocumentSelection.ts @@ -21,7 +21,7 @@ export function useDocumentSelection(): [ DanswerDocument[], (document: DanswerDocument) => void, () => void, - number, + number ] { const [selectedDocuments, setSelectedDocuments] = useState( [] diff --git a/web/src/components/Bubble.tsx b/web/src/components/Bubble.tsx index 1b559ec240..8bf6f19771 100644 --- a/web/src/components/Bubble.tsx +++ b/web/src/components/Bubble.tsx @@ -27,8 +27,8 @@ export function Bubble({ (notSelectable ? " bg-background cursor-default" : isSelected - ? " bg-hover cursor-pointer" - : " bg-background hover:bg-hover-light cursor-pointer") + ? " bg-hover cursor-pointer" + : " bg-background hover:bg-hover-light cursor-pointer") } onClick={onClick} > diff --git a/web/src/components/admin/connectors/table/ConnectorsTable.tsx b/web/src/components/admin/connectors/table/ConnectorsTable.tsx index be9448bac8..1a84fdcd78 100644 --- a/web/src/components/admin/connectors/table/ConnectorsTable.tsx +++ b/web/src/components/admin/connectors/table/ConnectorsTable.tsx @@ -87,7 +87,7 @@ export function StatusRow({ export interface ColumnSpecification< ConnectorConfigType, - ConnectorCredentialType, + ConnectorCredentialType > { header: string; key: string; @@ -101,7 +101,7 @@ export interface ColumnSpecification< export interface ConnectorsTableProps< ConnectorConfigType, - ConnectorCredentialType, + ConnectorCredentialType > { connectorIndexingStatuses: ConnectorIndexingStatus< ConnectorConfigType, diff --git a/web/src/components/admin/connectors/table/SingleUseConnectorsTable.tsx b/web/src/components/admin/connectors/table/SingleUseConnectorsTable.tsx index f1308a9950..47d8483579 100644 --- a/web/src/components/admin/connectors/table/SingleUseConnectorsTable.tsx +++ b/web/src/components/admin/connectors/table/SingleUseConnectorsTable.tsx @@ -46,7 +46,7 @@ const SingleUseConnectorStatus = ({ export function SingleUseConnectorsTable< ConnectorConfigType, - ConnectorCredentialType, + ConnectorCredentialType >({ connectorIndexingStatuses, liveCredential, diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx index 2a65feb08a..2c9928e314 100644 --- a/web/src/components/icons/icons.tsx +++ b/web/src/components/icons/icons.tsx @@ -51,6 +51,7 @@ import hubSpotIcon from "../../../public/HubSpot.png"; import document360Icon from "../../../public/Document360.png"; import googleSitesIcon from "../../../public/GoogleSites.png"; import zendeskIcon from "../../../public/Zendesk.svg"; +import dropboxIcon from "../../../public/Dropbox.png"; import sharepointIcon from "../../../public/Sharepoint.png"; import mediawikiIcon from "../../../public/MediaWiki.svg"; import wikipediaIcon from "../../../public/Wikipedia.svg"; @@ -604,6 +605,18 @@ export const ZendeskIcon = ({ ); +export const DropboxIcon = ({ + size = 16, + className = defaultTailwindCSS, +}: IconProps) => ( +
+ Logo +
+); + export const DiscourseIcon = ({ size = 16, className = defaultTailwindCSS, diff --git a/web/src/lib/assistants/fetchPersonaEditorInfoSS.ts b/web/src/lib/assistants/fetchPersonaEditorInfoSS.ts index b98564bee3..17484cd1b3 100644 --- a/web/src/lib/assistants/fetchPersonaEditorInfoSS.ts +++ b/web/src/lib/assistants/fetchPersonaEditorInfoSS.ts @@ -18,7 +18,7 @@ export async function fetchAssistantEditorInfoSS( existingPersona: Persona | null; tools: ToolSnapshot[]; }, - null, + null ] | [null, string] > { @@ -50,7 +50,7 @@ export async function fetchAssistantEditorInfoSS( Response, User | null, ToolSnapshot[] | null, - Response | null, + Response | null ]; if (!ccPairsInfoResponse.ok) { diff --git a/web/src/lib/sources.ts b/web/src/lib/sources.ts index 8e0820161f..af173e614e 100644 --- a/web/src/lib/sources.ts +++ b/web/src/lib/sources.ts @@ -4,6 +4,7 @@ import { ConfluenceIcon, DiscourseIcon, Document360Icon, + DropboxIcon, FileIcon, GithubIcon, GitlabIcon, @@ -153,6 +154,11 @@ const SOURCE_METADATA_MAP: SourceMap = { displayName: "Loopio", category: SourceCategory.AppConnection, }, + dropbox: { + icon: DropboxIcon, + displayName: "Dropbox", + category: SourceCategory.AppConnection, + }, sharepoint: { icon: SharepointIcon, displayName: "Sharepoint", diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index cd9ded58ad..55111b91d6 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -42,6 +42,7 @@ export type ValidSources = | "file" | "google_sites" | "loopio" + | "dropbox" | "sharepoint" | "zendesk" | "discourse" @@ -186,6 +187,8 @@ export interface GoogleSitesConfig { export interface ZendeskConfig {} +export interface DropboxConfig {} + export interface MediaWikiBaseConfig { connector_name: string; language_code: string; @@ -193,6 +196,7 @@ export interface MediaWikiBaseConfig { pages?: string[]; recurse_depth?: number; } + export interface MediaWikiConfig extends MediaWikiBaseConfig { hostname: string; } @@ -213,7 +217,7 @@ export interface IndexAttemptSnapshot { export interface ConnectorIndexingStatus< ConnectorConfigType, - ConnectorCredentialType, + ConnectorCredentialType > { cc_pair_id: number; name: string | null; @@ -357,6 +361,10 @@ export interface ZendeskCredentialJson { zendesk_token: string; } +export interface DropboxCredentialJson { + dropbox_access_token: string; +} + export interface SharepointCredentialJson { aad_client_id: string; aad_client_secret: string;