highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-06-05 20:49:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

Small improvements to connector UI (#4559)

Browse Source 
* Small improvements to connector UI

* Update web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* Fix last_permission_sync

* Handle cases where a source doesn't need group sync

* fix

---------

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
This commit is contained in:
Chris Weaver 2025-04-19 12:14:05 -07:00 committed by GitHub
parent 4b8ef4b151
commit 7f99c54527
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 117 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
backend/ee/onyx/external_permissions/sync_params.py
View File

@ -50,6 +50,12 @@ DOC_PERMISSIONS_FUNC_MAP: dict[DocumentSource, DocSyncFuncType] = {
DocumentSource.GMAIL: gmail_doc_sync,
}
def source_requires_doc_sync(source: DocumentSource) -> bool:
"""Checks if the given DocumentSource requires doc syncing."""
return source in DOC_PERMISSIONS_FUNC_MAP
# These functions update:
# - the user_email <-> external_user_group_id mapping
# in postgres without committing
@ -61,6 +67,11 @@ GROUP_PERMISSIONS_FUNC_MAP: dict[DocumentSource, GroupSyncFuncType] = {
}
def source_requires_external_group_sync(source: DocumentSource) -> bool:
"""Checks if the given DocumentSource requires external group syncing."""
return source in GROUP_PERMISSIONS_FUNC_MAP
GROUP_PERMISSIONS_IS_CC_PAIR_AGNOSTIC: set[DocumentSource] = {
DocumentSource.CONFLUENCE,
}

8
backend/onyx/background/indexing/run_indexing.py
View File

@ -407,8 +407,12 @@ def _run_indexing(
# don't use a checkpoint if we're explicitly indexing from
# the beginning in order to avoid weird interactions between
# checkpointing / failure handling.
if index_attempt.from_beginning:
# checkpointing / failure handling
# OR
# if the last attempt was successful
if index_attempt.from_beginning or (
most_recent_attempt and most_recent_attempt.status.is_successful()
):
checkpoint = connector_runner.connector.build_dummy_checkpoint()
else:
checkpoint = get_latest_valid_checkpoint(

47
backend/onyx/server/documents/models.py
View File

@ -1,5 +1,6 @@
from datetime import datetime
from datetime import timezone
from datetime import UTC
from typing import Any
from typing import Generic
from typing import TypeVar
@ -25,6 +26,7 @@ from onyx.db.models import TaskStatus
from onyx.server.models import FullUserSnapshot
from onyx.server.models import InvitedUserSnapshot
from onyx.server.utils import mask_credential_dict
from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop
class DocumentSyncStatus(BaseModel):
@ -227,10 +229,49 @@ class CCPairFullInfo(BaseModel):
# information on syncing/indexing
last_indexed: datetime | None
last_pruned: datetime | None
last_permission_sync: datetime | None
# accounts for both doc sync and group sync
last_full_permission_sync: datetime | None
overall_indexing_speed: float | None
latest_checkpoint_description: str | None
@classmethod
def _get_last_full_permission_sync(
cls, cc_pair_model: ConnectorCredentialPair
) -> datetime | None:
check_if_source_requires_external_group_sync = fetch_ee_implementation_or_noop(
"onyx.external_permissions.sync_params",
"source_requires_external_group_sync",
noop_return_value=False,
)
check_if_source_requires_doc_sync = fetch_ee_implementation_or_noop(
"onyx.external_permissions.sync_params",
"source_requires_doc_sync",
noop_return_value=False,
)
needs_group_sync = check_if_source_requires_external_group_sync(
cc_pair_model.connector.source
)
needs_doc_sync = check_if_source_requires_doc_sync(
cc_pair_model.connector.source
)
last_group_sync = (
cc_pair_model.last_time_external_group_sync
if needs_group_sync
else datetime.now(UTC)
)
last_doc_sync = (
cc_pair_model.last_time_perm_sync if needs_doc_sync else datetime.now(UTC)
)
# if either is still None at this point, it means sync is necessary but
# has never completed.
if last_group_sync is None or last_doc_sync is None:
return None
return min(last_group_sync, last_doc_sync)
@classmethod
def from_models(
cls,
@ -292,9 +333,7 @@ class CCPairFullInfo(BaseModel):
last_index_attempt.time_started if last_index_attempt else None
),
last_pruned=cc_pair_model.last_pruned,
last_permission_sync=(
last_index_attempt.time_started if last_index_attempt else None
),
last_full_permission_sync=cls._get_last_full_permission_sync(cc_pair_model),
overall_indexing_speed=overall_indexing_speed,
latest_checkpoint_description=None,
)

17
web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx
View File

@ -24,7 +24,7 @@ function convertObjectToString(obj: any): string | any {
return obj;
}
function buildConfigEntries(
export function buildConfigEntries(
obj: any,
sourceType: ValidSources
): { [key: string]: string } {
@ -189,24 +189,15 @@ export function AdvancedConfigDisplay({
}
export function ConfigDisplay({
connectorSpecificConfig,
sourceType,
configEntries,
onEdit,
}: {
connectorSpecificConfig: any;
sourceType: ValidSources;
configEntries: { [key: string]: string };
onEdit?: (key: string) => void;
}) {
const configEntries = Object.entries(
buildConfigEntries(connectorSpecificConfig, sourceType)
);
if (!configEntries.length) {
return null;
}
return (
<ul className="w-full divide-y divide-background-200 dark:divide-background-700">
{configEntries.map(([key, value]) => (
{Object.entries(configEntries).map(([key, value]) => (
<ConfigItem
key={key}
label={key}

31
web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
View File

@ -25,6 +25,7 @@ import {
TooltipProvider,
TooltipTrigger,
} from "@/components/ui/tooltip";
import { FaBarsProgress } from "react-icons/fa6";
export interface IndexingAttemptsTableProps {
ccPair: CCPairFullInfo;
@ -137,7 +138,35 @@ export function IndexingAttemptsTable({
</div>
</div>
</TableCell>
<TableCell>{indexAttempt.total_docs_indexed}</TableCell>
<TableCell>
<div className="flex items-center">
{indexAttempt.total_docs_indexed}
{indexAttempt.from_beginning && (
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<span className="cursor-help flex items-center">
<FaBarsProgress className="ml-2 h-3.5 w-3.5" />
</span>
</TooltipTrigger>
<TooltipContent>
This index attempt{" "}
{indexAttempt.status === "in_progress" ||
indexAttempt.status === "not_started"
? "is"
: "was"}{" "}
a full re-index. All documents from the source{" "}
{indexAttempt.status === "in_progress" ||
indexAttempt.status === "not_started"
? "are being "
: "were "}
synced into the system.
</TooltipContent>
</Tooltip>
</TooltipProvider>
)}
</div>
</TableCell>
<TableCell>
<div>
{indexAttempt.status === "success" && (

19
web/src/app/admin/connector/[ccPairId]/page.tsx
View File

@ -17,7 +17,11 @@ import Title from "@/components/ui/title";
import { useRouter } from "next/navigation";
import { useCallback, useEffect, useState, use } from "react";
import useSWR, { mutate } from "swr";
import { AdvancedConfigDisplay, ConfigDisplay } from "./ConfigDisplay";
import {
AdvancedConfigDisplay,
buildConfigEntries,
ConfigDisplay,
} from "./ConfigDisplay";
import DeletionErrorStatus from "./DeletionErrorStatus";
import { IndexingAttemptsTable } from "./IndexingAttemptsTable";
@ -648,7 +652,7 @@ function Main({ ccPairId }: { ccPairId: number }) {
Last Permission Synced
</div>
<div className="text-sm text-text-default">
{timeAgo(ccPair.last_permission_sync) ?? "-"}
{timeAgo(ccPair.last_full_permission_sync) ?? "-"}
</div>
</div>
)}
@ -672,16 +676,23 @@ function Main({ ccPairId }: { ccPairId: number }) {
</>
)}
{ccPair.connector.connector_specific_config &&
ccPair.connector.connector_specific_config.length > 0 && (
<>
<Title size="md" className="mt-10 mb-2">
Connector Configuration
</Title>
<Card className="px-8 py-4">
<ConfigDisplay
connectorSpecificConfig={ccPair.connector.connector_specific_config}
sourceType={ccPair.connector.source}
configEntries={buildConfigEntries(
ccPair.connector.connector_specific_config,
ccPair.connector.source
)}
/>
</Card>
</>
)}
<div className="mt-6">
<div className="flex">

2
web/src/app/admin/connector/[ccPairId]/types.ts
View File

@ -49,7 +49,7 @@ export interface CCPairFullInfo {
last_indexed: string | null;
last_pruned: string | null;
last_permission_sync: string | null;
last_full_permission_sync: string | null;
overall_indexing_speed: number | null;
latest_checkpoint_description: string | null;
}