highperfocused/danswer
1
0
Fork 0
mirror of https://github.com/danswer-ai/danswer.git synced 2025-04-08 03:48:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/onyx-dot-app/onyx into feature/email-whitelabeling

Browse Source
This commit is contained in:
Richard Kuo (Danswer) 2025-03-11 14:15:54 -07:00
commit cad09a7927
23 changed files with 251 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
backend/model_server/encoders.py
View File

@ -62,6 +62,60 @@ _OPENAI_MAX_INPUT_LEN = 2048
# Cohere allows up to 96 embeddings in a single embedding calling
_COHERE_MAX_INPUT_LEN = 96
# Authentication error string constants
_AUTH_ERROR_401 = "401"
_AUTH_ERROR_UNAUTHORIZED = "unauthorized"
_AUTH_ERROR_INVALID_API_KEY = "invalid api key"
_AUTH_ERROR_PERMISSION = "permission"
def is_authentication_error(error: Exception) -> bool:
"""Check if an exception is related to authentication issues.
Args:
error: The exception to check
Returns:
bool: True if the error appears to be authentication-related
"""
error_str = str(error).lower()
return (
_AUTH_ERROR_401 in error_str
or _AUTH_ERROR_UNAUTHORIZED in error_str
or _AUTH_ERROR_INVALID_API_KEY in error_str
or _AUTH_ERROR_PERMISSION in error_str
)
def format_embedding_error(
error: Exception,
service_name: str,
model: str | None,
provider: EmbeddingProvider,
status_code: int | None = None,
) -> str:
"""
Format a standardized error string for embedding errors.
"""
detail = f"Status {status_code}" if status_code else f"{type(error)}"
return (
f"{'HTTP error' if status_code else 'Exception'} embedding text with {service_name} - {detail}: "
f"Model: {model} "
f"Provider: {provider} "
f"Exception: {error}"
)
# Custom exception for authentication errors
class AuthenticationError(Exception):
"""Raised when authentication fails with a provider."""
def __init__(self, provider: str, message: str = "API key is invalid or expired"):
self.provider = provider
self.message = message
super().__init__(f"{provider} authentication failed: {message}")
class CloudEmbedding:
def __init__(
@ -92,31 +146,17 @@ class CloudEmbedding:
)
final_embeddings: list[Embedding] = []
try:
for text_batch in batch_list(texts, _OPENAI_MAX_INPUT_LEN):
response = await client.embeddings.create(
input=text_batch,
model=model,
dimensions=reduced_dimension or openai.NOT_GIVEN,
)
final_embeddings.extend(
[embedding.embedding for embedding in response.data]
)
return final_embeddings
except Exception as e:
error_string = (
f"Exception embedding text with OpenAI - {type(e)}: "
f"Model: {model} "
f"Provider: {self.provider} "
f"Exception: {e}"
for text_batch in batch_list(texts, _OPENAI_MAX_INPUT_LEN):
response = await client.embeddings.create(
input=text_batch,
model=model,
dimensions=reduced_dimension or openai.NOT_GIVEN,
)
logger.error(error_string)
# only log text when it's not an authentication error.
if not isinstance(e, openai.AuthenticationError):
logger.debug(f"Exception texts: {texts}")
raise RuntimeError(error_string)
final_embeddings.extend(
[embedding.embedding for embedding in response.data]
)
return final_embeddings
async def _embed_cohere(
self, texts: list[str], model: str | None, embedding_type: str
@ -155,7 +195,6 @@ class CloudEmbedding:
input_type=embedding_type,
truncation=True,
)
return response.embeddings
async def _embed_azure(
@ -239,22 +278,51 @@ class CloudEmbedding:
deployment_name: str | None = None,
reduced_dimension: int | None = None,
) -> list[Embedding]:
if self.provider == EmbeddingProvider.OPENAI:
return await self._embed_openai(texts, model_name, reduced_dimension)
elif self.provider == EmbeddingProvider.AZURE:
return await self._embed_azure(texts, f"azure/{deployment_name}")
elif self.provider == EmbeddingProvider.LITELLM:
return await self._embed_litellm_proxy(texts, model_name)
try:
if self.provider == EmbeddingProvider.OPENAI:
return await self._embed_openai(texts, model_name, reduced_dimension)
elif self.provider == EmbeddingProvider.AZURE:
return await self._embed_azure(texts, f"azure/{deployment_name}")
elif self.provider == EmbeddingProvider.LITELLM:
return await self._embed_litellm_proxy(texts, model_name)
embedding_type = EmbeddingModelTextType.get_type(self.provider, text_type)
if self.provider == EmbeddingProvider.COHERE:
return await self._embed_cohere(texts, model_name, embedding_type)
elif self.provider == EmbeddingProvider.VOYAGE:
return await self._embed_voyage(texts, model_name, embedding_type)
elif self.provider == EmbeddingProvider.GOOGLE:
return await self._embed_vertex(texts, model_name, embedding_type)
else:
raise ValueError(f"Unsupported provider: {self.provider}")
embedding_type = EmbeddingModelTextType.get_type(self.provider, text_type)
if self.provider == EmbeddingProvider.COHERE:
return await self._embed_cohere(texts, model_name, embedding_type)
elif self.provider == EmbeddingProvider.VOYAGE:
return await self._embed_voyage(texts, model_name, embedding_type)
elif self.provider == EmbeddingProvider.GOOGLE:
return await self._embed_vertex(texts, model_name, embedding_type)
else:
raise ValueError(f"Unsupported provider: {self.provider}")
except openai.AuthenticationError:
raise AuthenticationError(provider="OpenAI")
except httpx.HTTPStatusError as e:
if e.response.status_code == 401:
raise AuthenticationError(provider=str(self.provider))
error_string = format_embedding_error(
e,
str(self.provider),
model_name or deployment_name,
self.provider,
status_code=e.response.status_code,
)
logger.error(error_string)
logger.debug(f"Exception texts: {texts}")
raise RuntimeError(error_string)
except Exception as e:
if is_authentication_error(e):
raise AuthenticationError(provider=str(self.provider))
error_string = format_embedding_error(
e, str(self.provider), model_name or deployment_name, self.provider
)
logger.error(error_string)
logger.debug(f"Exception texts: {texts}")
raise RuntimeError(error_string)
@staticmethod
def create(
@ -569,6 +637,13 @@ async def process_embed_request(
gpu_type=gpu_type,
)
return EmbedResponse(embeddings=embeddings)
except AuthenticationError as e:
# Handle authentication errors consistently
logger.error(f"Authentication error: {e.provider}")
raise HTTPException(
status_code=401,
detail=f"Authentication failed: {e.message}",
)
except RateLimitError as e:
raise HTTPException(
status_code=429,

2
backend/requirements/dev.txt
View File

@ -1,4 +1,4 @@
black==23.3.0
black==23.7.0
boto3-stubs[s3]==1.34.133
celery-types==0.19.0
cohere==5.6.1

33
backend/scripts/debugging/onyx_redis.py
View File

@ -54,6 +54,7 @@ class OnyxRedisCommand(Enum):
purge_vespa_syncing = "purge_vespa_syncing"
get_user_token = "get_user_token"
delete_user_token = "delete_user_token"
add_invited_user = "add_invited_user"
def __str__(self) -> str:
return self.value
@ -163,6 +164,21 @@ def onyx_redis(
return 0
else:
return 2
elif command == OnyxRedisCommand.add_invited_user:
if not user_email:
logger.error("You must specify --user-email with add_invited_user")
return 1
current_invited_users = get_invited_users()
if user_email not in current_invited_users:
current_invited_users.append(user_email)
if dry_run:
logger.info(f"(DRY-RUN) Would add {user_email} to invited users")
else:
write_invited_users(current_invited_users)
logger.info(f"Added {user_email} to invited users")
else:
logger.info(f"{user_email} is already in the invited users list")
return 0
else:
pass
@ -441,23 +457,6 @@ if __name__ == "__main__":
if args.tenant_id:
CURRENT_TENANT_ID_CONTEXTVAR.set(args.tenant_id)
if args.command == "add_invited_user":
if not args.user_email:
print("Error: --user-email is required for add_invited_user command")
sys.exit(1)
current_invited_users = get_invited_users()
if args.user_email not in current_invited_users:
current_invited_users.append(args.user_email)
if args.dry_run:
print(f"(DRY-RUN) Would add {args.user_email} to invited users")
else:
write_invited_users(current_invited_users)
print(f"Added {args.user_email} to invited users")
else:
print(f"{args.user_email} is already in the invited users list")
sys.exit(0)
exitcode = onyx_redis(
command=args.command,
batch=args.batch,

2
backend/tests/integration/mock_services/mock_connector_server/Dockerfile
View File

@ -2,7 +2,7 @@ FROM python:3.11.7-slim-bookworm
WORKDIR /app
RUN pip install fastapi uvicorn
RUN pip install "pydantic-core>=2.28.0" fastapi uvicorn
COPY ./main.py /app/main.py

2
web/src/app/admin/actions/ActionEditor.tsx
View File

@ -111,7 +111,7 @@ function ActionForm({
<TextFormField
name="definition"
label="Definition"
subtext="Specify an OpenAPI schema that defines the APIs you want to make available as part of this tool."
subtext="Specify an OpenAPI schema that defines the APIs you want to make available as part of this action."
placeholder="Enter your OpenAPI schema here"
isTextArea={true}
defaultHeight="h-96"

2
web/src/app/admin/actions/new/page.tsx
View File

@ -12,7 +12,7 @@ export default function NewToolPage() {
<BackButton />
<AdminPageTitle
title="Create Tool"
title="Create Action"
icon={<ToolIcon size={32} className="my-auto" />}
/>

4
web/src/app/admin/actions/page.tsx
View File

@ -29,7 +29,7 @@ export default async function Page() {
<div className="mx-auto container">
<AdminPageTitle
icon={<ToolIcon size={32} className="my-auto" />}
title="Tools"
title="Actions"
/>
<Text className="mb-2">
@ -40,7 +40,7 @@ export default async function Page() {
<Separator />
<Title>Create an Action</Title>
<CreateButton href="/admin/tools/new" text="New Tool" />
<CreateButton href="/admin/actions/new" text="New Action" />
<Separator />

2
web/src/app/chat/ChatBanner.tsx
View File

@ -3,7 +3,7 @@
import { SettingsContext } from "@/components/settings/SettingsProvider";
import { useContext, useState, useRef, useLayoutEffect } from "react";
import { ChevronDownIcon } from "@/components/icons/icons";
import { MinimalMarkdown } from "@/components/chat/MinimalMarkdown";
import MinimalMarkdown from "@/components/chat/MinimalMarkdown";
export function ChatBanner() {
const settings = useContext(SettingsContext);

2
web/src/app/chat/ChatPage.tsx
View File

@ -109,7 +109,6 @@ import {
} from "@/components/resizable/constants";
import FixedLogo from "../../components/logo/FixedLogo";
import { MinimalMarkdown } from "@/components/chat/MinimalMarkdown";
import ExceptionTraceModal from "@/components/modals/ExceptionTraceModal";
import {
@ -138,6 +137,7 @@ import { useSidebarShortcut } from "@/lib/browserUtilities";
import { ConfirmEntityModal } from "@/components/modals/ConfirmEntityModal";
import { ChatSearchModal } from "./chat_search/ChatSearchModal";
import { ErrorBanner } from "./message/Resubmit";
import MinimalMarkdown from "@/components/chat/MinimalMarkdown";
const TEMP_USER_MESSAGE_ID = -1;
const TEMP_ASSISTANT_MESSAGE_ID = -2;

38
web/src/app/chat/ChatPopup.tsx
View File

@ -6,6 +6,7 @@ import { Button } from "@/components/ui/button";
import { useContext, useEffect, useState } from "react";
import ReactMarkdown from "react-markdown";
import remarkGfm from "remark-gfm";
import { transformLinkUri } from "@/lib/utils";
const ALL_USERS_INITIAL_POPUP_FLOW_COMPLETED =
"allUsersInitialPopupFlowCompleted";
@ -44,23 +45,26 @@ export function ChatPopup() {
return (
<Modal width="w-3/6 xl:w-[700px]" title={popupTitle}>
<>
<ReactMarkdown
className="prose text-text-800 dark:text-neutral-100 max-w-full"
components={{
a: ({ node, ...props }) => (
<a
{...props}
className="text-link hover:text-link-hover"
target="_blank"
rel="noopener noreferrer"
/>
),
p: ({ node, ...props }) => <p {...props} className="text-sm" />,
}}
remarkPlugins={[remarkGfm]}
>
{popupContent}
</ReactMarkdown>
<div className="overflow-y-auto max-h-[90vh] py-8 px-4 text-left">
<ReactMarkdown
className="prose text-text-800 dark:text-neutral-100 max-w-full"
components={{
a: ({ node, ...props }) => (
<a
{...props}
className="text-link hover:text-link-hover"
target="_blank"
rel="noopener noreferrer"
/>
),
p: ({ node, ...props }) => <p {...props} className="text-sm" />,
}}
remarkPlugins={[remarkGfm]}
urlTransform={transformLinkUri}
>
{popupContent}
</ReactMarkdown>
</div>
{showConsentError && (
<p className="text-red-500 text-sm mt-2">

3
web/src/app/chat/message/AgenticMessage.tsx
View File

@ -53,6 +53,7 @@ import { copyAll, handleCopy } from "./copyingUtils";
import { Button } from "@/components/ui/button";
import { RefreshCw } from "lucide-react";
import { ErrorBanner, Resubmit } from "./Resubmit";
import { transformLinkUri } from "@/lib/utils";
export const AgenticMessage = ({
isStreamingQuestions,
@ -336,6 +337,7 @@ export const AgenticMessage = ({
}}
remarkPlugins={[remarkGfm, remarkMath]}
rehypePlugins={[[rehypePrism, { ignoreMissing: true }], rehypeKatex]}
urlTransform={transformLinkUri}
>
{finalAlternativeContent}
</ReactMarkdown>
@ -349,6 +351,7 @@ export const AgenticMessage = ({
components={markdownComponents}
remarkPlugins={[remarkGfm, remarkMath]}
rehypePlugins={[[rehypePrism, { ignoreMissing: true }], rehypeKatex]}
urlTransform={transformLinkUri}
>
{streamedContent +
(!isComplete && !secondLevelGenerating ? " [*]() " : "")}

5
web/src/app/chat/message/MemoizedTextComponents.tsx
View File

@ -160,8 +160,9 @@ export const MemoizedLink = memo(
const handleMouseDown = () => {
let url = href || rest.children?.toString();
if (url && !url.startsWith("http://") && !url.startsWith("https://")) {
// Try to construct a valid URL
if (url && !url.includes("://")) {
// Only add https:// if the URL doesn't already have a protocol
const httpsUrl = `https://${url}`;
try {
new URL(httpsUrl);

4
web/src/app/chat/message/Messages.tsx
View File

@ -71,6 +71,7 @@ import remarkMath from "remark-math";
import rehypeKatex from "rehype-katex";
import "katex/dist/katex.min.css";
import { copyAll, handleCopy } from "./copyingUtils";
import { transformLinkUri } from "@/lib/utils";
const TOOLS_WITH_CUSTOM_HANDLING = [
SEARCH_TOOL_NAME,
@ -348,7 +349,7 @@ export const AIMessage = ({
a: anchorCallback,
p: paragraphCallback,
b: ({ node, className, children }: any) => {
return <span className={className}>||||{children}</span>;
return <span className={className}>{children}</span>;
},
code: ({ node, className, children }: any) => {
const codeText = extractCodeText(
@ -381,6 +382,7 @@ export const AIMessage = ({
components={markdownComponents}
remarkPlugins={[remarkGfm, remarkMath]}
rehypePlugins={[[rehypePrism, { ignoreMissing: true }], rehypeKatex]}
urlTransform={transformLinkUri}
>
{finalContent}
</ReactMarkdown>

5
web/src/app/chat/message/SubQuestionsDisplay.tsx
View File

@ -16,15 +16,15 @@ import ReactMarkdown from "react-markdown";
import { MemoizedAnchor } from "./MemoizedTextComponents";
import { MemoizedParagraph } from "./MemoizedTextComponents";
import { extractCodeText, preprocessLaTeX } from "./codeUtils";
import remarkGfm from "remark-gfm";
import remarkMath from "remark-math";
import rehypeKatex from "rehype-katex";
import remarkGfm from "remark-gfm";
import { CodeBlock } from "./CodeBlock";
import { CheckIcon, ChevronDown } from "lucide-react";
import { PHASE_MIN_MS, useStreamingMessages } from "./StreamingMessages";
import { CirclingArrowIcon } from "@/components/icons/icons";
import { handleCopy } from "./copyingUtils";
import { transformLinkUri } from "@/lib/utils";
export const StatusIndicator = ({ status }: { status: ToggleState }) => {
return (
@ -301,6 +301,7 @@ const SubQuestionDisplay: React.FC<{
components={markdownComponents}
remarkPlugins={[remarkGfm, remarkMath]}
rehypePlugins={[rehypeKatex]}
urlTransform={transformLinkUri}
>
{finalContent}
</ReactMarkdown>

3
web/src/components/admin/connectors/Field.tsx
View File

@ -33,6 +33,8 @@ import Link from "next/link";
import { CheckboxField } from "@/components/ui/checkbox";
import { CheckedState } from "@radix-ui/react-checkbox";
import { transformLinkUri } from "@/lib/utils";
export function SectionHeader({
children,
}: {
@ -432,6 +434,7 @@ export const MarkdownFormField = ({
<ReactMarkdown
className="prose dark:prose-invert"
remarkPlugins={[remarkGfm]}
urlTransform={transformLinkUri}
>
{field.value}
</ReactMarkdown>

33
web/src/components/chat/MinimalMarkdown.tsx
View File

@ -4,19 +4,26 @@ import {
MemoizedLink,
MemoizedParagraph,
} from "@/app/chat/message/MemoizedTextComponents";
import React, { useMemo } from "react";
import React, { useMemo, CSSProperties } from "react";
import ReactMarkdown from "react-markdown";
import remarkGfm from "remark-gfm";
import rehypePrism from "rehype-prism-plus";
import remarkMath from "remark-math";
import rehypeKatex from "rehype-katex";
import "katex/dist/katex.min.css";
import { transformLinkUri } from "@/lib/utils";
interface MinimalMarkdownProps {
content: string;
className?: string;
style?: CSSProperties;
}
export const MinimalMarkdown: React.FC<MinimalMarkdownProps> = ({
export default function MinimalMarkdown({
content,
className = "",
}) => {
style,
}: MinimalMarkdownProps) {
const markdownComponents = useMemo(
() => ({
a: MemoizedLink,
@ -34,12 +41,16 @@ export const MinimalMarkdown: React.FC<MinimalMarkdownProps> = ({
);
return (
<ReactMarkdown
className={`w-full text-wrap break-word prose dark:prose-invert ${className}`}
components={markdownComponents}
remarkPlugins={[remarkGfm]}
>
{content}
</ReactMarkdown>
<div style={style || {}} className={`${className}`}>
<ReactMarkdown
className="prose dark:prose-invert max-w-full text-sm break-words"
components={markdownComponents}
rehypePlugins={[[rehypePrism, { ignoreMissing: true }], rehypeKatex]}
remarkPlugins={[remarkGfm, remarkMath]}
urlTransform={transformLinkUri}
>
{content}
</ReactMarkdown>
</div>
);
};
}

2
web/src/components/chat/TextView.tsx
View File

@ -10,7 +10,7 @@ import {
} from "@/components/ui/dialog";
import { Download, XIcon, ZoomIn, ZoomOut } from "lucide-react";
import { OnyxDocument } from "@/lib/search/interfaces";
import { MinimalMarkdown } from "./MinimalMarkdown";
import MinimalMarkdown from "@/components/chat/MinimalMarkdown";
interface TextViewProps {
presentingDocument: OnyxDocument;

3
web/src/components/context/ModalContext.tsx
View File

@ -4,6 +4,7 @@ import React, { createContext, useContext, useState, useCallback } from "react";
import { NewTeamModal } from "../modals/NewTeamModal";
import NewTenantModal from "../modals/NewTenantModal";
import { User, NewTenantInfo } from "@/lib/types";
import { NEXT_PUBLIC_CLOUD_ENABLED } from "@/lib/constants";
type ModalContextType = {
showNewTeamModal: boolean;
@ -48,7 +49,7 @@ export const ModalProvider: React.FC<{
// Render all application-wide modals
const renderModals = () => {
if (!user) return null;
if (!user || !NEXT_PUBLIC_CLOUD_ENABLED) return null;
return (
<>

2
web/src/components/search/results/AnswerSection.tsx
View File

@ -1,6 +1,6 @@
import { Quote } from "@/lib/search/interfaces";
import { ResponseSection, StatusOptions } from "./ResponseSection";
import { MinimalMarkdown } from "@/components/chat/MinimalMarkdown";
import MinimalMarkdown from "@/components/chat/MinimalMarkdown";
const TEMP_STRING = "__$%^TEMP$%^__";

14
web/src/lib/constants.ts
View File

@ -91,3 +91,17 @@ export const NEXT_PUBLIC_INCLUDE_ERROR_POPUP_SUPPORT_LINK =
export const NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY =
process.env.NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY;
// Add support for custom URL protocols in markdown links
export const ALLOWED_URL_PROTOCOLS = [
"http:",
"https:",
"mailto:",
"tel:",
"slack:",
"vscode:",
"file:",
"sms:",
"spotify:",
"zoommtg:",
];

26
web/src/lib/utils.ts
View File

@ -1,5 +1,6 @@
import { clsx, type ClassValue } from "clsx";
import { twMerge } from "tailwind-merge";
import { ALLOWED_URL_PROTOCOLS } from "./constants";
export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs));
@ -8,3 +9,28 @@ export function cn(...inputs: ClassValue[]) {
export const truncateString = (str: string, maxLength: number) => {
return str.length > maxLength ? str.slice(0, maxLength - 1) + "..." : str;
};
/**
* Custom URL transformer function for ReactMarkdown
* Allows specific protocols to be used in markdown links
* We use this with the urlTransform prop in ReactMarkdown
*/
export function transformLinkUri(href: string) {
if (!href) return href;
const url = href.trim();
try {
const parsedUrl = new URL(url);
if (
ALLOWED_URL_PROTOCOLS.some((protocol) =>
parsedUrl.protocol.startsWith(protocol)
)
) {
return url;
}
} catch (e) {
// If it's not a valid URL with protocol, return the original href
return href;
}
return href;
}

8
web/tests/e2e/auth/password_management.spec.ts
View File

@ -2,7 +2,10 @@ import { test, expect } from "@chromatic-com/playwright";
import { loginAsRandomUser, loginAs } from "../utils/auth";
import { TEST_ADMIN2_CREDENTIALS, TEST_ADMIN_CREDENTIALS } from "../constants";
test("User changes password and logs in with new password", async ({
// test("User changes password and logs in with new password", async ({
// Skip this test for now
test.skip("User changes password and logs in with new password", async ({
page,
}) => {
// Clear browser context before starting the test
@ -45,7 +48,8 @@ test("User changes password and logs in with new password", async ({
test.use({ storageState: "admin2_auth.json" });
test("Admin resets own password and logs in with new password", async ({
// Skip this test for now
test.skip("Admin resets own password and logs in with new password", async ({
page,
}) => {
const { email: adminEmail, password: adminPassword } =

8
web/tests/e2e/chromaticSnapshots.json
View File

@ -88,11 +88,11 @@
}
},
{
"name": "Custom Assistants - Tools",
"path": "tools",
"pageTitle": "Tools",
"name": "Custom Assistants - Actions",
"path": "actions",
"pageTitle": "Actions",
"options": {
"paragraphText": "Tools allow assistants to retrieve information or take actions."
"paragraphText": "Actions allow assistants to retrieve information or take actions."
}
},
{