mirror of
https://github.com/danswer-ai/danswer.git
synced 2025-05-31 10:10:21 +02:00
Latex formatting (#3499)
This commit is contained in:
pablonyx
GitHub
parent
b458d504af
commit
59c774353a
214
web/package-lock.json
generated
214
web/package-lock.json
generated
@ -40,6 +40,7 @@
|
||||
"favicon-fetch": "^1.0.0",
|
||||
"formik": "^2.2.9",
|
||||
"js-cookie": "^3.0.5",
|
||||
"katex": "^0.16.17",
|
||||
"lodash": "^4.17.21",
|
||||
"lucide-react": "^0.454.0",
|
||||
"mdast-util-find-and-replace": "^3.0.1",
|
||||
@ -57,8 +58,10 @@
|
||||
"react-markdown": "^9.0.1",
|
||||
"react-select": "^5.8.0",
|
||||
"recharts": "^2.13.1",
|
||||
"rehype-katex": "^7.0.1",
|
||||
"rehype-prism-plus": "^2.0.0",
|
||||
"remark-gfm": "^4.0.0",
|
||||
"remark-math": "^6.0.0",
|
||||
"semver": "^7.5.4",
|
||||
"sharp": "^0.33.5",
|
||||
"stripe": "^17.0.0",
|
||||
@ -4775,6 +4778,12 @@
|
||||
"integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/katex": {
|
||||
"version": "0.16.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.7.tgz",
|
||||
"integrity": "sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/lodash": {
|
||||
"version": "4.17.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.4.tgz",
|
||||
@ -8116,6 +8125,51 @@
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/hast-util-from-dom": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/hast-util-from-dom/-/hast-util-from-dom-5.0.1.tgz",
|
||||
"integrity": "sha512-N+LqofjR2zuzTjCPzyDUdSshy4Ma6li7p/c3pA78uTwzFgENbgbUrm2ugwsOdcjI1muO+o6Dgzp9p8WHtn/39Q==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@types/hast": "^3.0.0",
|
||||
"hastscript": "^9.0.0",
|
||||
"web-namespaces": "^2.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/hast-util-from-dom/node_modules/hast-util-parse-selector": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz",
|
||||
"integrity": "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/hast": "^3.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/hast-util-from-dom/node_modules/hastscript": {
|
||||
"version": "9.0.0",
|
||||
"resolved": "https://registry.npmjs.org/hastscript/-/hastscript-9.0.0.tgz",
|
||||
"integrity": "sha512-jzaLBGavEDKHrc5EfFImKN7nZKKBdSLIdGvCwDZ9TfzbF2ffXiov8CKE445L2Z1Ek2t/m4SKQ2j6Ipv7NyUolw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/hast": "^3.0.0",
|
||||
"comma-separated-tokens": "^2.0.0",
|
||||
"hast-util-parse-selector": "^4.0.0",
|
||||
"property-information": "^6.0.0",
|
||||
"space-separated-tokens": "^2.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/hast-util-from-html": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/hast-util-from-html/-/hast-util-from-html-2.0.1.tgz",
|
||||
@ -8133,6 +8187,22 @@
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/hast-util-from-html-isomorphic": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/hast-util-from-html-isomorphic/-/hast-util-from-html-isomorphic-2.0.0.tgz",
|
||||
"integrity": "sha512-zJfpXq44yff2hmE0XmwEOzdWin5xwH+QIhMLOScpX91e/NSGPsAzNCvLQDIEPyO2TXi+lBmU6hjLIhV8MwP2kw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/hast": "^3.0.0",
|
||||
"hast-util-from-dom": "^5.0.0",
|
||||
"hast-util-from-html": "^2.0.0",
|
||||
"unist-util-remove-position": "^5.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/hast-util-from-parse5": {
|
||||
"version": "8.0.1",
|
||||
"resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-8.0.1.tgz",
|
||||
@ -8180,6 +8250,19 @@
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/hast-util-is-element": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-3.0.0.tgz",
|
||||
"integrity": "sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/hast": "^3.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/hast-util-parse-selector": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-3.1.1.tgz",
|
||||
@ -8243,6 +8326,22 @@
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/hast-util-to-text": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-4.0.2.tgz",
|
||||
"integrity": "sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/hast": "^3.0.0",
|
||||
"@types/unist": "^3.0.0",
|
||||
"hast-util-is-element": "^3.0.0",
|
||||
"unist-util-find-after": "^5.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/hast-util-whitespace": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz",
|
||||
@ -9226,6 +9325,31 @@
|
||||
"node": ">=4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/katex": {
|
||||
"version": "0.16.17",
|
||||
"resolved": "https://registry.npmjs.org/katex/-/katex-0.16.17.tgz",
|
||||
"integrity": "sha512-OyzSrXBllz+Jdc9Auiw0kt21gbZ4hkz8Q5srVAb2U9INcYIfGKbxe+bvNvEz1bQ/NrDeRRho5eLCyk/L03maAw==",
|
||||
"funding": [
|
||||
"https://opencollective.com/katex",
|
||||
"https://github.com/sponsors/katex"
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"commander": "^8.3.0"
|
||||
},
|
||||
"bin": {
|
||||
"katex": "cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/katex/node_modules/commander": {
|
||||
"version": "8.3.0",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz",
|
||||
"integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 12"
|
||||
}
|
||||
},
|
||||
"node_modules/keyv": {
|
||||
"version": "4.5.4",
|
||||
"resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz",
|
||||
@ -9559,6 +9683,25 @@
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/mdast-util-math": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/mdast-util-math/-/mdast-util-math-3.0.0.tgz",
|
||||
"integrity": "sha512-Tl9GBNeG/AhJnQM221bJR2HPvLOSnLE/T9cJI9tlc6zwQk2nPk/4f0cHkOdEixQPC/j8UtKDdITswvLAy1OZ1w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/hast": "^3.0.0",
|
||||
"@types/mdast": "^4.0.0",
|
||||
"devlop": "^1.0.0",
|
||||
"longest-streak": "^3.0.0",
|
||||
"mdast-util-from-markdown": "^2.0.0",
|
||||
"mdast-util-to-markdown": "^2.1.0",
|
||||
"unist-util-remove-position": "^5.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/mdast-util-mdx-expression": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.0.tgz",
|
||||
@ -9903,6 +10046,25 @@
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/micromark-extension-math": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/micromark-extension-math/-/micromark-extension-math-3.1.0.tgz",
|
||||
"integrity": "sha512-lvEqd+fHjATVs+2v/8kg9i5Q0AP2k85H0WUOwpIVvUML8BapsMvh1XAogmQjOCsLpoKRCVQqEkQBB3NhVBcsOg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/katex": "^0.16.0",
|
||||
"devlop": "^1.0.0",
|
||||
"katex": "^0.16.0",
|
||||
"micromark-factory-space": "^2.0.0",
|
||||
"micromark-util-character": "^2.0.0",
|
||||
"micromark-util-symbol": "^2.0.0",
|
||||
"micromark-util-types": "^2.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/micromark-factory-destination": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.0.tgz",
|
||||
@ -13987,6 +14149,7 @@
|
||||
"version": "9.0.1",
|
||||
"resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-9.0.1.tgz",
|
||||
"integrity": "sha512-186Gw/vF1uRkydbsOIkcGXw7aHq0sZOCRFFjGrr7b9+nVZg4UfA4enXCaxm4fUzecU38sWfrNDitGhshuU7rdg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/hast": "^3.0.0",
|
||||
"devlop": "^1.0.0",
|
||||
@ -14335,6 +14498,25 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/rehype-katex": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/rehype-katex/-/rehype-katex-7.0.1.tgz",
|
||||
"integrity": "sha512-OiM2wrZ/wuhKkigASodFoo8wimG3H12LWQaH8qSPVJn9apWKFSH3YOCtbKpBorTVw/eI7cuT21XBbvwEswbIOA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/hast": "^3.0.0",
|
||||
"@types/katex": "^0.16.0",
|
||||
"hast-util-from-html-isomorphic": "^2.0.0",
|
||||
"hast-util-to-text": "^4.0.0",
|
||||
"katex": "^0.16.0",
|
||||
"unist-util-visit-parents": "^6.0.0",
|
||||
"vfile": "^6.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/rehype-parse": {
|
||||
"version": "9.0.0",
|
||||
"resolved": "https://registry.npmjs.org/rehype-parse/-/rehype-parse-9.0.0.tgz",
|
||||
@ -14353,6 +14535,7 @@
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/rehype-prism-plus/-/rehype-prism-plus-2.0.0.tgz",
|
||||
"integrity": "sha512-FeM/9V2N7EvDZVdR2dqhAzlw5YI49m9Tgn7ZrYJeYHIahM6gcXpH0K1y2gNnKanZCydOMluJvX2cB9z3lhY8XQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"hast-util-to-string": "^3.0.0",
|
||||
"parse-numeric-range": "^1.3.0",
|
||||
@ -14376,6 +14559,7 @@
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.0.tgz",
|
||||
"integrity": "sha512-U92vJgBPkbw4Zfu/IiW2oTZLSL3Zpv+uI7My2eq8JxKgqraFdU8YUGicEJCEgSbeaG+QDFqIcwwfMTOEelPxuA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/mdast": "^4.0.0",
|
||||
"mdast-util-gfm": "^3.0.0",
|
||||
@ -14389,6 +14573,22 @@
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/remark-math": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/remark-math/-/remark-math-6.0.0.tgz",
|
||||
"integrity": "sha512-MMqgnP74Igy+S3WwnhQ7kqGlEerTETXMvJhrUzDikVZ2/uogJCb+WHUg97hK9/jcfc0dkD73s3LN8zU49cTEtA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/mdast": "^4.0.0",
|
||||
"mdast-util-math": "^3.0.0",
|
||||
"micromark-extension-math": "^3.0.0",
|
||||
"unified": "^11.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/remark-parse": {
|
||||
"version": "11.0.0",
|
||||
"resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz",
|
||||
@ -15682,6 +15882,20 @@
|
||||
"unist-util-visit-parents": "^6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/unist-util-find-after": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/unist-util-find-after/-/unist-util-find-after-5.0.0.tgz",
|
||||
"integrity": "sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/unist": "^3.0.0",
|
||||
"unist-util-is": "^6.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/unist-util-is": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz",
|
||||
|
||||
@ -42,6 +42,7 @@
|
||||
"favicon-fetch": "^1.0.0",
|
||||
"formik": "^2.2.9",
|
||||
"js-cookie": "^3.0.5",
|
||||
"katex": "^0.16.17",
|
||||
"lodash": "^4.17.21",
|
||||
"lucide-react": "^0.454.0",
|
||||
"mdast-util-find-and-replace": "^3.0.1",
|
||||
@ -59,8 +60,10 @@
|
||||
"react-markdown": "^9.0.1",
|
||||
"react-select": "^5.8.0",
|
||||
"recharts": "^2.13.1",
|
||||
"rehype-katex": "^7.0.1",
|
||||
"rehype-prism-plus": "^2.0.0",
|
||||
"remark-gfm": "^4.0.0",
|
||||
"remark-math": "^6.0.0",
|
||||
"semver": "^7.5.4",
|
||||
"sharp": "^0.33.5",
|
||||
"stripe": "^17.0.0",
|
||||
|
||||
@ -66,13 +66,17 @@ import RegenerateOption from "../RegenerateOption";
|
||||
import { LlmOverride } from "@/lib/hooks";
|
||||
import { ContinueGenerating } from "./ContinueMessage";
|
||||
import { MemoizedAnchor, MemoizedParagraph } from "./MemoizedTextComponents";
|
||||
import { extractCodeText } from "./codeUtils";
|
||||
import { extractCodeText, preprocessLaTeX } from "./codeUtils";
|
||||
import ToolResult from "../../../components/tools/ToolResult";
|
||||
import CsvContent from "../../../components/tools/CSVContent";
|
||||
import SourceCard, {
|
||||
SeeMoreBlock,
|
||||
} from "@/components/chat_search/sources/SourceCard";
|
||||
|
||||
import remarkMath from "remark-math";
|
||||
import rehypeKatex from "rehype-katex";
|
||||
import "katex/dist/katex.min.css";
|
||||
|
||||
const TOOLS_WITH_CUSTOM_HANDLING = [
|
||||
SEARCH_TOOL_NAME,
|
||||
INTERNET_SEARCH_TOOL_NAME,
|
||||
@ -224,6 +228,7 @@ export const AIMessage = ({
|
||||
setPresentingDocument?: (document: OnyxDocument) => void;
|
||||
}) => {
|
||||
const toolCallGenerating = toolCall && !toolCall.tool_result;
|
||||
|
||||
const processContent = (content: string | JSX.Element) => {
|
||||
if (typeof content !== "string") {
|
||||
return content;
|
||||
@ -242,12 +247,16 @@ export const AIMessage = ({
|
||||
|
||||
const lastMatch = matches[matches.length - 1];
|
||||
if (!lastMatch.endsWith("```")) {
|
||||
return content;
|
||||
return preprocessLaTeX(content);
|
||||
}
|
||||
}
|
||||
|
||||
return content + (!isComplete && !toolCallGenerating ? " [*]() " : "");
|
||||
return (
|
||||
preprocessLaTeX(content) +
|
||||
(!isComplete && !toolCallGenerating ? " [*]() " : "")
|
||||
);
|
||||
};
|
||||
|
||||
const finalContent = processContent(content as string);
|
||||
|
||||
const [isRegenerateHovered, setIsRegenerateHovered] = useState(false);
|
||||
@ -356,8 +365,8 @@ export const AIMessage = ({
|
||||
<ReactMarkdown
|
||||
className="prose max-w-full text-base"
|
||||
components={markdownComponents}
|
||||
remarkPlugins={[remarkGfm]}
|
||||
rehypePlugins={[[rehypePrism, { ignoreMissing: true }]]}
|
||||
remarkPlugins={[remarkGfm, remarkMath]}
|
||||
rehypePlugins={[[rehypePrism, { ignoreMissing: true }], rehypeKatex]}
|
||||
>
|
||||
{finalContent as string}
|
||||
</ReactMarkdown>
|
||||
|
||||
@ -59,3 +59,18 @@ export function extractCodeText(
|
||||
|
||||
return codeText || "";
|
||||
}
|
||||
|
||||
// This is a temporary solution to preprocess LaTeX in LLM output
|
||||
export const preprocessLaTeX = (content: string) => {
|
||||
// Replace block-level LaTeX delimiters \[ \] with $$ $$
|
||||
const blockProcessedContent = content.replace(
|
||||
/\\\[([\s\S]*?)\\\]/g,
|
||||
(_, equation) => `$$${equation}$$`
|
||||
);
|
||||
// Replace inline LaTeX delimiters \( \) with $ $
|
||||
const inlineProcessedContent = blockProcessedContent.replace(
|
||||
/\\\(([\s\S]*?)\\\)/g,
|
||||
(_, equation) => `$${equation}$`
|
||||
);
|
||||
return inlineProcessedContent;
|
||||
};
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user