mirror of
https://github.com/multica-ai/multica.git
synced 2026-07-05 13:29:44 +02:00
25104d1855
@tiptap/markdown parses via marked, whose tokenizer is O(n²) in document length. Opening a large markdown doc (issue description, agent instructions, …) froze the UI for tens of seconds: a 533KB plain-text doc took 61.8s to parse while the subsequent ProseMirror setContent was only 40ms. Upgrading marked doesn't help — already on 17.0.5, whose fix only covers `_`/`*` delimiter runs, not general prose. Parse large markdown in chunks instead of in one shot: split on blank lines outside fenced code blocks, parse each chunk independently, then concatenate the resulting docs. This drops marked's cost to O(n²/k) while producing a byte-identical document. Applied transparently at ContentEditor's two parse entry points (mount + WS-driven re-parse), gated at 50KB so normal small docs stay on the single-parse fast path. 533KB: parse 61.8s -> 0.95s (65x), open 100s -> 3.2s (31x). Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
63 lines
2.3 KiB
TypeScript
63 lines
2.3 KiB
TypeScript
import type { JSONContent } from "@tiptap/core";
|
|
|
|
/**
|
|
* Above this source size, ContentEditor parses markdown in chunks instead of in
|
|
* one shot. `@tiptap/markdown` parses via `marked`, whose tokenizer is O(n²) in
|
|
* document length (measured: 533KB plain text → 61.8s parse, while the following
|
|
* ProseMirror setContent is only 40ms). Whole-document parse is the bottleneck;
|
|
* below this threshold the single-parse path is fast enough and stays in use.
|
|
*/
|
|
export const MARKDOWN_CHUNK_THRESHOLD = 50_000;
|
|
|
|
export interface MarkdownManagerLike {
|
|
parse(markdown: string): JSONContent;
|
|
}
|
|
|
|
/**
|
|
* Parse markdown into a ProseMirror JSON doc in chunks to dodge marked's O(n²).
|
|
*
|
|
* Splitting into k chunks and parsing each independently drops the cost to
|
|
* O(n²/k) — marked only ever scans within one small chunk. Cuts happen only at
|
|
* blank lines OUTSIDE fenced code blocks, so every chunk is a complete sequence
|
|
* of block nodes; concatenating the per-chunk docs reproduces the same document
|
|
* a single parse would have produced.
|
|
*
|
|
* Known limitation: a "loose" list (items separated by blank lines) straddling a
|
|
* chunk boundary may render as two adjacent lists. Acceptable trade-off vs. a
|
|
* minute-long freeze, and only reachable on documents past the threshold.
|
|
*/
|
|
export function parseMarkdownChunked(
|
|
manager: MarkdownManagerLike,
|
|
markdown: string,
|
|
chunkSize = 16_000,
|
|
): JSONContent {
|
|
const lines = markdown.split("\n");
|
|
const chunks: string[] = [];
|
|
let current: string[] = [];
|
|
let currentLen = 0;
|
|
let inFence = false;
|
|
|
|
for (const line of lines) {
|
|
// Track fenced code blocks so a cut never lands inside one.
|
|
if (/^\s*(```|~~~)/.test(line)) inFence = !inFence;
|
|
current.push(line);
|
|
currentLen += line.length + 1;
|
|
|
|
// Cut only at a paragraph boundary (blank line) outside a fence, once the
|
|
// accumulated chunk is large enough.
|
|
if (currentLen >= chunkSize && !inFence && line.trim() === "") {
|
|
chunks.push(current.join("\n"));
|
|
current = [];
|
|
currentLen = 0;
|
|
}
|
|
}
|
|
if (current.length) chunks.push(current.join("\n"));
|
|
|
|
const merged: JSONContent = { type: "doc", content: [] };
|
|
for (const chunk of chunks) {
|
|
const doc = manager.parse(chunk);
|
|
if (doc.content) merged.content!.push(...doc.content);
|
|
}
|
|
return merged;
|
|
}
|