diff --git a/lib/markdown.test.mjs b/lib/markdown.test.mjs new file mode 100644 index 000000000..3cbe6978a --- /dev/null +++ b/lib/markdown.test.mjs @@ -0,0 +1,104 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { normalizeDisplayMath } from "./markdown.ts"; + +describe("normalizeDisplayMath", () => { + describe("single-line $$…$$", () => { + it("splits a single-line block into three lines", () => { + assert.equal(normalizeDisplayMath("$$a + b$$"), "$$\na + b\n$$"); + }); + + it("preserves the indent of the surrounding list item", () => { + assert.equal( + normalizeDisplayMath("- item:\n $$a + b$$\n- next"), + "- item:\n $$\n a + b\n $$\n- next", + ); + }); + }); + + describe("multi-line blocks with glued delimiters", () => { + it("moves a glued opening delimiter to its own line", () => { + const input = "$$\n\\frac{a}{b} = c\n { + const input = "$$\nx = y\nz = w$$\n\nafter"; + assert.equal(normalizeDisplayMath(input), "$$\nx = y\nz = w\n$$\n\nafter"); + }); + }); + + describe("blocks nested in GFM list items", () => { + it("re-indents lazy content lines of an indented bare-fence block", () => { + const input = "- item:\n $$\nx = y\n $$\n- next"; + assert.equal(normalizeDisplayMath(input), "- item:\n $$\n x = y\n $$\n- next"); + }); + + it("re-indents partially indented content lines", () => { + const input = "- item:\n $$\n x = y\n $$\n- next"; + assert.equal(normalizeDisplayMath(input), "- item:\n $$\n x = y\n $$\n- next"); + }); + + it("does not use a sibling list item's formula as a closing fence", () => { + const input = "- first\n $$x = y\n- second\n $$z = w$$\n- third"; + assert.equal( + normalizeDisplayMath(input), + "- first\n $$x = y\n- second\n $$\n z = w\n $$\n- third", + ); + }); + + it("does not scan a bare fence past a sibling list item", () => { + const input = "- first\n $$\nx = y\n- second\n $$z = w$$\n- third"; + assert.equal( + normalizeDisplayMath(input), + "- first\n $$\nx = y\n- second\n $$\n z = w\n $$\n- third", + ); + }); + }); + + describe("blocks that must stay untouched", () => { + it("leaves a top-level block with detached delimiters untouched", () => { + const input = "$$\n\\frac{a}{b}\n$$\n\nend"; + assert.equal(normalizeDisplayMath(input), input); + }); + + it("leaves content inside fenced code blocks untouched", () => { + const input = "```\n$$ not math $$\n$$\n```\n\nreal $$x = 1$$ end"; + const normalized = normalizeDisplayMath(input); + assert.ok(normalized.includes("```\n$$ not math $$\n$$\n```")); + // every `$$` is preserved: 3 inside the fence + 2 in inline math + assert.equal(normalized.match(/\$\$/g)?.length, 5); + }); + + it("leaves inline math and plain prose untouched", () => { + const input = "text $x = 1$ and $$a + b$$ more"; + assert.equal(normalizeDisplayMath(input), input); + }); + + it("does not treat a glued opener with mid-line $$ as a block", () => { + const input = "$$x$$ and text"; + assert.equal(normalizeDisplayMath(input), input); + }); + }); + + describe("\\[ … \\] blocks", () => { + it("normalizes single-line brackets", () => { + assert.equal(normalizeDisplayMath("\\[a + b\\]"), "$$\na + b\n$$"); + }); + + it("keeps content indented when nested in a list item", () => { + assert.equal( + normalizeDisplayMath("- item:\n \\[a + b\\]\n- next"), + "- item:\n $$\n a + b\n $$\n- next", + ); + }); + + it("normalizes multi-line brackets without double-indenting", () => { + assert.equal( + normalizeDisplayMath("- item:\n \\[\n x = y\n \\]\n- next"), + "- item:\n $$\n x = y\n $$\n- next", + ); + }); + }); +}); diff --git a/lib/markdown.ts b/lib/markdown.ts index d984268cb..b82679f43 100644 --- a/lib/markdown.ts +++ b/lib/markdown.ts @@ -21,6 +21,7 @@ export function normalizeDisplayMath(markdown: string): string { let fence: { marker: string; size: number } | null = null; let inlineCodeMarkerSize = 0; let rawCodeTag: string | null = null; + const unmatchedDisplayMathUntil = new Map(); for (let index = 0; index < lines.length; index++) { const line = lines[index]; @@ -73,7 +74,16 @@ export function normalizeDisplayMath(markdown: string): string { if (bracketDisplayOneLine) { const math = bracketDisplayOneLine[2].trim(); if (math) { - normalized.push(`${bracketDisplayOneLine[1]}$$`, math, `${bracketDisplayOneLine[1]}$$`); + // Keep the content line indented together with the `$$` fence. When the + // formula is nested inside a GFM list item (indented `$$`), a content line + // at column 0 becomes a "lazy continuation" line, which makes remark-math + // mis-parse the fence pair: the opening `$$` turns into an empty math node + // and the closing one swallows the rest of the document as math content. + normalized.push( + `${bracketDisplayOneLine[1]}$$`, + `${bracketDisplayOneLine[1]}${math}`, + `${bracketDisplayOneLine[1]}$$`, + ); continue; } } @@ -82,9 +92,13 @@ export function normalizeDisplayMath(markdown: string): string { if (bracketDisplayStart) { const closingIndex = findBracketDisplayClose(lines, index + 1); if (closingIndex !== -1) { + // Same lazy-continuation guard as above: indent content lines that sit at + // column 0 so the block stays parseable when nested inside a list item. normalized.push( `${bracketDisplayStart[1]}$$`, - ...lines.slice(index + 1, closingIndex), + ...lines.slice(index + 1, closingIndex).map((mathLine) => + indentDisplayMathContent(mathLine, bracketDisplayStart[1]), + ), `${bracketDisplayStart[1]}$$`, ); index = closingIndex; @@ -96,7 +110,74 @@ export function normalizeDisplayMath(markdown: string): string { if (displayMathMatch) { const math = displayMathMatch[2].trim(); if (math) { - normalized.push(`${displayMathMatch[1]}$$`, math, `${displayMathMatch[1]}$$`); + // See the comment on bracketDisplayOneLine: without matching indentation, + // a formula nested in a GFM list item is mis-parsed by remark-math and the + // text after the formula renders as a garbled KaTeX error block. + normalized.push( + `${displayMathMatch[1]}$$`, + `${displayMathMatch[1]}${math}`, + `${displayMathMatch[1]}$$`, + ); + continue; + } + } + + // remark-math requires both `$$` delimiters to sit on their own lines, but + // models also emit display math as a multi-line block where the opening `$$` + // is glued to the first formula line and/or the closing `$$` is glued to the + // end of the last one (`$$x = 1` + `y = 2$$`). Without normalization such a + // block swallows the following text as math content and renders as garbage. + const displayMathMultiLine = line.match(/^([ \t]{0,3})\$\$(.+)$/); + if (displayMathMultiLine) { + const indent = displayMathMultiLine[1]; + const firstLine = displayMathMultiLine[2].trimEnd(); + // Only treat this as a block opener if no other `$$` is embedded mid-line + // (e.g. `$$x$$ and text` stays untouched and is rendered as inline math). + if (firstLine && !firstLine.includes("$$")) { + const closing = findDisplayMathClose( + lines, + index + 1, + indent, + unmatchedDisplayMathUntil, + ); + if (closing) { + normalized.push(`${indent}$$`, `${indent}${firstLine}`); + for (let j = index + 1; j < closing.index; j++) { + normalized.push(indentDisplayMathContent(lines[j], indent)); + } + if (closing.content) normalized.push(`${indent}${closing.content}`); + normalized.push(`${indent}$$`); + index = closing.index; + continue; + } + } + } + + // Bare `$$` opener (possibly indented inside a GFM list item). Two problems + // need fixing: (1) when the closing `$$` is glued to the last content line + // (e.g. `z = w$$`) remark-math never finds a valid closing fence and swallows + // the rest of the document; (2) inside a list item, content lines at column 0 + // are lazy continuations that break the math flow. Both are fixed by moving + // the closing `$$` to its own line and re-indenting lazy content lines. + // A column-0 block with a properly detached closing `$$` is left untouched + // (remark-math already parses it correctly). + const displayMathBareOpen = line.match(/^([ \t]{0,3})\$\$\s*$/); + if (displayMathBareOpen) { + const indent = displayMathBareOpen[1]; + const closing = findDisplayMathClose( + lines, + index + 1, + indent, + unmatchedDisplayMathUntil, + ); + if (closing && (closing.glued || indent !== "")) { + normalized.push(`${indent}$$`); + for (let j = index + 1; j < closing.index; j++) { + normalized.push(indentDisplayMathContent(lines[j], indent)); + } + if (closing.content) normalized.push(`${indent}${closing.content}`); + normalized.push(`${indent}$$`); + index = closing.index; continue; } } @@ -107,6 +188,79 @@ export function normalizeDisplayMath(markdown: string): string { return normalized.join(lineBreak); } +interface DisplayMathClose { + index: number; + content: string; + glued: boolean; +} + +function findDisplayMathClose( + lines: string[], + startIndex: number, + indent: string, + unmatchedUntil: Map, +): DisplayMathClose | null { + const knownUnmatchedUntil = unmatchedUntil.get(indent); + if (knownUnmatchedUntil !== undefined && startIndex < knownUnmatchedUntil) return null; + + for (let index = startIndex; index < lines.length; index++) { + const line = lines[index]; + if (isDisplayMathFence(line, indent)) return { index, content: "", glued: false }; + + // A new Markdown block cannot belong to the preceding formula. In particular, + // do not let a later sibling list item provide a closing `$$` for this block. + if (isDisplayMathBlockBoundary(line) || isDisplayMathOpeningLine(line)) { + unmatchedUntil.set(indent, index); + return null; + } + + const content = getDisplayMathGluedCloseContent(line, indent); + if (content !== null) return { index, content, glued: true }; + } + + // Multiple unmatched glued openers with the same indentation previously each + // scanned to EOF. Cache this range so the overall search remains linear. + unmatchedUntil.set(indent, lines.length); + return null; +} + +function isDisplayMathFence(line: string, indent: string): boolean { + if (indent === "") return /^ {0,3}\$\$\s*$/.test(line); + return line.startsWith(indent) && /^\$\$\s*$/.test(line.slice(indent.length)); +} + +function getDisplayMathGluedCloseContent(line: string, indent: string): string | null { + if (!line.startsWith(indent)) return null; + + const match = line.slice(indent.length).match(/^(.+?)\$\$\s*$/); + if (!match) return null; + + const content = match[1].trimEnd(); + return content && !content.includes("$$") ? content : null; +} + +function isDisplayMathOpeningLine(line: string): boolean { + return /^ {0,3}\$\$(?:\S|[ \t]+\S)/.test(line); +} + +function isDisplayMathBlockBoundary(line: string): boolean { + return ( + /^ {0,3}(`{3,}|~{3,})/.test(line) || + /^[ \t]*(?:[-+*]|\d{1,9}[.)])(?:[ \t]+|$)/.test(line) || + /^ {0,3}#{1,6}(?:[ \t]+|$)/.test(line) || + /^ {0,3}>/.test(line) || + /<(code|pre|script|style)\b/i.test(line) + ); +} + +function indentDisplayMathContent(line: string, indent: string): string { + if (!indent || !line || line.startsWith("\t")) return line; + + const leadingSpaces = line.match(/^ */)?.[0].length ?? 0; + if (leadingSpaces >= indent.length) return line; + return `${indent.slice(leadingSpaces)}${line}`; +} + function findBracketDisplayClose(lines: string[], startIndex: number): number { for (let index = startIndex; index < lines.length; index++) { const line = lines[index]; diff --git a/package.json b/package.json index f8702ddd6..41f0f51c8 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "start": "next start -H 127.0.0.1 -p 30141", "start:lan": "next start -H 0.0.0.0 -p 30141", "lint": "eslint .", + "test": "node --experimental-strip-types --test lib/markdown.test.mjs", "release": "npm version patch --no-git-tag-version && npm run build && npm publish --access public" }, "dependencies": {