diff --git a/tests/test_renderer_js.py b/tests/test_renderer_js.py index 280d6b2c..416cd780 100644 --- a/tests/test_renderer_js.py +++ b/tests/test_renderer_js.py @@ -1549,3 +1549,352 @@ def test_streaming_apply_marks_buffer_only_on_success() -> None: assert ".catch(function (e) {" in body[chain_at : chain_at + 3500], ( "every mermaid chain link must settle back to fulfilled" ) + + +# --------------------------------------------------------------------------- +# Renderer containment escapes (frontend-render-containment-brief) +# +# The renderer protects structural blocks with in-band NUL-framed sentinels +# (NUL + two-letter-tag + index + NUL, e.g. code-block 0 -> chr(0)+"CB0"+chr(0)). +# escapeHtml preserves U+0000, so model/tool text carrying such a sequence used +# to FORGE a sentinel: the shared restore pass rewrote every match, duplicating +# or relocating a protected block (B1), printing literal "undefined" for an +# out-of-range index (B2), or injecting a restored span across a container (B3). +# Fix 1 strips U+0000 (NUL) at the TOP-LEVEL render entry only, so no forged +# NUL survives to frame a sentinel while generated (recursive-frame) sentinels +# are left intact. Only NUL is stripped — every other control byte survives so +# code fences show pasted source verbatim. Inputs build NUL via chr(0) (never +# a literal escape) per the brief. +# --------------------------------------------------------------------------- + +_NUL = chr(0) + + +def test_forged_code_block_sentinel_does_not_duplicate_block() -> None: + """B1: prose carrying a forged ``chr(0)+CB0+chr(0)`` used to make the + shared restore pass emit the protected code block a SECOND time (content + spoofing / relocation). Stripping NUL at the entry neutralises the + forgery: exactly one code block, no leaked sentinel.""" + md = "```python\nprint('hi')\n```\n\nprose " + _NUL + "CB0" + _NUL + " end" + out = _render(md) + assert out.count("
") == 1, "forged CB sentinel duplicated the block:\n" + out
+    assert out.count("print(") == 1
+    assert _NUL not in out, "raw NUL / forged sentinel leaked into output"
+
+
+def test_forged_out_of_range_sentinel_does_not_print_undefined() -> None:
+    """B2: ``chr(0)+IC7+chr(0)`` with no inline codes used to restore
+    ``inlineCodes[7]`` -> literal ``undefined`` in the rendered text.  After
+    the entry strip the forged framing is gone, so no ``undefined`` appears."""
+    out = _render("text " + _NUL + "IC7" + _NUL + " tail")
+    assert "undefined" not in out, "out-of-range forged sentinel printed 'undefined':\n" + out
+    assert _NUL not in out
+
+
+def test_control_strip_preserves_legit_fence_and_inline() -> None:
+    """Fix 1 must not disturb legitimately generated sentinels: a normal
+    fence and inline-code span still render after the entry strip (the strip
+    only removes caller-supplied control chars, which are never valid data)."""
+    out = _render("Here is `inline` and a block:\n\n```py\nx = 1\n```")
+    assert "inline" in out
+    assert "
 None:
+    """The entry strip removes ONLY NUL (the sentinel-framing byte), so a code
+    fence still shows pasted control bytes (terminal output, ANSI escapes)
+    verbatim.  Stripping the whole C0/DEL range would silently corrupt code
+    samples; only NUL can forge a sentinel."""
+    esc = chr(27)  # ANSI escape — legitimate in pasted terminal output
+    out = _render("```\nbefore " + esc + "[0m after " + _NUL + " end\n```")
+    assert esc in out, "ESC (0x1b) must survive inside a code fence:\n" + repr(out)
+    assert _NUL not in out, "NUL must still be stripped (sentinel-framing byte)"
+    assert "before " in out and " end" in out
+
+
+def test_forged_inline_sentinel_not_injected_inside_fence() -> None:
+    """B3: a forged ``chr(0)+IC0+chr(0)`` placed inside a real code fence
+    used to be substituted AFTER the fence was restored (CB restores before
+    IC), injecting a real ```` span into the ``
``.  With a genuine
+    inline-code span present (so inlineCodes[0] exists), the forged reference
+    must NOT clone it into the code block."""
+    md = "`real`\n\n```text\nbefore " + _NUL + "IC0" + _NUL + " after\n```"
+    out = _render(md)
+    assert out.count("real") == 1, "forged IC sentinel injected into 
:\n" + out
+    assert _NUL not in out
+    assert "before IC0 after" in out, "fence body should show the inert forged tag as text"
+
+
+def test_nul_strip_scoped_to_top_level_call() -> None:
+    """Structural pin for the PLAUSIBLE placement refinement: the NUL strip
+    lives inside the ``_fnDepth === 0`` guard of the exported wrapper, NOT in
+    ``_renderMarkdownBody`` (which runs at every recursion depth).  An
+    unconditional strip would shred the generated sentinels that recursive
+    ``
``/footnote frames legitimately carry — foreclosing the + recursive-frame fix. Recursion must reach raw text with its sentinels.""" + body = _RENDERER_JS.read_text(encoding="utf-8") + assert "_NUL_STRIP_RE" in body + wrapper = body.index("export function renderMarkdown(text)") + body_fn = body.index("function _renderMarkdownBody(text)") + seg = body[wrapper:body_fn] + guard_at = seg.index("_fnDepth === 0") + strip_at = seg.index("_NUL_STRIP_RE", guard_at) + incr_at = seg.index("_fnDepth++") + assert guard_at < strip_at < incr_at, ( + "the NUL strip must run inside the top-level (_fnDepth === 0) " + "guard, before the depth increment" + ) + assert "_NUL_STRIP_RE" not in body[body_fn:], ( + "strip must not live in _renderMarkdownBody (would run at every depth)" + ) + + +def test_recursive_frame_degrades_without_literal_undefined() -> None: + """Fix 2 floor for the NEW-1 residual: a recursive render frame + (``
`` body, footnote definition) whose fresh block arrays cannot + resolve an outer-scope sentinel must NOT print the literal word + ``undefined``. The restore callbacks return the (inert) matched sentinel + instead. (This asserts only the ``undefined`` floor — Fix 5 is what makes + the body actually render; the raw sentinel that the node harness preserves + here is dropped by a real browser's tokenizer.)""" + details = _render("
\nx\n\n```py\nsecret_code()\n```\n\n
") + assert "undefined" not in details, "code-in-
printed 'undefined':\n" + details + footnote = _render("See[^1].\n\n[^1]: a `snippet` ok") + assert "undefined" not in footnote, "inline-code-in-footnote printed 'undefined':\n" + footnote + + +def test_standalone_code_block_not_wrapped_in_paragraph() -> None: + """Fix 6 (NEW-3): code blocks need the ``

SENTINEL

`` unwrap variant + that DT/BQ/MB/TB already have. Without it a lone fenced block emits + ``

``, which a real browser splits into a stray empty + ``

`` before the ``

``.  The unwrap removes the wrapping paragraph."""
+    out = _render("```py\nx = 1\n```")
+    assert "
" not in out, "code block still wrapped in a paragraph:\n" + out
+    assert out.strip().startswith("
"), "code block should not be paragraph-wrapped:\n" + out
+
+
+# ---------------------------------------------------------------------------
+# Fix 3 — blockquote-in-fence (B4): fence protection must run before (and
+# mask) the line-based blockquote pass, with the fence open anchored to line
+# start so a blockquoted fence (`> ```) is NOT matched at column > 0.
+# ---------------------------------------------------------------------------
+
+
+def test_blockquote_inside_fence_not_extracted() -> None:
+    """B4 (the common one, no special chars): ``> `` lines INSIDE a code
+    fence used to be scooped out by the blockquote pre-pass (which ran first)
+    and rendered as a real ``
`` nested in ``
`` — a
+    shell transcript or quoted-email code block would sprout a headline.  The
+    fence pass now runs first and masks the region."""
+    out = _render("```text\nplain\n> quoted\nafter\n```")
+    assert "
" not in out, "blockquote extracted from inside a fence:\n" + out + assert "
 None:
+    """A fence nested inside a blockquote (``> ```` ``) must still render as a
+    code block WITHIN the ``
``. Anchoring the fence open to line + start means it is not matched at column > 0, so the blockquote pass + extracts the ``> `` run and its recursive render handles the fence. (Pins + that we did not over-correct by simply hoisting the fence pass — which + would have swallowed the blockquoted fence as ``undefined``.)""" + out = _render("> ```\n> code\n> ```") + assert "
" in out + assert "
code
" in out, "blockquoted fence lost its code:\n" + out + assert "undefined" not in out + assert _NUL not in out + + +def test_indented_fence_still_renders_as_code() -> None: + """The open anchor allows up to 3 spaces of indentation (CommonMark), so a + legitimately indented fence (e.g. under a list item) still renders as code + rather than as a paragraph of literal backticks. A bare ``^`` anchor + would have dropped it.""" + out = _render(" ```py\n x = 1\n ```") + assert "
 open anchored to line start (B5). The details pass ran
+# with an unanchored open, so a `
` mentioned mid-line inside inline +# code matched across the backtick spans and swallowed the DT sentinel / +# lost the content between them. +# --------------------------------------------------------------------------- + + +def test_inline_code_details_tag_not_consumed_by_details_pass() -> None: + """B5: ``Use `
` then `
` to fold`` must render two + inline-code spans of the literal tags — NOT a real
element with + the text between the spans swallowed.""" + out = _render("Use `
` then `
` to fold.") + assert "<details>" in out, "opening
tag not shown as literal code:\n" + out + assert "</details>" in out, "closing
tag not shown as literal code:\n" + out + assert "
" not in out, "a real
element was wrongly created:\n" + out + assert out.count("") == 2, "expected two inline-code spans:\n" + out + + +def test_block_details_still_renders() -> None: + """No-regression: a genuine multi-line
block (at line start) + still renders as a real disclosure element.""" + out = _render("
\nMore\n\nBody text here.\n\n
") + assert "
More" in out + assert "Body text here." in out + + +def test_oneline_details_still_renders() -> None: + """No-regression: the common one-line form must survive the open anchor + (anchoring the CLOSE too would break this — do not).""" + out = _render("
xy
") + assert "
x" in out + assert "y" in out and out.rstrip().endswith("
") + + +def test_details_inside_fence_stays_literal() -> None: + """Lock the behavior Fix 5a must preserve: a
shown INSIDE a code + fence is masked by the (earlier) fence pass and must stay literal escaped + code, never extracted into a real element.""" + out = _render("```html\n
sx
\n```") + assert "
" not in out, "details inside a fence was wrongly extracted:\n" + out
+
+
+# ---------------------------------------------------------------------------
+# Fix 5 (NEW-1) — recursive-frame content loss.  renderMarkdown recurses for
+# 
bodies and footnote definitions. When those bodies were extracted +# AFTER the fence/inline-code/math passes, they carried outer-scope sentinels +# that the recursive call — with fresh, empty block arrays — could not resolve, +# so a code block / inline code / math inside them rendered as `undefined` (or, +# after the Fix 2 floor, an inert `CB0`/`IC0` sentinel) — silent content loss. +# The structural fix extracts
from RAW markdown (before fence/inline +# protection, fence-aware) and collects footnote definitions before the inline +# passes, so each recursion sees raw content. +# --------------------------------------------------------------------------- + + +def test_code_block_in_details_renders_code() -> None: + """NEW-1 (a), the headline case: a fenced code block inside
must + render the CODE, not `undefined` and not an inert `CB0` sentinel.""" + out = _render("
\nx\n\n```py\nsecret_code()\n```\n\n
") + assert "secret_code()" in out, "code inside
was lost:\n" + out + assert "
 None:
+    """NEW-1 generalises to any recursive block: a blockquote inside 
+ must render as a real
, not a lost/inert sentinel.""" + out = _render("
\nx\n\n> quoted\n\n
") + assert "
" in out, "blockquote inside
was lost:\n" + out + assert "quoted" in out + assert _NUL not in out + + +def test_inline_code_in_footnote_renders() -> None: + """NEW-1 (b): inline code in a footnote definition must render as a real + span in the footnote section, not `undefined`/`IC0`.""" + out = _render("See[^1].\n\n[^1]: uses `code` here") + assert "code" in out, "inline code in footnote def was lost:\n" + out + assert "undefined" not in out + assert _NUL not in out + + +def test_math_in_footnote_renders() -> None: + r"""NEW-1 (b), math variant: display/inline math in a footnote definition + must reach KaTeX, not restore to `undefined`/`MB0`.""" + out = _render("See[^1].\n\n[^1]: with \\(x^2\\) inline") + assert '' in out, "math in footnote def was lost:\n" + out + assert "undefined" not in out + assert _NUL not in out + + +# --------------------------------------------------------------------------- +# Review round-1 regression pins: the details pass runs AFTER fence protection +# (fence-masking, not offset math, provides fence-awareness), and both the +# fence and details opens allow arbitrary leading indent. +# --------------------------------------------------------------------------- + + +def test_details_close_tag_shown_in_fenced_example_does_not_close_block() -> None: + """A `
` shown as example code inside a fence must NOT close the + real disclosure early. Because the fence pass runs first and masks the + example as a sentinel, the details close matches only the real trailing + tag; the fenced example renders as literal code inside the block.""" + md = "
\ns\n\n```html\n
\n```\n\n
" + out = _render(md) + assert '
' in out, "fenced example was swallowed:\n" + out
+    assert "</details>" in out, "example 
should be literal code:\n" + out + assert out.strip().startswith("
s"), out + assert out.rstrip().endswith("
"), "real block closed early / stray text:\n" + out + assert _NUL not in out + + +def test_deeply_indented_fence_renders_as_code() -> None: + """A fence indented 4+ spaces (as when nested under a list item) still + tokenises as a code block — the open anchor allows arbitrary indent, so we + don't regress deeply-nested code samples to literal backticks.""" + out = _render(" ```py\n x = 1\n ```") + assert "
 None:
+    """A code fence that OPENS on the same line as a list marker (`- ```py`)
+    still tokenises as a code block inside the list item.  The open matches
+    after an optional list marker, which is re-emitted before the sentinel so
+    the list pass still sees the item.  Regression guard: a bare `^[ \\t]*`
+    anchor (no list-marker allowance) destroyed the block and leaked the raw
+    backticks + language tag as text."""
+    for src in ["- ```py\n  print(1)\n  ```", "1. ```py\n   print(1)\n   ```"]:
+        out = _render(src)
+        assert "
" in out, "list structure lost:\n" + out
+
+
+def test_nested_list_fence_stays_nested() -> None:
+    """A fenced code block as a NESTED sub-item keeps its nesting level: the
+    fence pass re-emits the leading indent before the sentinel, so the list
+    pass still reads the sub-item's indentation.  Regression guard: dropping
+    the indent flattened the code block to a top-level sibling of the parent."""
+    out = _render("- parent\n  - ```py\n    code\n    ```")
+    assert "parent" in out
+    assert "
") == 2, "nested list fence flattened to a sibling:\n" + out
+
+
+def test_big_ordered_marker_fence_is_protected() -> None:
+    r"""A fence opening on a 10+ digit ordered-list marker line is still
+    protected — the marker alternation uses ``\d+``, matching the list pass,
+    not a capped ``\d{1,9}`` that would leave the fence unprotected."""
+    out = _render("1234567890. ```py\ncode\n```")
+    assert "
 None:
+    """A fenced code block continuing a footnote definition renders INSIDE the
+    footnote section (the fence pass re-emits the 2-space indent the
+    continuation scan needs; the restore round-trip then resolves it there)."""
+    out = _render("See[^1].\n\n[^1]: note\n  ```py\n  x=1\n  ```")
+    assert 'class="footnotes"' in out
+    assert out.find(" out.find('class="footnotes"'), (
+        "fenced code in a footnote rendered outside the footnote section:\n" + out
+    )
+    assert "x=1" in out
+
+
+def test_indented_details_is_extracted() -> None:
+    """An indented `
` (e.g. under a list item) is still extracted into + a real disclosure element — the open anchor allows leading whitespace, + while a mid-line `
` inside inline code still is not (B5).""" + out = _render("
xy
") + assert "
x" in out, "indented
not extracted:\n" + out + assert "y" in out diff --git a/turnstone/shared_static/renderer.js b/turnstone/shared_static/renderer.js index e213cb8a..bac1d19b 100644 --- a/turnstone/shared_static/renderer.js +++ b/turnstone/shared_static/renderer.js @@ -268,6 +268,15 @@ function _langToCssClass(lang) { // the cap the nested body renders as escaped plain text: degraded, visible. var _MD_MAX_DEPTH = 100; +// U+0000 (NUL) only. Structural sentinels are NUL-framed (chr(0)+tag+idx+ +// chr(0)) and renderer.js is the sole NUL producer, so stripping NUL at the +// top-level entry closes every forgery path — while leaving every OTHER +// control byte intact, because a code fence must show pasted source verbatim +// (terminal output legitimately carries ESC/FF/VT/DEL, none of which can forge +// a sentinel). Authored as the literal \x00 escape (only \uXXXX decodes to a +// raw byte in this toolchain), matching the file's \x00 sentinel convention. +var _NUL_STRIP_RE = /\x00/g; + export function renderMarkdown(text) { if (_fnDepth >= _MD_MAX_DEPTH) { return "

" + escapeHtml(String(text == null ? "" : text)) + "

"; @@ -276,7 +285,21 @@ export function renderMarkdown(text) { // messages). Depth accounting rides a try/finally: a throw anywhere in the // body used to strand _fnDepth elevated, freezing _fnScopeId so footnote // anchor ids collided across every later message. - if (_fnDepth === 0) _fnScopeId++; + // + // Strip caller-supplied NUL, but at the TOP-LEVEL call ONLY. The renderer + // frames structural blocks with in-band NUL sentinels (NUL+tag+index+NUL) + // and escapeHtml preserves U+0000, so model/tool text carrying that shape + // could forge a sentinel and duplicate/relocate a block or print "undefined" + // (B1/B2/B3). renderer.js is the sole NUL producer and every restore regex + // is NUL-framed, so removing NUL here erases every forgery path — and ONLY + // NUL, so a fenced code block still shows pasted control bytes (ESC/FF/VT/ + // DEL) verbatim. Depth 0 only: recursive frames (blockquote/details/ + // footnote bodies) legitimately carry generated sentinels and an + // unconditional strip would shred them. NUL is never valid content. + if (_fnDepth === 0) { + text = String(text == null ? "" : text).replace(_NUL_STRIP_RE, ""); + _fnScopeId++; + } _fnDepth++; try { return _renderMarkdownBody(text); @@ -285,10 +308,148 @@ export function renderMarkdown(text) { } } +// Restore-pass callback factory: one guarded closure per protected-block +// array. Returns the matched sentinel `m` when the index is out of range (an +// inert placeholder the tokenizer strips the NUL from) instead of the array's +// `undefined`. Factored so the ~12 restore passes share one implementation +// and can't drift (e.g. a callback reading the wrong array after a copy-paste). +function _restorer(arr) { + return function (m, idx) { + var v = arr[parseInt(idx)]; + return v === undefined ? m : v; + }; +} + function _renderMarkdownBody(text) { - // Pre-pass: extract blockquote blocks and recursively render. - // Must run FIRST (before code/math protection) so the recursive call - // processes raw markdown, not text with outer-scope placeholders. + // Protect code blocks before the line-based passes (blockquote, + // table) that would otherwise scoop a `> ` / `|` line out of a fenced body + // and render it as real markdown nested in
 (B4: shell
+  // transcripts, quoted email, markdown-about-markdown).  The open matches at
+  // line start after optional indent (group 1) AND an optional list marker
+  // (group 2: `- `, `1. `), so a fence opening on a list-item's marker line
+  // (`- ```py`) still tokenises; both the indent and the marker are re-emitted
+  // before the sentinel so the fence keeps its document position (see the
+  // callback).  A blockquoted fence (`> ```) is NOT matched — `>` is not indent
+  // or a list marker — so the blockquote pass below extracts that `> ` run and
+  // its recursive render handles it.
+  //
+  // The opening-run length is captured and required on the close via
+  // backreference so a 4-backtick outer fence wrapping a 3-backtick inner
+  // (common when embedding markdown-about-markdown or lang-tagged snippets
+  // inside another code block) is tokenised as one outer block with the inner
+  // triple-backticks preserved verbatim — the prior `` ```...``` `` regex
+  // treated the outer-open and inner-open as a single fence pair, stranding
+  // the rest of the content with visible \x00CB{n}\x00 sentinels.
+  //
+  // Two constraints below close the gap that mid-stream buffers expose:
+  //
+  //   1. Content can't contain its own close pattern — `(?!\3)` (group 3 is
+  //      the backtick run; groups 1-2 are the indent and optional list marker)
+  //      inside the content quantifier blocks the lazy matcher from extending
+  //      across another N-backtick run. Without this, a buffer like
+  //      ```mermaid\n\n```python\n\n```  would extend
+  //      mermaid's content all the way to the FINAL ```, swallowing python
+  //      and handing mermaid a wrong (and incomplete-looking) source. With
+  //      the lookahead, content stops at the first matching run and the open
+  //      simply doesn't match anything until a true close arrives. Inner
+  //      backticks of a SMALLER count (e.g. 3-backtick inner inside a
+  //      4-backtick outer) still pass since `\3` is the OPEN count.
+  //
+  //   2. The close must live at a line boundary — `[ \t]*(?=\n|$)` after
+  //      `\3` forbids the close from being immediately followed by a language
+  //      tag, so ```python opening another fence can't masquerade as the
+  //      previous fence's close.
+  //
+  // Together these mean an unclosed fence stays as plain markdown until its
+  // true close arrives — no intermediate parse errors flash through mermaid /
+  // hljs while a stream is in flight.
+  // codeBlockRaw keeps each fence's RAW source next to its rendered HTML, so
+  // the 
pre-pass below can restore a fenced body to raw markdown for + // its recursive render (NEW-1) rather than an unresolvable outer sentinel. + // Populated only when the text contains a
tag — its sole reader — + // so the common no-
render skips the per-fence slice. (When there + // is no
, the details .replace matches nothing and its CB->raw + // restore never runs, so the empty array is never read.) + var codeBlocks = []; + var codeBlockRaw = []; + var needFenceRaw = /
/i.test(text); + text = text.replace( + /^([ \t]*)((?:[-*+]|\d+[.)])[ \t]+)?(```+)([^\s`]*)\n((?:(?!\3)[\s\S])*?)\3[ \t]*(?=\n|$)/gm, + function (m, indent, marker, _open, lang, code) { + var cssLang = _langToCssClass(lang); + codeBlocks.push( + "
" +
+          escapeHtml(code.replace(/\n$/, "")) +
+          "
", + ); + // Store the fence source WITHOUT the re-emitted indent/marker prefix so + // the
CB->raw restore substitutes the fence alone (no double); + // only when a
is present to read it (kept index-aligned with + // codeBlocks because every fence pushes to both in that case). + if (needFenceRaw) { + codeBlockRaw.push( + m.slice(indent.length + (marker ? marker.length : 0)), + ); + } + // Re-emit the leading indent AND any list marker so the fence keeps its + // place in the document: a nested list item (` - ```py`) stays nested, + // `- ```py` stays a list item, and a fence continuing a footnote def + // keeps the 2-space indent the continuation scan needs. The CB

-unwrap + // (restore pass) tolerates that preserved indent, so an own-line indented + // fence still leaves no stray

(NEW-3). A blockquoted fence (`> ```) + // stays excluded: `>` is neither indent nor a list marker. + return ( + indent + (marker || "") + "\x00CB" + (codeBlocks.length - 1) + "\x00" + ); + }, + ); + + // Extract

blocks and recursively render — AFTER fence protection, + // BEFORE the blockquote/inline/math passes. Running after fence means a + // fenced code block inside
is already a \x00CB\x00 sentinel; it is + // restored to its RAW source (codeBlockRaw) before the recursive + // renderMarkdown, so the code renders in-frame instead of restoring to + // `undefined` / an inert sentinel against the recursion's empty arrays + // (NEW-1, silent content loss). Fence-first also masks a `
` shown + // as example code (so it can't close the block early) and a whole
+ // shown inside a fence (so it stays literal) — no separate fence-awareness + // needed. The open is anchored to line start (`^[ \t]*`: allows indentation, + // e.g. under a list, but not a mid-line `
` inside inline code — B5); + // the close stays unanchored so the one-line + //
xy
form still matches. + var detailsBlocks = []; + text = text.replace( + /^[ \t]*
\s*\n?([\s\S]*?)<\/details>/gim, + function (m, inner) { + // Restore any fenced bodies to raw markdown so the recursion re-renders + // them in its own frame (the outer codeBlocks entry is then unused). + inner = inner.replace(/\x00CB(\d+)\x00/g, _restorer(codeBlockRaw)); + var sumMatch = inner.match( + /^\s*([\s\S]*?)<\/summary>\s*\n?([\s\S]*)/i, + ); + var html; + if (sumMatch) { + html = + "
" + + inlineMarkdown(sumMatch[1].trim()) + + "" + + renderMarkdown(sumMatch[2]) + + "
"; + } else { + html = "
" + renderMarkdown(inner) + "
"; + } + detailsBlocks.push(html); + return "\x00DT" + (detailsBlocks.length - 1) + "\x00"; + }, + ); + + // Pre-pass: extract blockquote blocks and recursively render. Runs after + // fence protection (a fenced `> ` line is already masked as a \x00CB\x00 + // sentinel, so it is not scooped here — B4) but before the remaining + // code/math protection, so the recursive call still processes raw markdown, + // not text with outer-scope placeholders. var bqBlocks = []; (function () { var blines = text.split("\n"); @@ -350,80 +511,6 @@ function _renderMarkdownBody(text) { text = result.join("\n"); })(); - // Protect code blocks. The opening-run length is captured and - // required on the close via backreference so a 4-backtick outer - // fence wrapping a 3-backtick inner (common when embedding - // markdown-about-markdown or lang-tagged snippets inside another - // code block) is tokenised as one outer block with the inner - // triple-backticks preserved verbatim — the prior `` ```...``` `` - // regex treated the outer-open and inner-open as a single fence - // pair, stranding the rest of the content with visible - // \x00CB{n}\x00 sentinels. - // - // Two constraints below close the gap that mid-stream buffers - // expose: - // - // 1. Content can't contain its own close pattern — `(?!\1)` - // inside the content quantifier blocks the lazy matcher - // from extending across another N-backtick run. Without - // this, a buffer like ```mermaid\n\n```python\n - // \n``` would extend mermaid's content all the - // way to the FINAL ```, swallowing python and handing - // mermaid a wrong (and incomplete-looking) source. With - // the lookahead, content stops at the first matching run - // and the open simply doesn't match anything until a true - // close arrives. Inner backticks of a SMALLER count (e.g. - // 3-backtick inner inside a 4-backtick outer) still pass - // since `\1` is the OPEN count, not just three. - // - // 2. The close must live at a line boundary — `[ \t]*(?=\n|$)` - // after `\1` forbids the close from being immediately - // followed by a language tag, so ```python opening another - // fence can't masquerade as the previous fence's close. - // - // Together these mean an unclosed fence stays as plain markdown - // until its true close arrives — no intermediate parse errors - // flash through mermaid / hljs while a stream is in flight. - var codeBlocks = []; - text = text.replace( - /(```+)([^\s`]*)\n((?:(?!\1)[\s\S])*?)\1[ \t]*(?=\n|$)/g, - function (m, _open, lang, code) { - var cssLang = _langToCssClass(lang); - codeBlocks.push( - "
" +
-          escapeHtml(code.replace(/\n$/, "")) +
-          "
", - ); - return "\x00CB" + (codeBlocks.length - 1) + "\x00"; - }, - ); - - // Protect
blocks (safe HTML — attribute-free only) - var detailsBlocks = []; - text = text.replace( - /
\s*\n?([\s\S]*?)<\/details>/gi, - function (m, inner) { - var sumMatch = inner.match( - /^\s*([\s\S]*?)<\/summary>\s*\n?([\s\S]*)/i, - ); - var html; - if (sumMatch) { - html = - "
" + - inlineMarkdown(sumMatch[1].trim()) + - "" + - renderMarkdown(sumMatch[2]) + - "
"; - } else { - html = "
" + renderMarkdown(inner) + "
"; - } - detailsBlocks.push(html); - return "\x00DT" + (detailsBlocks.length - 1) + "\x00"; - }, - ); - // Protect inline code FIRST so backtick spans containing math // delimiters (e.g. `` `$$x$$` `` or `` `\[x\]` ``) stay literal. // Display math used to run first, but that lets the math regex @@ -700,7 +787,20 @@ function _renderMarkdownBody(text) { var result = out.join("\n"); - // Append footnote section if any definitions were collected + // Append footnote section if any definitions were collected. + // + // Each definition body is rendered by a recursive renderMarkdown call. The + // body was collected AFTER the inline-code/math passes, so it may carry + // outer-scope sentinels (e.g. `code` in a footnote -> a \x00IC\x00 sentinel). + // The recursion can't resolve those against its own fresh, empty arrays, but + // the restore guard (Fix 2) leaves the sentinel intact instead of emitting + // "undefined"; because this section is appended to `result` BEFORE the + // restore passes below — whose inlineCodes/mathBlocks are still populated — + // the OUTER restore resolves it, so inline code / math in a footnote renders + // correctly. A FENCED block continuing a footnote definition works the same + // way: the fence pass re-emits its 2-space indent before the sentinel, so + // the continuation scan still collects it and the round-trip restores the + // code inside the footnote item. var fnKeys = Object.keys(footnoteDefs); if (fnKeys.length > 0) { var fnHtml = @@ -727,40 +827,31 @@ function _renderMarkdownBody(text) { result += fnHtml; } - // Restore protected blocks - result = result.replace(/\x00CB(\d+)\x00/g, function (m, idx) { - return codeBlocks[parseInt(idx)]; - }); - result = result.replace(/

\x00DT(\d+)\x00<\/p>/g, function (m, idx) { - return detailsBlocks[parseInt(idx)]; - }); - result = result.replace(/\x00DT(\d+)\x00/g, function (m, idx) { - return detailsBlocks[parseInt(idx)]; - }); - result = result.replace(/

\x00BQ(\d+)\x00<\/p>/g, function (m, idx) { - return bqBlocks[parseInt(idx)]; - }); - result = result.replace(/\x00BQ(\d+)\x00/g, function (m, idx) { - return bqBlocks[parseInt(idx)]; - }); - result = result.replace(/

\x00MB(\d+)\x00<\/p>/g, function (m, idx) { - return mathBlocks[parseInt(idx)]; - }); - result = result.replace(/\x00MB(\d+)\x00/g, function (m, idx) { - return mathBlocks[parseInt(idx)]; - }); - result = result.replace(/

\x00TB(\d+)\x00<\/p>/g, function (m, idx) { - return tableBlocks[parseInt(idx)]; - }); - result = result.replace(/\x00TB(\d+)\x00/g, function (m, idx) { - return tableBlocks[parseInt(idx)]; - }); - result = result.replace(/\x00IC(\d+)\x00/g, function (m, idx) { - return inlineCodes[parseInt(idx)]; - }); - result = result.replace(/\x00IM(\d+)\x00/g, function (m, idx) { - return inlineMaths[parseInt(idx)]; - }); + // Restore protected blocks through the _restorer factory (out-of-range + // indices — reachable only via a recursive frame whose fresh array can't + // resolve an outer-scope sentinel, the NEW-1 residual — leave the inert + // sentinel rather than the literal "undefined"). Each block type unwraps a + // `

SENTINEL

` paragraph first (the line pass wraps a lone sentinel in + //

) so the browser doesn't split a stray empty

off the block; the + // bare form follows. The CB unwrap also tolerates surrounding whitespace + // (`

\x00CB0\x00

`) because the fence pass re-emits an own-line + // fence's leading indent before the sentinel — the other block types emit + // their sentinel at column 0, so they don't need it. + result = result.replace( + /

[ \t]*\x00CB(\d+)\x00[ \t]*<\/p>/g, + _restorer(codeBlocks), + ); + result = result.replace(/\x00CB(\d+)\x00/g, _restorer(codeBlocks)); + result = result.replace(/

\x00DT(\d+)\x00<\/p>/g, _restorer(detailsBlocks)); + result = result.replace(/\x00DT(\d+)\x00/g, _restorer(detailsBlocks)); + result = result.replace(/

\x00BQ(\d+)\x00<\/p>/g, _restorer(bqBlocks)); + result = result.replace(/\x00BQ(\d+)\x00/g, _restorer(bqBlocks)); + result = result.replace(/

\x00MB(\d+)\x00<\/p>/g, _restorer(mathBlocks)); + result = result.replace(/\x00MB(\d+)\x00/g, _restorer(mathBlocks)); + result = result.replace(/

\x00TB(\d+)\x00<\/p>/g, _restorer(tableBlocks)); + result = result.replace(/\x00TB(\d+)\x00/g, _restorer(tableBlocks)); + result = result.replace(/\x00IC(\d+)\x00/g, _restorer(inlineCodes)); + result = result.replace(/\x00IM(\d+)\x00/g, _restorer(inlineMaths)); return result; }