diff --git a/extensions/matrix/src/channel.message-adapter.test.ts b/extensions/matrix/src/channel.message-adapter.test.ts index 83e377a0b001..53798b5ae83b 100644 --- a/extensions/matrix/src/channel.message-adapter.test.ts +++ b/extensions/matrix/src/channel.message-adapter.test.ts @@ -259,7 +259,7 @@ describe("matrix channel message adapter", () => { }); }); - it("declares bullets as the markdown table default", () => { - expect(matrixPlugin.messaging?.defaultMarkdownTableMode).toBe("bullets"); + it("declares native blocks as the markdown table default", () => { + expect(matrixPlugin.messaging?.defaultMarkdownTableMode).toBe("block"); }); }); diff --git a/extensions/matrix/src/channel.ts b/extensions/matrix/src/channel.ts index e4ec8fd94931..4722fc91afd6 100644 --- a/extensions/matrix/src/channel.ts +++ b/extensions/matrix/src/channel.ts @@ -475,7 +475,7 @@ export const matrixPlugin: ChannelPlugin = }).map(projectMatrixConversationBinding), }, messaging: { - defaultMarkdownTableMode: "bullets", + defaultMarkdownTableMode: "block", targetPrefixes: ["matrix"], targetIdComparison: "case-sensitive", normalizeTarget: normalizeMatrixMessagingTarget, diff --git a/extensions/matrix/src/matrix/actions/messages.test.ts b/extensions/matrix/src/matrix/actions/messages.test.ts index 3944c29abce7..1c1b24006255 100644 --- a/extensions/matrix/src/matrix/actions/messages.test.ts +++ b/extensions/matrix/src/matrix/actions/messages.test.ts @@ -19,6 +19,7 @@ function installMatrixActionTestRuntime(): void { channel: { text: { resolveMarkdownTableMode: () => "code", + resolveTextChunkLimit: () => 4_000, convertMarkdownTables: (text: string) => text, }, }, diff --git a/extensions/matrix/src/matrix/format-profile.ts b/extensions/matrix/src/matrix/format-profile.ts new file mode 100644 index 000000000000..d26dc851fbaa --- /dev/null +++ b/extensions/matrix/src/matrix/format-profile.ts @@ -0,0 +1,83 @@ +// Matrix helper module declares formatting capabilities and shared projections. +import type { MarkdownTableMode } from "openclaw/plugin-sdk/config-contracts"; +import { + convertMarkdownTables, + type FormatCapabilityProfile, + renderMarkdownWithMarkers, +} from "openclaw/plugin-sdk/text-chunking"; + +export type MatrixSpoilerMarkers = { open: string; close: string; padding: string }; +export type MatrixSpoilerProtection = { markdown: string; markers?: MatrixSpoilerMarkers }; + +export function createMatrixPrivateMarkers( + markdown: string, + exhaustedMessage: string, +): MatrixSpoilerMarkers { + const used = new Set(Array.from(markdown, (character) => character.charCodeAt(0))); + for (const match of markdown.matchAll(/&#(?:x([0-9a-f]+)|(\d+));/giu)) { + const radix = match[1] ? 16 : 10; + const value = Number.parseInt(match[1] ?? match[2] ?? "", radix); + if (Number.isFinite(value) && value <= 0xffff) { + used.add(value); + } + } + const markers: string[] = []; + for (let code = 0xe000; code <= 0xf8ff && markers.length < 3; code += 1) { + if (!used.has(code)) { + markers.push(String.fromCharCode(code)); + } + } + if (markers.length < 3) { + throw new Error(exhaustedMessage); + } + return { open: markers[0] ?? "", close: markers[1] ?? "", padding: markers[2] ?? "" }; +} + +export const MATRIX_FORMAT_PROFILE = { + mechanism: "html", + constructs: { + bold: "native", + italic: "native", + underline: "native", + strikethrough: "native", + spoiler: "native", + codeInline: "native", + codeBlock: "native", + codeLanguage: "native", + linkLabel: "native", + heading: "native", + bulletList: "native", + orderedList: "native", + taskList: "fallback", + table: "native", + blockquote: "native", + image: "fallback", + mention: "native", + }, + chunk: { limit: 4_000, unit: "chars" }, +} satisfies FormatCapabilityProfile; + +export function isMarkdownEscaped(markdown: string, index: number): boolean { + let slashCount = 0; + let cursor = index - 1; + while (cursor >= 0 && markdown[cursor] === "\\") { + slashCount += 1; + cursor -= 1; + } + return slashCount % 2 === 1; +} + +export function projectMatrixMarkdown(markdown: string): string { + const normalized = (markdown ?? "").replace(/\r\n?/gu, "\n"); + return renderMarkdownWithMarkers( + { text: normalized, styles: [], links: [] }, + { styleMarkers: {}, escapeText: (text) => text }, + MATRIX_FORMAT_PROFILE, + ); +} + +export function renderMatrixMarkdownTables(markdown: string, mode: MarkdownTableMode): string { + const useNativeTable = + MATRIX_FORMAT_PROFILE.constructs.table === "native" && (mode === "off" || mode === "block"); + return useNativeTable ? markdown : convertMarkdownTables(markdown, mode); +} diff --git a/extensions/matrix/src/matrix/format-spoiler-ranges.ts b/extensions/matrix/src/matrix/format-spoiler-ranges.ts new file mode 100644 index 000000000000..d54ffbffa4b6 --- /dev/null +++ b/extensions/matrix/src/matrix/format-spoiler-ranges.ts @@ -0,0 +1,212 @@ +// Matrix helper module resolves spoiler delimiters in ordinary Markdown inline blocks. +import MarkdownIt from "markdown-it"; +import { findCodeRegions, isInsideCode, tokenizeHtmlTags } from "openclaw/plugin-sdk/text-chunking"; +import { isMarkdownEscaped, projectMatrixMarkdown } from "./format-profile.js"; +import { findMatrixTableSourceRanges } from "./format-table-ranges.js"; + +const spoilerParser = new MarkdownIt({ html: false, linkify: true, typographer: false }); + +function findInlineMetadataRanges( + markdown: string, + references: ReadonlySet, +): Array<{ start: number; end: number }> { + const ranges: Array<{ start: number; end: number }> = []; + const labelStack: number[] = []; + const codeRegions = findCodeRegions(markdown); + const underlineTags = [...tokenizeHtmlTags(markdown)].filter( + (tag) => tag.name === "u" || tag.name === "ins", + ); + for (let index = 0; index < markdown.length - 1; index += 1) { + const underlineTag = underlineTags.find((tag) => tag.start === index); + if (underlineTag) { + index = underlineTag.end - 1; + continue; + } + if (isInsideCode(index, codeRegions)) { + continue; + } + if (markdown[index] === "\n" && markdown[index + 1] === "\n") { + labelStack.length = 0; + continue; + } + if (markdown[index] === "[" && !isMarkdownEscaped(markdown, index)) { + labelStack.push(index); + continue; + } + if ( + markdown[index] === "]" && + markdown[index + 1] === "(" && + !isMarkdownEscaped(markdown, index) && + labelStack.pop() !== undefined + ) { + const start = index + 2; + let cursor = start; + while (/[\s]/u.test(markdown[cursor] ?? "")) { + cursor += 1; + } + const destination = spoilerParser.helpers.parseLinkDestination( + markdown, + cursor, + markdown.length, + ); + if (destination.ok) { + cursor = destination.pos; + while (/[\s]/u.test(markdown[cursor] ?? "")) { + cursor += 1; + } + const title = spoilerParser.helpers.parseLinkTitle(markdown, cursor, markdown.length); + if (title.ok) { + cursor = title.pos; + while (/[\s]/u.test(markdown[cursor] ?? "")) { + cursor += 1; + } + } + } + if (destination.ok && markdown[cursor] === ")") { + ranges.push({ start, end: cursor + 1 }); + index = cursor; + } + continue; + } + if ( + markdown[index] === "]" && + markdown[index + 1] === "[" && + !isMarkdownEscaped(markdown, index) && + labelStack.pop() !== undefined + ) { + let end = index + 2; + while (end < markdown.length && (markdown[end] !== "]" || isMarkdownEscaped(markdown, end))) { + end += 1; + } + const reference = spoilerParser.utils.normalizeReference(markdown.slice(index + 2, end)); + if (end < markdown.length && references.has(reference)) { + ranges.push({ start: index + 2, end }); + index = end; + } + continue; + } + if (markdown[index] === "]" && !isMarkdownEscaped(markdown, index)) { + labelStack.pop(); + } + const autolink = /^<[A-Za-z][A-Za-z0-9+.-]{1,31}:[^<>\s]*>/u.exec(markdown.slice(index)); + if (autolink && !isMarkdownEscaped(markdown, index)) { + ranges.push({ start: index, end: index + autolink[0].length }); + index += autolink[0].length - 1; + continue; + } + const emailAutolink = /^<[^<>\s@]+@[^<>\s@]+>/u.exec(markdown.slice(index)); + if (emailAutolink && !isMarkdownEscaped(markdown, index)) { + ranges.push({ start: index, end: index + emailAutolink[0].length }); + index += emailAutolink[0].length - 1; + } + } + return ranges; +} + +export function findMatrixMarkdownMetadataRanges( + markdown: string, +): Array<{ start: number; end: number }> { + const env: { references?: Record } = {}; + const tokens = spoilerParser.parse(markdown, env); + const references = new Set(Object.keys(env.references ?? {})); + const lineStarts = [0]; + for (let index = 0; index < markdown.length; index += 1) { + if (markdown[index] === "\n") { + lineStarts.push(index + 1); + } + } + lineStarts.push(markdown.length); + const ranges = tokens.flatMap((token) => { + if (token.type !== "inline" || !token.map) { + return []; + } + const start = lineStarts[token.map[0]] ?? 0; + const end = lineStarts[token.map[1]] ?? markdown.length; + return findInlineMetadataRanges(markdown.slice(start, end), references).map((range) => ({ + start: start + range.start, + end: start + range.end, + })); + }); + for (const match of markdown.matchAll(/^\s*\[[^\]\n]+\]:\s*.+$/gmu)) { + const start = match.index ?? 0; + const labelEnd = match[0].indexOf("]:"); + const reference = spoilerParser.utils.normalizeReference(match[0].slice(1, labelEnd)); + if (references.has(reference)) { + let end = start + match[0].length; + const continuation = /^\n[ \t]+(?:"[^"\n]*"|'[^'\n]*'|\([^\n)]*\))[ \t]*/u.exec( + markdown.slice(end), + ); + end += continuation?.[0].length ?? 0; + ranges.push({ start, end }); + } + } + const codeRegions = findCodeRegions(markdown); + for (const match of spoilerParser.linkify.match(markdown) ?? []) { + if (!isInsideCode(match.index, codeRegions)) { + ranges.push({ start: match.index, end: match.lastIndex }); + } + } + return ranges; +} + +export function findMatrixSpoilerDelimiterOffsets(markdown: string): number[] { + const projected = projectMatrixMarkdown(markdown); + const tokens = spoilerParser.parse(projected, {}); + const lineStarts = [0]; + for (let index = 0; index < projected.length; index += 1) { + if (projected[index] === "\n") { + lineStarts.push(index + 1); + } + } + lineStarts.push(projected.length); + const excludedRanges = [ + ...findCodeRegions(projected), + ...findMatrixMarkdownMetadataRanges(projected), + ...[...tokenizeHtmlTags(projected)].flatMap((tag) => + tag.name === "u" || tag.name === "ins" ? [{ start: tag.start, end: tag.end }] : [], + ), + ]; + const offsets: number[] = []; + for (const token of tokens) { + // Table-cell inline tokens have no source map because pipes belong to GFM table grammar. + if (token.type !== "inline" || !token.map) { + continue; + } + const start = lineStarts[token.map[0]] ?? 0; + const end = lineStarts[token.map[1]] ?? projected.length; + const candidates: number[] = []; + for (let index = start; index < end - 1; index += 1) { + if (projected[index] !== "|" || projected[index + 1] !== "|") { + continue; + } + const excluded = excludedRanges.some((range) => index >= range.start && index < range.end); + if (isMarkdownEscaped(projected, index) || excluded) { + continue; + } + candidates.push(index); + index += 1; + } + candidates.length -= candidates.length % 2; + offsets.push(...candidates); + } + return [...new Set(offsets)].toSorted((left, right) => left - right); +} + +export function hasMatrixSpoilerMetadataCollision(markdown: string): boolean { + const projected = projectMatrixMarkdown(markdown); + const ordinary = new Set(findMatrixSpoilerDelimiterOffsets(projected)); + const tables = findMatrixTableSourceRanges(projected); + for (let index = 0; index < projected.length - 1; index += 1) { + if (projected[index] !== "|" || projected[index + 1] !== "|") { + continue; + } + if (ordinary.has(index) || isMarkdownEscaped(projected, index)) { + continue; + } + if (tables.some((range) => index >= range.start && index < range.end)) { + continue; + } + return true; + } + return false; +} diff --git a/extensions/matrix/src/matrix/format-table-ranges.ts b/extensions/matrix/src/matrix/format-table-ranges.ts new file mode 100644 index 000000000000..e6ddab4243a7 --- /dev/null +++ b/extensions/matrix/src/matrix/format-table-ranges.ts @@ -0,0 +1,23 @@ +// Matrix helper module resolves source ranges owned by Markdown table blocks. +import MarkdownIt from "markdown-it"; + +const tableParser = new MarkdownIt({ html: false, linkify: false, typographer: false }); + +export function findMatrixTableSourceRanges( + markdown: string, +): Array<{ start: number; end: number }> { + const lineStarts = [0]; + for (let index = 0; index < markdown.length; index += 1) { + if (markdown[index] === "\n") { + lineStarts.push(index + 1); + } + } + lineStarts.push(markdown.length); + return tableParser.parse(markdown, {}).flatMap((token) => { + if (token.type !== "table_open" || !token.map) { + return []; + } + const start = lineStarts[token.map[0]] ?? 0; + return [{ start, end: lineStarts[token.map[1]] ?? markdown.length }]; + }); +} diff --git a/extensions/matrix/src/matrix/format.test.ts b/extensions/matrix/src/matrix/format.test.ts index 5c888be28ad9..0bcec7ecd98a 100644 --- a/extensions/matrix/src/matrix/format.test.ts +++ b/extensions/matrix/src/matrix/format.test.ts @@ -1,6 +1,13 @@ // Matrix tests cover format plugin behavior. import { describe, expect, it } from "vitest"; -import { markdownToMatrixHtml, renderMarkdownToMatrixHtmlWithMentions } from "./format.js"; +import { findMatrixSpoilerDelimiterOffsets } from "./format-spoiler-ranges.js"; +import { + MATRIX_FORMAT_PROFILE, + markdownToMatrixBody, + markdownToMatrixHtml, + renderMarkdownToMatrixHtmlWithMentions, + renderMatrixMarkdownTables, +} from "./format.js"; function createMentionClient(selfUserId = "@bot:example.org") { return { @@ -8,6 +15,184 @@ function createMentionClient(selfUserId = "@bot:example.org") { } as unknown as import("./sdk.js").MatrixClient; } +const MATRIX_FORMAT_GOLDENS = [ + { + name: "spoiler", + markdown: "before ||secret|| after", + previousHtml: "

before ||secret|| after

", + html: "

before secret after

", + body: "before [Spoiler] after", + }, + { + name: "authored underline", + markdown: "under and inserted", + previousHtml: "

<u>under</u> and <ins>inserted</ins>

", + html: "

under and inserted

", + body: "under and inserted", + }, + { + name: "native table", + markdown: "| Name | Age |\n|---|---|\n| Alice | 30 |", + previousHtml: "

Alice
\n• Age: 30

", + html: "\n\n\n\n\n\n\n\n\n\n\n\n\n
NameAge
Alice30
", + body: "| Name | Age |\n|---|---|\n| Alice | 30 |", + }, +] as const; + +describe("Matrix formatting migration goldens", () => { + for (const golden of MATRIX_FORMAT_GOLDENS) { + it(`${golden.name}: emits the authorized before-to-after payload`, () => { + expect(markdownToMatrixHtml(golden.markdown)).toBe(golden.html); + expect(markdownToMatrixBody(golden.markdown)).toBe(golden.body); + expect(golden.html).not.toBe(golden.previousHtml); + }); + } + + it("declares the Matrix HTML profile and keeps explicit table fallbacks", () => { + expect(MATRIX_FORMAT_PROFILE).toMatchObject({ + mechanism: "html", + constructs: { spoiler: "native", underline: "native", table: "native" }, + chunk: { limit: 4_000, unit: "chars" }, + }); + expect(renderMatrixMarkdownTables(MATRIX_FORMAT_GOLDENS[2].markdown, "block")).toBe( + MATRIX_FORMAT_GOLDENS[2].markdown, + ); + expect(renderMatrixMarkdownTables(MATRIX_FORMAT_GOLDENS[2].markdown, "bullets")).toBe( + "**Alice**\n• Age: 30", + ); + expect( + markdownToMatrixHtml(MATRIX_FORMAT_GOLDENS[2].markdown, { tableMode: "off" }), + ).not.toContain(""); + }); + + it("keeps escaped literal pipes separate from a following spoiler", () => { + const markdown = "\\|\\| literal ||secret||"; + expect(markdownToMatrixHtml(markdown)).toBe( + "

|| literal secret

", + ); + expect(markdownToMatrixBody(markdown)).toBe("|| literal [Spoiler]"); + }); + + it("does not treat pipes in link destinations as spoiler delimiters", () => { + const markdown = "[docs\nmore](https://example.test/a(b)||literal||) ||secret||"; + expect(findMatrixSpoilerDelimiterOffsets(markdown)).toEqual([ + markdown.indexOf("||secret||"), + markdown.lastIndexOf("||"), + ]); + const html = markdownToMatrixHtml(markdown); + expect(html).not.toContain("secret"); + }); + + it("recognizes an overlapping spoiler opener after an escaped pipe", () => { + const markdown = "\\|||secret||"; + expect(markdownToMatrixHtml(markdown)).toBe("

|secret

"); + expect(markdownToMatrixBody(markdown)).toBe("|[Spoiler]"); + }); + + it("pairs spoilers across a soft line break within one paragraph", () => { + const markdown = "before ||first\nsecond|| after"; + expect(markdownToMatrixHtml(markdown)).toContain( + "first
\nsecond", + ); + expect(markdownToMatrixBody(markdown)).toBe("before [Spoiler] after"); + }); + + it("does not mistake an escaped closing bracket for a link label", () => { + const markdown = "\\](||secret||)"; + expect(markdownToMatrixHtml(markdown)).toContain("secret"); + expect(markdownToMatrixBody(markdown)).not.toContain("secret"); + }); + + it("does not reuse a completed link label for later visible text", () => { + const markdown = "[x](https://example.test) then ](||secret||)"; + expect(markdownToMatrixHtml(markdown)).toContain("](secret)"); + expect(markdownToMatrixBody(markdown)).not.toContain("secret"); + }); + + it("excludes spoiler-looking pipes in valid link titles", () => { + const markdown = '[x](https://example.test "note ) ||literal||") ||secret||'; + const html = markdownToMatrixHtml(markdown); + expect(html).not.toContain("secret"); + expect(markdownToMatrixBody(markdown)).not.toContain("secret"); + }); + + it("scopes link metadata to blocks and preserves reference identifiers", () => { + const stale = "[unfinished\n \n](||secret||)"; + expect(markdownToMatrixHtml(stale)).toContain("secret"); + expect(markdownToMatrixBody(stale)).not.toContain("secret"); + + const reference = "[visible][id||x||]\n\n[id||x||]: https://example.test"; + expect(markdownToMatrixHtml(reference)).not.toContain("secret"); + }); + + it("keeps invalid autolinks and code-span brackets in visible spoiler parsing", () => { + const invalidAutolink = ""; + expect(markdownToMatrixBody(invalidAutolink)).not.toContain("secret"); + + const codeBracket = "[x `]`](https://example.test/a||b) ||secret||"; + expect(markdownToMatrixHtml(codeBracket)).not.toContain("secret"); + expect(markdownToMatrixBody(codeBracket)).not.toContain("secret"); + }); + + it("finds unescaped ends of reference identifiers", () => { + const markdown = "[x][id\\]||x] then ||secret||\n\n[id\\]||x]: https://example.test"; + expect(markdownToMatrixHtml(markdown)).not.toContain("secret"); + expect(markdownToMatrixBody(markdown)).not.toContain("secret"); + }); + + it("keeps spoiler formatting inside image fallback labels", () => { + const markdown = "![||secret||](https://example.test/image.png)"; + expect(markdownToMatrixHtml(markdown)).toContain("secret"); + expect(markdownToMatrixBody(markdown)).not.toContain("secret"); + }); + + it("keeps spoiler spans nested when they cross bold formatting", () => { + const markdown = "**||secret** more||"; + const html = markdownToMatrixHtml(markdown); + expect(html).not.toContain(" more"); + expect(markdownToMatrixBody(markdown)).not.toContain("secret"); + }); + + it("follows parsed autolink and resolved-reference metadata", () => { + const autolink = " ||secret||"; + expect(markdownToMatrixHtml(autolink)).not.toContain("secret"); + + const unresolved = "[x][missing||secret||]"; + expect(markdownToMatrixHtml(unresolved)).toContain("secret"); + + const invalidDefinition = "[id]: ||secret||"; + expect(markdownToMatrixHtml(invalidDefinition)).toContain( + "secret", + ); + }); + + it("excludes bare linkified URLs and underline tag attributes", () => { + const bare = "https://example.test/a||literal|| then ||secret||"; + expect(markdownToMatrixHtml(bare)).not.toContain("secret"); + expect(markdownToMatrixBody(bare)).not.toContain("secret"); + + const underline = 'text then ||secret||'; + expect(markdownToMatrixHtml(underline)).not.toContain("secret"); + expect(markdownToMatrixBody(underline)).not.toContain("secret"); + }); + + it("leaves compact empty-cell pipes to native table grammar", () => { + const markdown = "| A | B | C |\n|---|---|---|\n| x || y || z |"; + expect(findMatrixSpoilerDelimiterOffsets(markdown)).toEqual([]); + expect(markdownToMatrixHtml(markdown)).toContain("
"); + expect(markdownToMatrixBody(markdown)).toBe(markdown); + }); + + it("fails closed when every private marker is already present", () => { + const privateUse = Array.from({ length: 0x1900 }, (_, index) => + String.fromCharCode(0xe000 + index), + ).join(""); + expect(() => markdownToMatrixHtml(`${privateUse} ||secret||`)).toThrow( + "exhausted its private marker pool", + ); + }); +}); + describe("markdownToMatrixHtml", () => { it("renders basic inline formatting", () => { const html = markdownToMatrixHtml("hi _there_ **boss** `code`"); diff --git a/extensions/matrix/src/matrix/format.ts b/extensions/matrix/src/matrix/format.ts index 3ecae3aa3e94..b7fbd458aca4 100644 --- a/extensions/matrix/src/matrix/format.ts +++ b/extensions/matrix/src/matrix/format.ts @@ -1,10 +1,32 @@ // Matrix helper module supports format behavior. import MarkdownIt from "markdown-it"; +import type { MarkdownTableMode } from "openclaw/plugin-sdk/config-contracts"; import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime"; import { isAutoLinkedFileRef } from "openclaw/plugin-sdk/text-autolink-runtime"; +import { + markdownToIR, + renderMarkdownWithMarkers, + tokenizeHtmlTags, +} from "openclaw/plugin-sdk/text-chunking"; +import { + createMatrixPrivateMarkers, + isMarkdownEscaped, + MATRIX_FORMAT_PROFILE, + projectMatrixMarkdown, +} from "./format-profile.js"; +import type { MatrixSpoilerMarkers, MatrixSpoilerProtection } from "./format-profile.js"; +import { + findMatrixSpoilerDelimiterOffsets, + hasMatrixSpoilerMetadataCollision, +} from "./format-spoiler-ranges.js"; import type { MatrixClient } from "./sdk.js"; import { isMatrixQualifiedUserId } from "./target-ids.js"; +export { MATRIX_FORMAT_PROFILE, renderMatrixMarkdownTables } from "./format-profile.js"; +const MATRIX_STYLE_MARKERS = { + underline: { open: "", close: "" }, + spoiler: { open: "", close: "" }, +} as const; const md = new MarkdownIt({ html: false, linkify: true, @@ -23,6 +45,7 @@ export type MatrixMentions = { type MarkdownToken = ReturnType[number]; type MarkdownInlineToken = NonNullable[number]; +type MarkdownInlineRule = Parameters[2]; type MatrixMentionCandidate = { raw: string; start: number; @@ -40,6 +63,48 @@ const MATRIX_MENTION_USER_ID_PATTERN = new RegExp( ); const TRIMMABLE_MENTION_SUFFIX = /[),.!?:;\]]/; +const parseMatrixUnderline: MarkdownInlineRule = (state, silent) => { + if (state.src.charCodeAt(state.pos) !== 0x3c) { + return false; + } + const tag = tokenizeHtmlTags(state.src.slice(state.pos)).next().value; + if (!tag || tag.start !== 0 || (tag.name !== "u" && tag.name !== "ins")) { + return false; + } + if (!silent) { + const token = state.push( + tag.selfClosing ? "text" : tag.closing ? "matrix_underline_close" : "matrix_underline_open", + tag.selfClosing ? "" : "u", + tag.selfClosing ? 0 : tag.closing ? -1 : 1, + ); + if (tag.selfClosing) { + token.content = tag.raw; + } + } + state.pos += tag.end; + return true; +}; + +md.inline.ruler.before("html_inline", "matrix_underline", parseMatrixUnderline); +md.renderer.rules.matrix_underline_open = () => MATRIX_STYLE_MARKERS.underline.open; +md.renderer.rules.matrix_underline_close = () => MATRIX_STYLE_MARKERS.underline.close; +md.renderer.rules.matrix_spoiler_open = () => MATRIX_STYLE_MARKERS.spoiler.open; +md.renderer.rules.matrix_spoiler_close = () => MATRIX_STYLE_MARKERS.spoiler.close; +md.core.ruler.after("inline", "matrix_spoilers", (state) => { + const markers = (state.env as { matrixSpoilerMarkers?: MatrixSpoilerMarkers }) + .matrixSpoilerMarkers; + if (!markers) { + return; + } + for (const token of state.tokens as MarkdownToken[]) { + if (token.children?.length) { + token.children = normalizeMatrixSpoilerNesting( + injectProtectedMatrixSpoilers(token.children, markers), + ); + } + } +}); + function shouldSuppressAutoLink( tokens: Parameters>[0], idx: number, @@ -53,7 +118,12 @@ function shouldSuppressAutoLink( return Boolean(href && label && isAutoLinkedFileRef(href, label)); } -md.renderer.rules.image = (tokens, idx) => escapeHtml(tokens[idx]?.content ?? ""); +md.renderer.rules.image = (tokens, idx, options, env, self) => { + const token = tokens[idx]; + return token?.children?.length + ? self.renderInline(token.children, options, env) + : escapeHtml(token?.content ?? ""); +}; md.renderer.rules.html_block = (tokens, idx) => escapeHtml(tokens[idx]?.content ?? ""); md.renderer.rules.html_inline = (tokens, idx) => escapeHtml(tokens[idx]?.content ?? ""); @@ -99,16 +169,6 @@ function maskEscapedMentions(markdown: string): string { return masked; } -function isMarkdownEscaped(markdown: string, idx: number): boolean { - let slashCount = 0; - let cursor = idx - 1; - while (cursor >= 0 && markdown[cursor] === "\\") { - slashCount += 1; - cursor -= 1; - } - return slashCount % 2 === 1; -} - function restoreEscapedMentions(text: string): string { return text.replaceAll(ESCAPED_MENTION_SENTINEL, "@"); } @@ -216,6 +276,100 @@ function createTextToken(sample: MarkdownInlineToken, content: string): Markdown return token; } +function injectProtectedMatrixSpoilers( + tokens: MarkdownInlineToken[], + markers: MatrixSpoilerMarkers, +): MarkdownInlineToken[] { + const result: MarkdownInlineToken[] = []; + for (const token of tokens) { + if (token.type !== "text") { + if (token.children?.length) { + token.children = normalizeMatrixSpoilerNesting( + injectProtectedMatrixSpoilers(token.children, markers), + ); + } + result.push(token); + continue; + } + let cursor = 0; + for (let index = 0; index < token.content.length; index += 1) { + const marker = token.content[index]; + if ( + (marker !== markers.open && marker !== markers.close) || + token.content[index + 1] !== markers.padding + ) { + continue; + } + if (index > cursor) { + result.push(createTextToken(token, token.content.slice(cursor, index))); + } + result.push( + createToken( + token, + marker === markers.open ? "matrix_spoiler_open" : "matrix_spoiler_close", + "span", + marker === markers.open ? 1 : -1, + ), + ); + index += 1; + cursor = index + 1; + } + if (cursor < token.content.length) { + result.push(createTextToken(token, token.content.slice(cursor))); + } + } + return result; +} + +function copyInlineToken( + sample: MarkdownInlineToken, + type: string, + tag: string, + nesting: number, +): MarkdownInlineToken { + const token = createToken(sample, type, tag, nesting); + token.markup = sample.markup; + token.attrs = sample.attrs ? [...sample.attrs] : null; + return token; +} + +function normalizeMatrixSpoilerNesting(tokens: MarkdownInlineToken[]): MarkdownInlineToken[] { + const result: MarkdownInlineToken[] = []; + const stack: MarkdownInlineToken[] = []; + for (const token of tokens) { + if (token.nesting === 1) { + stack.push(token); + result.push(token); + continue; + } + if (token.nesting !== -1) { + result.push(token); + continue; + } + const openIndex = stack.findLastIndex((open) => open.tag === token.tag); + if (openIndex < 0) { + result.push(token); + continue; + } + if (openIndex === stack.length - 1) { + stack.pop(); + result.push(token); + continue; + } + const crossing = stack.splice(openIndex + 1); + for (const open of crossing.toReversed()) { + result.push(copyInlineToken(open, open.type.replace(/_open$/u, "_close"), open.tag, -1)); + } + stack.pop(); + result.push(token); + for (const open of crossing) { + result.push(copyInlineToken(open, open.type, open.tag, 1)); + stack.push(open); + } + } + return result; +} + function createMentionLinkTokens(params: { sample: MarkdownInlineToken; href: string; @@ -374,18 +528,96 @@ function compactLooseListTokens(tokens: MarkdownToken[]): void { } } -export function markdownToMatrixHtml(markdown: string): string { - const tokens = md.parse(markdown ?? "", {}); +export function markdownToMatrixHtml( + markdown: string, + options: { tableMode?: MarkdownTableMode } = {}, +): string { + if (hasMatrixSpoilerMetadataCollision(markdown)) { + return renderMatrixFallbackHtml(markdown); + } + const tokens = parseMatrixMarkdown(projectMatrixMarkdown(markdown), options.tableMode); compactLooseListTokens(tokens); return md.renderer.render(tokens, md.options, {}).trimEnd(); } +export function protectMatrixSpoilerDelimiters(markdown: string): MatrixSpoilerProtection { + const offsets = findMatrixSpoilerDelimiterOffsets(markdown); + if (offsets.length === 0) { + return { markdown }; + } + const markers = createMatrixPrivateMarkers( + markdown, + "Matrix spoiler formatting exhausted its private marker pool", + ); + let protectedMarkdown = ""; + let cursor = 0; + for (const [index, offset] of offsets.entries()) { + const marker = index % 2 === 0 ? markers.open : markers.close; + protectedMarkdown += `${markdown.slice(cursor, offset)}${marker}${markers.padding}`; + cursor = offset + 2; + } + protectedMarkdown += markdown.slice(cursor); + return { markdown: protectedMarkdown, markers }; +} + +function parseMatrixMarkdown(markdown: string, tableMode?: MarkdownTableMode): MarkdownToken[] { + const protectedSpoilers = protectMatrixSpoilerDelimiters(markdown); + if (tableMode === "off") { + md.disable("table"); + } + try { + return md.parse(protectedSpoilers.markdown, { + matrixSpoilerMarkers: protectedSpoilers.markers, + }); + } finally { + if (tableMode === "off") { + md.enable("table"); + } + } +} + +export function markdownToMatrixBody(markdown: string): string { + const projected = projectMatrixMarkdown(markdown); + const offsets = findMatrixSpoilerDelimiterOffsets(projected); + const metadataCollision = hasMatrixSpoilerMetadataCollision(projected); + if (offsets.length === 0 && !metadataCollision) { + return projected; + } + let body = projected; + if (metadataCollision) { + body = "[Spoiler]"; + } else { + for (let index = offsets.length - 2; index >= 0; index -= 2) { + const open = offsets[index]; + const close = offsets[index + 1]; + if (open !== undefined && close !== undefined) { + body = `${body.slice(0, open)}[Spoiler]${body.slice(close + 2)}`; + } + } + } + const ir = markdownToIR(body, { + enableHtmlUnderline: true, + headingStyle: "rich", + linkify: true, + }); + return renderMarkdownWithMarkers( + ir, + { styleMarkers: {}, escapeText: (text) => text }, + MATRIX_FORMAT_PROFILE, + ); +} + +function renderMatrixFallbackHtml(markdown: string): string { + return `

${escapeHtml(markdownToMatrixBody(markdown)).replaceAll("\n", "
\n")}

`; +} + async function resolveMarkdownMentionState(params: { markdown: string; client: MatrixClient; + tableMode?: MarkdownTableMode; }): Promise<{ tokens: MarkdownToken[]; mentions: MatrixMentions }> { - const markdown = maskEscapedMentions(params.markdown ?? ""); - const tokens = md.parse(markdown, {}); + const markdown = maskEscapedMentions(projectMatrixMarkdown(params.markdown)); + const tokens = parseMatrixMarkdown(markdown, params.tableMode); restoreEscapedMentionsInBlockTokens(tokens); const selfUserId = await resolveMatrixSelfUserId(params.client); const userIds: string[] = []; @@ -430,8 +662,17 @@ export async function resolveMatrixMentionsInMarkdown(params: { export async function renderMarkdownToMatrixHtmlWithMentions(params: { markdown: string; client: MatrixClient; + tableMode?: MarkdownTableMode; }): Promise<{ html?: string; mentions: MatrixMentions }> { const state = await resolveMarkdownMentionState(params); + if (hasMatrixSpoilerMetadataCollision(params.markdown)) { + const redacted = markdownToMatrixBody(params.markdown); + const redactedState = await resolveMarkdownMentionState({ + ...params, + markdown: redacted, + }); + return { html: renderMatrixFallbackHtml(params.markdown), mentions: redactedState.mentions }; + } compactLooseListTokens(state.tokens); const html = md.renderer.render(state.tokens, md.options, {}).trimEnd(); return { diff --git a/extensions/matrix/src/matrix/send.test.ts b/extensions/matrix/src/matrix/send.test.ts index 77cf7dff818f..3dde608b1620 100644 --- a/extensions/matrix/src/matrix/send.test.ts +++ b/extensions/matrix/src/matrix/send.test.ts @@ -3,7 +3,9 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { PluginRuntime } from "../../runtime-api.js"; import { setMatrixRuntime } from "../runtime.js"; import { voteMatrixPoll } from "./actions/polls.js"; +import { markdownToMatrixBody, markdownToMatrixHtml } from "./format.js"; import { + chunkMatrixText, editMessageMatrix, sendMessageMatrix, sendPollMatrix, @@ -30,9 +32,10 @@ const isVoiceCompatibleAudioMock = vi.fn( const resolveTextChunkLimitMock = vi.fn< (cfg: unknown, channel: unknown, accountId?: unknown) => number >(() => 4000); -const resolveMarkdownTableModeMock = vi.fn(() => "code"); -const convertMarkdownTablesMock = vi.fn((text: string) => text); -const chunkMarkdownTextWithModeMock = vi.fn((text: string) => (text ? [text] : [])); +const resolveMarkdownTableModeMock = vi.fn((_params?: unknown) => "code"); +const chunkMarkdownTextWithModeMock = vi.fn< + (text: string, limit?: number, mode?: unknown) => string[] +>((text) => (text ? [text] : [])); vi.mock("openclaw/plugin-sdk/plugin-config-runtime", async () => { const actual = await vi.importActual( @@ -70,9 +73,10 @@ const runtimeStub = { resolveTextChunkLimitMock(cfg, channel, accountId), resolveChunkMode: () => "length", chunkMarkdownText: (text: string) => (text ? [text] : []), - chunkMarkdownTextWithMode: (text: string) => chunkMarkdownTextWithModeMock(text), - resolveMarkdownTableMode: () => resolveMarkdownTableModeMock(), - convertMarkdownTables: (text: string) => convertMarkdownTablesMock(text), + chunkMarkdownTextWithMode: (text: string, limit: number, mode: unknown) => + chunkMarkdownTextWithModeMock(text, limit, mode), + resolveMarkdownTableMode: (params: unknown) => resolveMarkdownTableModeMock(params), + convertMarkdownTables: (text: string) => text, }, }, } as unknown as PluginRuntime; @@ -167,6 +171,12 @@ function expectTextReceiptPart(value: unknown, platformMessageId: string) { expect(part.kind).toBe("text"); } +function splitTextAtLimit(text: string, limit = text.length): string[] { + return Array.from({ length: Math.ceil(text.length / limit) }, (_, index) => + text.slice(index * limit, (index + 1) * limit), + ); +} + function resetMatrixSendRuntimeMocks() { setMatrixRuntime(runtimeStub); loadOutboundMediaFromUrlMock.mockReset().mockImplementation( @@ -211,13 +221,196 @@ function resetMatrixSendRuntimeMocks() { isVoiceCompatibleAudioMock.mockReset().mockReturnValue(false); resolveTextChunkLimitMock.mockReset().mockReturnValue(4000); resolveMarkdownTableModeMock.mockReset().mockReturnValue("code"); - convertMarkdownTablesMock.mockReset().mockImplementation((text: string) => text); chunkMarkdownTextWithModeMock .mockReset() .mockImplementation((text: string) => (text ? [text] : [])); applyMatrixSendRuntimeStub(); } +describe("Matrix formatted chunk boundaries", () => { + beforeEach(() => { + resetMatrixSendRuntimeMocks(); + }); + + it("closes and reopens spoilers without exposing chunked secret text", () => { + const secret = "secret ".repeat(8).trim(); + resolveTextChunkLimitMock.mockReturnValue(20); + chunkMarkdownTextWithModeMock.mockImplementation(splitTextAtLimit); + + const { chunks } = chunkMatrixText(`before ||${secret}|| after`, { + cfg: {} as never, + tableMode: "block", + }); + + expect(chunks.length).toBeGreaterThan(1); + expect(chunks.every((chunk) => chunk.length <= 20)).toBe(true); + for (const chunk of chunks) { + expect(markdownToMatrixBody(chunk)).not.toContain("secret"); + expect(markdownToMatrixHtml(chunk)).not.toContain("||"); + } + }); + + it("does not pair an unmatched paragraph delimiter with a later spoiler", () => { + const secret = "secret ".repeat(8).trim(); + resolveTextChunkLimitMock.mockReturnValue(20); + chunkMarkdownTextWithModeMock.mockImplementation(splitTextAtLimit); + + const { chunks } = chunkMatrixText(`first ||\n\nsecond ||${secret}||`, { + cfg: {} as never, + tableMode: "block", + }); + + expect(chunks.join("")).toContain("[Spoiler]"); + expect(chunks.every((chunk) => !markdownToMatrixBody(chunk).includes("secret"))).toBe(true); + }); + + it("keeps a spoiler-bearing message whole when it already fits", () => { + const markdown = `||${"x".repeat(14)}||`; + resolveTextChunkLimitMock.mockReturnValue(20); + + expect(chunkMatrixText(markdown, { cfg: {} as never, tableMode: "block" }).chunks).toEqual([ + markdown, + ]); + expect(chunkMarkdownTextWithModeMock).not.toHaveBeenCalled(); + }); + + it("closes and reopens authored underline across chunk boundaries", () => { + const markdown = `${"underlined ".repeat(6).trim()}`; + resolveTextChunkLimitMock.mockReturnValue(20); + chunkMarkdownTextWithModeMock.mockImplementation(splitTextAtLimit); + + const { chunks } = chunkMatrixText(markdown, { cfg: {} as never, tableMode: "block" }); + + expect(chunks.length).toBeGreaterThan(1); + expect(chunks.every((chunk) => chunk.length <= 20)).toBe(true); + expect(chunks.every((chunk) => markdownToMatrixHtml(chunk).includes(""))).toBe(true); + }); + + it("keeps underline-looking tags inside code literal", () => { + const markdown = `\`\` \\ ${"plain ".repeat(8)}`; + resolveTextChunkLimitMock.mockReturnValue(20); + chunkMarkdownTextWithModeMock.mockImplementation(splitTextAtLimit); + + const { chunks } = chunkMatrixText(markdown, { cfg: {} as never, tableMode: "block" }); + + expect(chunks.join("")).toBe(markdown.trim()); + expect(chunks.join("")).not.toContain(""); + }); + + it("keeps underline-looking tags inside link metadata literal", () => { + const markdown = `[x](https://example.test "literal ") ${"plain ".repeat(8)}`; + resolveTextChunkLimitMock.mockReturnValue(28); + chunkMarkdownTextWithModeMock.mockImplementation(splitTextAtLimit); + + const { chunks } = chunkMatrixText(markdown, { cfg: {} as never, tableMode: "block" }); + + expect(chunks.join("")).toBe(markdown.trim()); + expect(chunks.join("")).not.toContain(""); + }); + + it("keeps nested underline depth across chunk boundaries", () => { + const markdown = `outer inner ${"tail ".repeat(8)}`; + resolveTextChunkLimitMock.mockReturnValue(24); + chunkMarkdownTextWithModeMock.mockImplementation(splitTextAtLimit); + + const { chunks } = chunkMatrixText(markdown, { cfg: {} as never, tableMode: "block" }); + + expect(chunks.length).toBeGreaterThan(1); + expect(chunks.every((chunk) => chunk.length <= 24)).toBe(true); + expect(chunks.every((chunk) => markdownToMatrixHtml(chunk).includes(""))).toBe(true); + }); + + it("drops padding-only chunks from long authored underline tags", () => { + const markdown = `content`; + resolveTextChunkLimitMock.mockReturnValue(20); + chunkMarkdownTextWithModeMock.mockImplementation(splitTextAtLimit); + + const { chunks } = chunkMatrixText(markdown, { cfg: {} as never, tableMode: "block" }); + + expect( + chunks.every((chunk) => chunk.replaceAll("", "").replaceAll("", "").trim().length > 0), + ).toBe(true); + }); + + it("keeps spoiler and underline nesting valid across chunks", () => { + const markdown = `||${"nested ".repeat(8).trim()}||`; + resolveTextChunkLimitMock.mockReturnValue(24); + chunkMarkdownTextWithModeMock.mockImplementation(splitTextAtLimit); + + const { chunks } = chunkMatrixText(markdown, { cfg: {} as never, tableMode: "block" }); + + expect(chunks.every((chunk) => chunk.length <= 24)).toBe(true); + expect( + chunks.every((chunk) => { + const html = markdownToMatrixHtml(chunk); + return html.includes("") && html.includes(""); + }), + ).toBe(true); + }); + + it("falls back to table bullets when a native table cannot fit one event", () => { + resolveTextChunkLimitMock.mockReturnValue(30); + chunkMarkdownTextWithModeMock.mockImplementation(splitTextAtLimit); + const prepared = chunkMatrixText( + "| Name | Description |\n|---|---|\n| Alice | a long description that crosses the limit |", + { cfg: {} as never, tableMode: "block" }, + ); + + const rendered = prepared.chunks.join("\n"); + expect(rendered).toContain("**Alice**"); + expect(rendered).toContain("• Description:"); + expect(rendered).toContain("description that crosses the"); + expect(rendered).not.toContain("|---|---|"); + }); + + it("keeps a small native table in a long message", () => { + const table = "| A | B |\n|---|---|\n| 1 | 2 |"; + resolveTextChunkLimitMock.mockReturnValue(40); + chunkMarkdownTextWithModeMock.mockImplementation(splitTextAtLimit); + + const { chunks } = chunkMatrixText(`${"prose ".repeat(10)}\n\n${table}`, { + cfg: {} as never, + tableMode: "block", + }); + + expect(chunks).toContain(table); + }); + + it("preserves indentation after a native table segment", () => { + const table = "| A | B |\n|---|---|\n| 1 | 2 |"; + const code = " indented code"; + resolveTextChunkLimitMock.mockReturnValue(40); + chunkMarkdownTextWithModeMock.mockImplementation(splitTextAtLimit); + + const { chunks } = chunkMatrixText(`${"prose ".repeat(10)}\n\n${table}\n\n${code}`, { + cfg: {} as never, + tableMode: "block", + }); + + expect(chunks).toContain(code); + }); + + it("recognizes aligned tables and ignores table examples inside fences", () => { + const aligned = "| A | B |\n| ---: | :---: |\n| 1 | 2 |\n| 3 | 4 |"; + resolveTextChunkLimitMock.mockReturnValue(35); + chunkMarkdownTextWithModeMock.mockImplementation(splitTextAtLimit); + expect( + chunkMatrixText(aligned, { cfg: {} as never, tableMode: "block" }).chunks.join("\n"), + ).toContain("• B:"); + + const fenced = `\`\`\`\n${aligned}\n\`\`\``; + expect(chunkMatrixText(fenced, { cfg: {} as never, tableMode: "block" }).chunks.join("")).toBe( + fenced, + ); + + const shortDivider = "A|B\n-| -\nbar"; + resolveTextChunkLimitMock.mockReturnValue(8); + expect( + chunkMatrixText(shortDivider, { cfg: {} as never, tableMode: "block" }).chunks.join("\n"), + ).toContain("**bar**"); + }); +}); + describe("sendMessageMatrix media", () => { beforeEach(() => { resetMatrixSendRuntimeMocks(); @@ -661,15 +854,15 @@ describe("sendMessageMatrix threads", () => { it("returns ordered event ids for chunked text sends", async () => { const { client, sendMessage } = makeClient(); + resolveTextChunkLimitMock.mockReturnValue(6); sendMessage .mockReset() .mockResolvedValueOnce("$m1") .mockResolvedValueOnce("$m2") .mockResolvedValueOnce("$m3"); - convertMarkdownTablesMock.mockImplementation(() => "part1|part2|part3"); chunkMarkdownTextWithModeMock.mockImplementation((text: string) => text.split("|")); - const result = await sendMessageMatrix("room:!room:example", "ignored", { + const result = await sendMessageMatrix("room:!room:example", "part1|part2|part3", { client, cfg: {} as never, }); @@ -687,16 +880,16 @@ describe("sendMessageMatrix threads", () => { it("reports the first Matrix event before a later event fails", async () => { const { client, sendMessage } = makeClient(); + resolveTextChunkLimitMock.mockReturnValue(5); sendMessage .mockReset() .mockResolvedValueOnce("$m1") .mockRejectedValueOnce(new Error("second event failed")); - convertMarkdownTablesMock.mockImplementation(() => "part1|part2"); chunkMarkdownTextWithModeMock.mockImplementation((text: string) => text.split("|")); const onDeliveryResult = vi.fn(); await expect( - sendMessageMatrix("room:!room:example", "ignored", { + sendMessageMatrix("room:!room:example", "part1|part2", { client, cfg: {} as never, onDeliveryResult, @@ -708,10 +901,10 @@ describe("sendMessageMatrix threads", () => { it("merges extra content into only the first chunked text event", async () => { const { client, sendMessage } = makeClient(); - convertMarkdownTablesMock.mockImplementation(() => "first|second|third"); + resolveTextChunkLimitMock.mockReturnValue(6); chunkMarkdownTextWithModeMock.mockImplementation((text: string) => text.split("|")); - await sendMessageMatrix("room:!room:example", "ignored", { + await sendMessageMatrix("room:!room:example", "first|second|third", { client, cfg: {} as never, extraContent: { "com.openclaw.approval": { id: "req-1" } }, @@ -733,13 +926,12 @@ describe("sendSingleTextMessageMatrix", () => { resetMatrixSendRuntimeMocks(); }); - it("rejects single-event sends when converted text exceeds the Matrix limit", async () => { + it("rejects single-event sends when rendered text exceeds the Matrix limit", async () => { const { client, sendMessage } = makeClient(); resolveTextChunkLimitMock.mockReturnValue(5); - convertMarkdownTablesMock.mockImplementation(() => "123456"); await expect( - sendSingleTextMessageMatrix("room:!room:example", "1234", { + sendSingleTextMessageMatrix("room:!room:example", "123456", { client, cfg: {} as never, }), @@ -748,6 +940,38 @@ describe("sendSingleTextMessageMatrix", () => { expect(sendMessage).not.toHaveBeenCalled(); }); + it("keeps native tables in the body and formatted body when the profile selects blocks", async () => { + const { client, sendMessage } = makeClient(); + const markdown = "| Name | Age |\n|---|---|\n| Alice | 30 |"; + resolveMarkdownTableModeMock.mockReturnValue("block"); + + await sendSingleTextMessageMatrix("room:!room:example", markdown, { + client, + cfg: {} as never, + }); + + const content = sentContent(sendMessage); + expect(content.body).toBe(markdown); + expect(content.formatted_body).toContain("
"); + expect(content.formatted_body).toContain(""); + expect(resolveMarkdownTableModeMock).toHaveBeenCalledWith( + expect.objectContaining({ channel: "matrix", supportsBlockTables: true }), + ); + }); + + it("keeps spoiler text out of the Matrix plain fallback", async () => { + const { client, sendMessage } = makeClient(); + + await sendSingleTextMessageMatrix("room:!room:example", "before ||secret|| after", { + client, + cfg: {} as never, + }); + + const content = sentContent(sendMessage); + expect(content.body).toBe("before [Spoiler] after"); + expect(content.formatted_body).toBe("

before secret after

"); + }); + it("supports quiet draft preview sends without mention metadata", async () => { const { client, sendMessage } = makeClient(); @@ -963,6 +1187,17 @@ describe("editMessageMatrix mentions", () => { expect(newContent(content)[MATRIX_OPENCLAW_FINALIZED_PREVIEW_KEY]).toBe(true); }); + it("preserves Markdown-significant indentation in edits", async () => { + const { client, sendMessage } = makeClient(); + + await editMessageMatrix("room:!room:example", "$original", " code", { + client, + cfg: {} as never, + }); + + expect(newContent(sentContent(sendMessage)).body).toBe(" code"); + }); + it("edits threaded originals with a pure replace relation", async () => { const { client, getEvent, sendMessage } = makeClient(); getEvent.mockResolvedValue({ diff --git a/extensions/matrix/src/matrix/send.ts b/extensions/matrix/src/matrix/send.ts index 458a3ff99b4d..07abe256e5cb 100644 --- a/extensions/matrix/src/matrix/send.ts +++ b/extensions/matrix/src/matrix/send.ts @@ -3,15 +3,14 @@ import { createMessageReceiptFromOutboundResults, type MessageReceiptPartKind, } from "openclaw/plugin-sdk/channel-outbound"; -import type { MarkdownTableMode } from "openclaw/plugin-sdk/markdown-table-runtime"; import { requireRuntimeConfig } from "openclaw/plugin-sdk/plugin-config-runtime"; import type { PollInput } from "../runtime-api.js"; -import { getMatrixRuntime } from "../runtime.js"; import type { CoreConfig } from "../types.js"; import { loadOutboundMediaFromUrl } from "./outbound-media-runtime.js"; import { buildPollStartContent, M_POLL_START } from "./poll-types.js"; import { buildMatrixReactionContent } from "./reaction-common.js"; import type { MatrixClient } from "./sdk.js"; +import { chunkMatrixText, prepareMatrixSingleText } from "./send/chunking.js"; import { resolveMediaMaxBytes, withResolvedMatrixControlClient, @@ -47,23 +46,10 @@ import { type MatrixTextMsgType, } from "./send/types.js"; -const MATRIX_TEXT_LIMIT = 4000; -const getCore = () => getMatrixRuntime(); - +export { chunkMatrixText, prepareMatrixSingleText } from "./send/chunking.js"; export { resolveMatrixMentionsForBody } from "./send/formatting.js"; export { resolveMatrixRoomId } from "./send/targets.js"; -type MatrixPreparedSingleText = { - trimmedText: string; - convertedText: string; - singleEventLimit: number; - fitsInSingleEvent: boolean; -}; - -type MatrixPreparedChunkedText = MatrixPreparedSingleText & { - chunks: string[]; -}; - type MatrixClientResolveOpts = { client?: MatrixClient; cfg?: CoreConfig; @@ -179,57 +165,6 @@ async function resolvePreviousEditMentions(params: { }); } -export function prepareMatrixSingleText( - text: string, - opts: { - cfg: CoreConfig; - accountId?: string; - tableMode?: MarkdownTableMode; - }, -): MatrixPreparedSingleText { - const trimmedText = text.trim(); - const cfg = requireRuntimeConfig(opts.cfg, "Matrix text preparation") as CoreConfig; - const tableMode = - opts.tableMode ?? - getCore().channel.text.resolveMarkdownTableMode({ - cfg, - channel: "matrix", - accountId: opts.accountId, - }); - const convertedText = getCore().channel.text.convertMarkdownTables(trimmedText, tableMode); - const singleEventLimit = Math.min( - getCore().channel.text.resolveTextChunkLimit(cfg, "matrix", opts.accountId), - MATRIX_TEXT_LIMIT, - ); - return { - trimmedText, - convertedText, - singleEventLimit, - fitsInSingleEvent: convertedText.length <= singleEventLimit, - }; -} - -export function chunkMatrixText( - text: string, - opts: { - cfg: CoreConfig; - accountId?: string; - tableMode?: MarkdownTableMode; - }, -): MatrixPreparedChunkedText { - const preparedText = prepareMatrixSingleText(text, opts); - const cfg = requireRuntimeConfig(opts.cfg, "Matrix text chunking") as CoreConfig; - const chunkMode = getCore().channel.text.resolveChunkMode(cfg, "matrix", opts.accountId); - return { - ...preparedText, - chunks: getCore().channel.text.chunkMarkdownTextWithMode( - preparedText.convertedText, - preparedText.singleEventLimit, - chunkMode, - ), - }; -} - export async function sendMessageMatrix( to: string, message: string | undefined, @@ -249,7 +184,7 @@ export async function sendMessageMatrix( async (client) => { const roomId = await resolveMatrixRoomId(client, to); const cfg = requireRuntimeConfig(opts.cfg, "Matrix send") as CoreConfig; - const { chunks } = chunkMatrixText(trimmedMessage, { + const { chunks, tableMode } = chunkMatrixText(trimmedMessage, { cfg, accountId: opts.accountId, }); @@ -336,6 +271,7 @@ export async function sendMessageMatrix( client, content, markdown: captionMarkdown, + tableMode, }); const eventId = await sendContent(content, receiptKind); lastMessageId = eventId ?? lastMessageId; @@ -347,8 +283,8 @@ export async function sendMessageMatrix( // transcript follow-up attached to the same reply/thread context. const followupRelation = useVoice || threadId ? relation : undefined; for (const chunk of textChunks) { - const text = chunk.trim(); - if (!text) { + const text = chunk; + if (!text.trim()) { continue; } const followup = buildTextContent(text, followupRelation); @@ -356,6 +292,7 @@ export async function sendMessageMatrix( client, content: followup, markdown: text, + tableMode, }); const followupEventId = await sendContent(followup, "text"); lastMessageId = followupEventId ?? lastMessageId; @@ -365,8 +302,8 @@ export async function sendMessageMatrix( } } else { for (const chunk of chunks.length ? chunks : [""]) { - const text = chunk.trim(); - if (!text) { + const text = chunk; + if (!text.trim()) { continue; } const content = buildTextContent(text, relation); @@ -374,6 +311,7 @@ export async function sendMessageMatrix( client, content, markdown: text, + tableMode, }); const eventId = await sendContent(content, "text"); lastMessageId = eventId ?? lastMessageId; @@ -499,17 +437,23 @@ export async function sendSingleTextMessageMatrix( live?: boolean; }, ): Promise { - const { trimmedText, convertedText, singleEventLimit, fitsInSingleEvent } = - prepareMatrixSingleText(text, { - cfg: opts.cfg, - accountId: opts.accountId, - }); + const { + trimmedText, + convertedText, + singleEventLimit, + eventTextLength, + fitsInSingleEvent, + tableMode, + } = prepareMatrixSingleText(text, { + cfg: opts.cfg, + accountId: opts.accountId, + }); if (!trimmedText) { throw new Error("Matrix single-message send requires text"); } if (!fitsInSingleEvent) { throw new Error( - `Matrix single-message text exceeds limit (${convertedText.length} > ${singleEventLimit})`, + `Matrix single-message text exceeds limit (${eventTextLength} > ${singleEventLimit})`, ); } return await withResolvedMatrixSendClient( @@ -535,6 +479,7 @@ export async function sendSingleTextMessageMatrix( content, markdown: convertedText, includeMentions: opts.includeMentions, + tableMode, }); // MSC4357: mark the initial message as live so supporting clients start // rendering a streaming animation immediately. @@ -602,12 +547,11 @@ export async function editMessageMatrix( async (client) => { const resolvedRoom = await resolveMatrixRoomId(client, roomId); const cfg = requireRuntimeConfig(opts.cfg, "Matrix message edit") as CoreConfig; - const tableMode = getCore().channel.text.resolveMarkdownTableMode({ + const { convertedText, tableMode } = prepareMatrixSingleText(newText, { cfg, - channel: "matrix", accountId: opts.accountId, + preserveWhitespace: true, }); - const convertedText = getCore().channel.text.convertMarkdownTables(newText, tableMode); const newContent = withMatrixExtraContentFields( buildTextContent(convertedText, undefined, { msgtype: opts.msgtype, @@ -619,6 +563,7 @@ export async function editMessageMatrix( content: newContent, markdown: convertedText, includeMentions: opts.includeMentions, + tableMode, }); const previousEvent = await getPreviousMatrixEvent(client, resolvedRoom, originalEventId); const replaceMentions = @@ -649,7 +594,7 @@ export async function editMessageMatrix( // m.new_content still see properly formatted text (with HTML). const content: Record = { ...newContent, - body: `* ${convertedText}`, + body: `* ${newContent.body}`, ...(typeof newContent.formatted_body === "string" ? { formatted_body: `* ${newContent.formatted_body}` } : {}), diff --git a/extensions/matrix/src/matrix/send/chunking.ts b/extensions/matrix/src/matrix/send/chunking.ts new file mode 100644 index 000000000000..98b0f86e0024 --- /dev/null +++ b/extensions/matrix/src/matrix/send/chunking.ts @@ -0,0 +1,274 @@ +// Matrix helper module prepares and chunks outbound formatted text. +import type { MarkdownTableMode } from "openclaw/plugin-sdk/markdown-table-runtime"; +import { requireRuntimeConfig } from "openclaw/plugin-sdk/plugin-config-runtime"; +import { findCodeRegions, isInsideCode, tokenizeHtmlTags } from "openclaw/plugin-sdk/text-chunking"; +import { getMatrixRuntime } from "../../runtime.js"; +import type { CoreConfig } from "../../types.js"; +import { + createMatrixPrivateMarkers, + isMarkdownEscaped, + type MatrixSpoilerMarkers, + type MatrixSpoilerProtection, +} from "../format-profile.js"; +import { + findMatrixMarkdownMetadataRanges, + hasMatrixSpoilerMetadataCollision, +} from "../format-spoiler-ranges.js"; +import { findMatrixTableSourceRanges } from "../format-table-ranges.js"; +import { + markdownToMatrixBody, + MATRIX_FORMAT_PROFILE, + protectMatrixSpoilerDelimiters, + renderMatrixMarkdownTables, +} from "../format.js"; + +type MatrixPreparedSingleText = { + trimmedText: string; + convertedText: string; + singleEventLimit: number; + eventTextLength: number; + fitsInSingleEvent: boolean; + tableMode: MarkdownTableMode; +}; + +type MatrixPreparedChunkedText = MatrixPreparedSingleText & { + chunks: string[]; +}; + +const getCore = () => getMatrixRuntime(); + +function protectMatrixUnderlineTags(markdown: string): MatrixSpoilerProtection { + const codeRegions = findCodeRegions(markdown); + const metadataRanges = findMatrixMarkdownMetadataRanges(markdown); + const tags = [...tokenizeHtmlTags(markdown)].filter( + (tag) => + (tag.name === "u" || tag.name === "ins") && + !tag.selfClosing && + !isInsideCode(tag.start, codeRegions) && + !isMarkdownEscaped(markdown, tag.start) && + !metadataRanges.some((range) => tag.start >= range.start && tag.start < range.end), + ); + if (tags.length === 0) { + return { markdown }; + } + const markers = createMatrixPrivateMarkers( + markdown, + "Matrix underline chunking exhausted its private marker pool", + ); + let depth = 0; + const replacements = tags.flatMap((tag) => { + if (!tag.closing) { + depth += 1; + return [{ tag, marker: depth === 1 ? markers.open : "" }]; + } + if (depth === 0) { + return []; + } + depth -= 1; + return [{ tag, marker: depth === 0 ? markers.close : "" }]; + }); + let protectedMarkdown = markdown; + for (const { tag, marker } of replacements.toReversed()) { + protectedMarkdown = `${protectedMarkdown.slice(0, tag.start)}${marker}${markers.padding.repeat(tag.raw.length - marker.length)}${protectedMarkdown.slice(tag.end)}`; + } + return { markdown: protectedMarkdown, markers }; +} + +type MatrixChunkStyle = "spoiler" | "underline"; + +function restoreMatrixStyleChunks( + chunks: string[], + spoiler: MatrixSpoilerMarkers | undefined, + underline: MatrixSpoilerMarkers | undefined, +): string[] { + const stack: MatrixChunkStyle[] = []; + const syntax = { + spoiler: { open: "||", close: "||", markers: spoiler }, + underline: { open: "", close: "", markers: underline }, + } as const; + return chunks.map((chunk) => { + let restored = stack.map((style) => syntax[style].open).join(""); + for (const character of chunk) { + const opening = (Object.keys(syntax) as MatrixChunkStyle[]).find( + (style) => character === syntax[style].markers?.open, + ); + const closing = (Object.keys(syntax) as MatrixChunkStyle[]).find( + (style) => character === syntax[style].markers?.close, + ); + if (opening) { + stack.push(opening); + restored += syntax[opening].open; + } else if (closing) { + const stackIndex = stack.lastIndexOf(closing); + if (stackIndex >= 0) { + const above = stack.slice(stackIndex + 1); + restored += above + .toReversed() + .map((style) => syntax[style].close) + .join(""); + restored += syntax[closing].close; + stack.splice(stackIndex, 1); + restored += above.map((style) => syntax[style].open).join(""); + } + } else if (character !== spoiler?.padding && character !== underline?.padding) { + restored += character; + } + } + return ( + restored + + stack + .toReversed() + .map((style) => syntax[style].close) + .join("") + ); + }); +} + +function splitMatrixTableSegments(markdown: string): Array<{ table: boolean; text: string }> { + const segments: Array<{ table: boolean; text: string }> = []; + let cursor = 0; + for (const range of findMatrixTableSourceRanges(markdown)) { + const plain = markdown.slice(cursor, range.start).replace(/(?:[ \t]*\n)+$/u, ""); + if (plain.trim()) { + segments.push({ table: false, text: plain }); + } + const rawTable = markdown.slice(range.start, range.end).trimEnd(); + const indent = /^ +/u.exec(rawTable)?.[0] ?? ""; + const table = indent + ? rawTable + .split("\n") + .map((line) => (line.startsWith(indent) ? line.slice(indent.length) : line)) + .join("\n") + : rawTable; + segments.push({ table: true, text: table }); + cursor = range.end; + } + const tail = markdown.slice(cursor).replace(/^(?:[ \t]*\n)+/u, ""); + if (tail.trim()) { + segments.push({ table: false, text: tail }); + } + return segments; +} + +export function prepareMatrixSingleText( + text: string, + opts: { + cfg: CoreConfig; + accountId?: string; + tableMode?: MarkdownTableMode; + preserveWhitespace?: boolean; + }, +): MatrixPreparedSingleText { + const normalizedText = text.replace(/\r\n?/gu, "\n"); + const trimmedText = opts.preserveWhitespace ? normalizedText : normalizedText.trim(); + const cfg = requireRuntimeConfig(opts.cfg, "Matrix text preparation") as CoreConfig; + const tableMode = + opts.tableMode ?? + getCore().channel.text.resolveMarkdownTableMode({ + cfg, + channel: "matrix", + accountId: opts.accountId, + supportsBlockTables: MATRIX_FORMAT_PROFILE.constructs.table === "native", + }); + const singleEventLimit = Math.min( + getCore().channel.text.resolveTextChunkLimit(cfg, "matrix", opts.accountId), + MATRIX_FORMAT_PROFILE.chunk.limit, + ); + const convertedText = renderMatrixMarkdownTables(trimmedText, tableMode); + const eventTextLength = Math.max( + convertedText.length, + markdownToMatrixBody(convertedText).length, + ); + return { + trimmedText, + convertedText, + singleEventLimit, + eventTextLength, + fitsInSingleEvent: eventTextLength <= singleEventLimit, + tableMode, + }; +} + +export function chunkMatrixText( + text: string, + opts: { + cfg: CoreConfig; + accountId?: string; + tableMode?: MarkdownTableMode; + preserveWhitespace?: boolean; + }, +): MatrixPreparedChunkedText { + const preparedText = prepareMatrixSingleText(text, opts); + if (preparedText.fitsInSingleEvent) { + return { + ...preparedText, + chunks: preparedText.convertedText ? [preparedText.convertedText] : [], + }; + } + const cfg = requireRuntimeConfig(opts.cfg, "Matrix text chunking") as CoreConfig; + const chunkMode = getCore().channel.text.resolveChunkMode(cfg, "matrix", opts.accountId); + const collisionRedacted = hasMatrixSpoilerMetadataCollision(preparedText.convertedText) + ? markdownToMatrixBody(preparedText.convertedText) + : undefined; + const chunkSegment = (segmentText: string): string[] => { + const sourceText = hasMatrixSpoilerMetadataCollision(segmentText) + ? markdownToMatrixBody(segmentText) + : segmentText; + const protectedUnderline = protectMatrixUnderlineTags(sourceText); + const protectedSpoilers = protectMatrixSpoilerDelimiters(protectedUnderline.markdown); + const wrapperReserve = + (protectedSpoilers.markers ? 4 : 0) + (protectedUnderline.markers ? 7 : 0); + const privateMarkers = [protectedSpoilers.markers, protectedUnderline.markers].flatMap( + (markers) => (markers ? [markers.open, markers.close, markers.padding] : []), + ); + let reserve = wrapperReserve; + while (reserve < preparedText.singleEventLimit) { + const protectedChunks = getCore().channel.text.chunkMarkdownTextWithMode( + protectedSpoilers.markdown, + preparedText.singleEventLimit - reserve, + chunkMode, + ); + const restored = restoreMatrixStyleChunks( + protectedChunks, + protectedSpoilers.markers, + protectedUnderline.markers, + ).filter((_, index) => { + const source = privateMarkers.reduce( + (value, marker) => value.replaceAll(marker, ""), + protectedChunks[index] ?? "", + ); + return source.length > 0; + }); + const overflow = Math.max( + 0, + ...restored.map( + (chunk) => + Math.max(chunk.length, markdownToMatrixBody(chunk).length) - + preparedText.singleEventLimit, + ), + ); + if (overflow === 0) { + return restored; + } + reserve += overflow; + } + throw new Error("Matrix text chunk limit is too small for formatted content"); + }; + const chunks = + collisionRedacted !== undefined + ? chunkSegment(collisionRedacted) + : preparedText.tableMode === "block" + ? splitMatrixTableSegments(preparedText.convertedText).flatMap((segment) => { + if (!segment.table) { + return chunkSegment(segment.text); + } + return segment.text.length <= preparedText.singleEventLimit + ? [segment.text] + : chunkSegment(renderMatrixMarkdownTables(segment.text, "bullets")); + }) + : chunkSegment(preparedText.convertedText); + return { + ...preparedText, + chunks, + }; +} diff --git a/extensions/matrix/src/matrix/send/formatting.ts b/extensions/matrix/src/matrix/send/formatting.ts index 340aa1811bec..2e0e6d709838 100644 --- a/extensions/matrix/src/matrix/send/formatting.ts +++ b/extensions/matrix/src/matrix/send/formatting.ts @@ -1,7 +1,9 @@ +import type { MarkdownTableMode } from "openclaw/plugin-sdk/markdown-table-runtime"; // Matrix helper module supports formatting behavior. import { isVoiceMessageCompatibleAudio } from "openclaw/plugin-sdk/media-runtime"; import { getMatrixRuntime } from "../../runtime.js"; import { + markdownToMatrixBody, markdownToMatrixHtml, resolveMatrixMentionsInMarkdown, renderMarkdownToMatrixHtmlWithMentions, @@ -26,17 +28,20 @@ async function renderMatrixFormattedContent(params: { client: MatrixClient; markdown?: string | null; includeMentions?: boolean; -}): Promise<{ html?: string; mentions?: MatrixMentions }> { + tableMode?: MarkdownTableMode; +}): Promise<{ body: string; html?: string; mentions?: MatrixMentions }> { const markdown = params.markdown ?? ""; + const body = markdownToMatrixBody(markdown); if (params.includeMentions === false) { - const html = markdownToMatrixHtml(markdown).trimEnd(); - return { html: html || undefined }; + const html = markdownToMatrixHtml(markdown, { tableMode: params.tableMode }).trimEnd(); + return { body, html: html || undefined }; } const { html, mentions } = await renderMarkdownToMatrixHtmlWithMentions({ markdown, client: params.client, + tableMode: params.tableMode, }); - return { html, mentions }; + return { body, html, mentions }; } export function buildTextContent( @@ -64,12 +69,15 @@ export async function enrichMatrixFormattedContent(params: { content: MatrixFormattedContent; markdown?: string | null; includeMentions?: boolean; + tableMode?: MarkdownTableMode; }): Promise { - const { html, mentions } = await renderMatrixFormattedContent({ + const { body, html, mentions } = await renderMatrixFormattedContent({ client: params.client, markdown: params.markdown, includeMentions: params.includeMentions, + tableMode: params.tableMode, }); + params.content.body = body || params.content.body; if (mentions) { params.content["m.mentions"] = mentions; } else {
Alice