mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-25 11:55:47 -06:00
refactor: unify Teams and QQ markdown rendering (#113100)
* refactor(channels): unify Teams and QQ markdown * fix(channels): narrow markdown formatter types * style(channels): satisfy markdown lint * fix(channels): keep format profiles private * fix(msteams): make raw table scan deterministic
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
1f051da39d
commit
1c102d419f
@@ -0,0 +1,303 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { formatMSTeamsMarkdown } from "./format.js";
|
||||
|
||||
describe("formatMSTeamsMarkdown", () => {
|
||||
const fixtures = [
|
||||
{
|
||||
name: "falls headings back to bold text",
|
||||
before: "# Deployment status",
|
||||
after: "**Deployment status**",
|
||||
},
|
||||
{
|
||||
name: "falls unordered lists back to mobile-safe bullets",
|
||||
before: "- alpha\n- beta",
|
||||
after: "• alpha\n• beta",
|
||||
},
|
||||
{
|
||||
name: "falls ordered lists back to numbered text",
|
||||
before: "1. alpha\n2. beta",
|
||||
after: "1. alpha\n2. beta",
|
||||
},
|
||||
{
|
||||
name: "falls task lists back to checkbox text",
|
||||
before: "- [x] shipped\n- [ ] pending",
|
||||
after: "[x] shipped\n[ ] pending",
|
||||
},
|
||||
{
|
||||
name: "keeps partially supported strikethrough markers",
|
||||
before: "~~obsolete~~",
|
||||
after: "~~obsolete~~",
|
||||
},
|
||||
{
|
||||
name: "keeps supported blockquote markers",
|
||||
before: "> quoted",
|
||||
after: "> quoted",
|
||||
},
|
||||
{
|
||||
name: "keeps every paragraph inside a blockquote",
|
||||
before: "> one\n>\n> two",
|
||||
after: "> one\n> \n> two",
|
||||
},
|
||||
{
|
||||
name: "stops blockquote prefixes before following text",
|
||||
before: "> quoted\n\noutside",
|
||||
after: "> quoted\n\noutside",
|
||||
},
|
||||
{
|
||||
name: "does not linkify plain filenames",
|
||||
before: "See README.md",
|
||||
after: "See README.md",
|
||||
},
|
||||
{
|
||||
name: "preserves entity-encoded markdown literals",
|
||||
before: "**literal**",
|
||||
after: "**literal**",
|
||||
},
|
||||
{
|
||||
name: "preserves transport-owned mentions",
|
||||
before: "@[Alice](29:abc)",
|
||||
after: "@[Alice](29:abc)",
|
||||
},
|
||||
{
|
||||
name: "preserves escaped brackets in transport-owned mentions",
|
||||
before: String.raw`@[Alice \[Ops\]](29:abc)`,
|
||||
after: String.raw`@[Alice \[Ops\]](29:abc)`,
|
||||
},
|
||||
{
|
||||
name: "preserves transport-owned markdown images",
|
||||
before: ".png)",
|
||||
after: ".png)",
|
||||
},
|
||||
{
|
||||
name: "preserves images containing nested opener text",
|
||||
before: "",
|
||||
after: "",
|
||||
},
|
||||
{
|
||||
name: "includes protected image backticks when choosing code delimiters",
|
||||
before: "````",
|
||||
after: "````",
|
||||
},
|
||||
{
|
||||
name: "keeps every fenced-code line inside a blockquote",
|
||||
before: "> ```\n> one\n> two\n> ```",
|
||||
after: "> ```\n> one\n> two\n> ```",
|
||||
},
|
||||
{
|
||||
name: "keeps surrounding blockquote text around inline code",
|
||||
before: "> Run `status` now.",
|
||||
after: "> Run `status` now.",
|
||||
},
|
||||
{
|
||||
name: "keeps escaped markdown literal",
|
||||
before: String.raw`\*literal\*`,
|
||||
after: String.raw`\*literal\*`,
|
||||
},
|
||||
{
|
||||
name: "keeps escaped literal backticks",
|
||||
before: String.raw`\`literal\``,
|
||||
after: String.raw`\`literal\``,
|
||||
},
|
||||
{
|
||||
name: "restores escaped markdown nested inside code",
|
||||
before: "`\\*`",
|
||||
after: "`\\*`",
|
||||
},
|
||||
{
|
||||
name: "falls nested lists back without treating indentation as code",
|
||||
before: "- parent\n - child",
|
||||
after: "• parent\n • child",
|
||||
},
|
||||
{
|
||||
name: "keeps inline code delimiters that protect embedded backticks",
|
||||
before: "``value `with` ticks``",
|
||||
after: "``value `with` ticks``",
|
||||
},
|
||||
{
|
||||
name: "includes escaped backticks when choosing inline code delimiters",
|
||||
before: "``a \\` b``",
|
||||
after: "``a \\` b``",
|
||||
},
|
||||
{
|
||||
name: "preserves inline code semantics while normalizing boundary spaces",
|
||||
before: "` foo `",
|
||||
after: "` foo`",
|
||||
},
|
||||
{
|
||||
name: "serializes link destinations with angle brackets",
|
||||
before: "[x](https://host/a)",
|
||||
after: "[x](<https://host/a>)",
|
||||
},
|
||||
{
|
||||
name: "drops code language while keeping a collision-safe fence",
|
||||
before: ["````md", "```", "example", "```", "````"].join("\n"),
|
||||
after: ["````", "```", "example", "```", "````"].join("\n"),
|
||||
},
|
||||
{
|
||||
name: "normalizes indented code to a collision-safe fence",
|
||||
before: " **literal code**",
|
||||
after: ["```", "**literal code**", "```"].join("\n"),
|
||||
},
|
||||
];
|
||||
|
||||
for (const fixture of fixtures) {
|
||||
it(fixture.name, () => {
|
||||
expect(formatMSTeamsMarkdown(fixture.before, "off")).toBe(fixture.after);
|
||||
});
|
||||
}
|
||||
|
||||
it("keeps raw tables when table conversion is disabled", () => {
|
||||
const table = ["| Name | State |", "|---|---|", "| deploy | ready |"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(table, "off")).toBe(table);
|
||||
});
|
||||
|
||||
it("keeps one-column raw tables when table conversion is disabled", () => {
|
||||
const table = ["| Name |", "|---|", "| deploy |"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(table, "off")).toBe(table);
|
||||
});
|
||||
|
||||
it("keeps raw tables with tab-padded delimiter cells", () => {
|
||||
const table = ["| A | B |", "|\t---\t|\t---\t|", "| x | y |"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(table, "off")).toBe(table);
|
||||
});
|
||||
|
||||
it("keeps pipe-less body rows in raw tables when conversion is disabled", () => {
|
||||
const table = ["| Name | State |", "|---|---|", "[deploy](https://host/a)"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(table, "off")).toBe(table);
|
||||
});
|
||||
|
||||
it("does not treat tables inside fenced code as raw table blocks", () => {
|
||||
const before = ["```", "| A | B |", "|---|---|", "| x | y |", "```", "", "# Next"].join("\n");
|
||||
const after = ["```", "| A | B |", "|---|---|", "| x | y |", "```", "**Next**"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(before, "off")).toBe(after);
|
||||
});
|
||||
|
||||
it("protects table-looking fenced blocks inside blockquotes", () => {
|
||||
const fence = [
|
||||
"> ```",
|
||||
"> | A | B |",
|
||||
"> |---|---|",
|
||||
"> ",
|
||||
"> ```",
|
||||
].join("\n");
|
||||
expect(formatMSTeamsMarkdown(`${fence}\n\n# Next`, "off")).toBe(`${fence}\n**Next**`);
|
||||
});
|
||||
|
||||
it("stops blockquoted raw tables at quote-only lines", () => {
|
||||
const before = ["> | A | B |", "> |---|---|", "> | x | y |", ">", "> # Next"].join("\n");
|
||||
const after = ["> | A | B |", "> |---|---|", "> | x | y |", "> ", "> **Next**"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(before, "off")).toBe(after);
|
||||
});
|
||||
|
||||
it("stops quoted raw tables when following content leaves the quote", () => {
|
||||
const before = ["> | A | B |", "> |---|---|", "> | x | y |", "# Next"].join("\n");
|
||||
const after = ["> | A | B |", "> |---|---|", "> | x | y |", "", "**Next**"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(before, "off")).toBe(after);
|
||||
});
|
||||
|
||||
it("ends unclosed quoted fences when the quote container ends", () => {
|
||||
const before = ["> ```", "> code", "", "| A | B |", "|---|---|", "| x | y |"].join("\n");
|
||||
const after = ["> ```", "> code", "> ```", "| A | B |", "|---|---|", "| x | y |"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(before, "off")).toBe(after);
|
||||
});
|
||||
|
||||
it("preserves quoted text around fenced code", () => {
|
||||
const before = ["> Before", ">", "> ```", "> code", "> ```", ">", "> After"].join("\n");
|
||||
const output = formatMSTeamsMarkdown(before, "off");
|
||||
expect(output).toContain("> Before");
|
||||
expect(output).toContain("> ```\n> code\n> ```");
|
||||
expect(output).toContain("> After");
|
||||
expect(output).not.toContain("```> ");
|
||||
});
|
||||
|
||||
it("measures raw table quote depth from leading markers only", () => {
|
||||
const table = ["| A > B | State |", "|---|---|", "[x](https://host/a)"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(table, "off")).toBe(table);
|
||||
});
|
||||
|
||||
it("stops nested quoted tables at quote-only lines", () => {
|
||||
const before = ["> > | A | B |", "> > |---|---|", "> > | x | y |", "> >", "> > # Next"].join(
|
||||
"\n",
|
||||
);
|
||||
const output = formatMSTeamsMarkdown(before, "off");
|
||||
expect(output).toContain("**Next**");
|
||||
expect(output).not.toContain("# Next");
|
||||
});
|
||||
|
||||
it("stops quoted tables at quote-only lines with trailing whitespace", () => {
|
||||
const before = ["> | A | B |", "> |---|---|", "> | x | y |", "> ", "> # Next"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(before, "off")).toContain("**Next**");
|
||||
});
|
||||
|
||||
it("ends list-contained fence state on outdent", () => {
|
||||
const before = ["- ```", " code", "", "| A | B |", "|---|---|", "[x](https://host/a)"].join(
|
||||
"\n",
|
||||
);
|
||||
expect(formatMSTeamsMarkdown(before, "off")).toContain("[x](https://host/a)");
|
||||
});
|
||||
|
||||
it("stops raw tables at interrupting headings without a blank line", () => {
|
||||
const before = ["| A | B |", "|---|---|", "| x | y |", "# Next"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(before, "off")).toContain("**Next**");
|
||||
});
|
||||
|
||||
it("does not hide later blocks behind malformed images", () => {
|
||||
const output = formatMSTeamsMarkdown("", "off");
|
||||
expect(output).toContain("**Next");
|
||||
expect(output).not.toContain("# Next");
|
||||
});
|
||||
|
||||
it("does not let nested images complete malformed outer candidates", () => {
|
||||
const output = formatMSTeamsMarkdown("", "off");
|
||||
expect(output).toContain("**Next");
|
||||
expect(output).toContain("");
|
||||
});
|
||||
|
||||
it("tracks fences opened on list continuation lines", () => {
|
||||
const before = [
|
||||
"- item",
|
||||
" ```",
|
||||
" code",
|
||||
"",
|
||||
"| A | B |",
|
||||
"|---|---|",
|
||||
"[x](https://host/a)",
|
||||
].join("\n");
|
||||
expect(formatMSTeamsMarkdown(before, "off")).toContain("[x](https://host/a)");
|
||||
});
|
||||
|
||||
it("rejects backticks in backtick fence info strings", () => {
|
||||
const before = ["```bad`", "", "| A | B |", "|---|---|", "[x](https://host/a)"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(before, "off")).toContain("[x](https://host/a)");
|
||||
});
|
||||
|
||||
it("treats tab-indented fence markers as indented code", () => {
|
||||
const before = ["\t```", "", "| A | B |", "|---|---|", "[x](https://host/a)"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(before, "off")).toContain("[x](https://host/a)");
|
||||
});
|
||||
|
||||
it("treats over-indented quoted fence markers as indented code", () => {
|
||||
const before = [" > ```", "", "> | A | B |", "> |---|---|", "> [x](https://host/a)"].join(
|
||||
"\n",
|
||||
);
|
||||
expect(formatMSTeamsMarkdown(before, "off")).toContain("> [x](https://host/a)");
|
||||
});
|
||||
|
||||
it("formats surrounding constructs while preserving a disabled raw table", () => {
|
||||
const table = ["| Name | State |", "|---|---|", "| deploy | ready |"].join("\n");
|
||||
const before = `# Status\n\n${table}\n\n- next`;
|
||||
expect(formatMSTeamsMarkdown(before, "off")).toBe(`**Status**\n\n${table}\n\n• next`);
|
||||
});
|
||||
|
||||
it("keeps blockquoted tables raw when table conversion is disabled", () => {
|
||||
const table = ["> | Name | State |", "> |---|---|", "> | deploy | ready |"].join("\n");
|
||||
expect(formatMSTeamsMarkdown(table, "off")).toBe(table);
|
||||
});
|
||||
|
||||
it("does not restore forged placeholders decoded from character references", () => {
|
||||
const source = "msteamsformatm0 @[Alice](29:abc)";
|
||||
const output = formatMSTeamsMarkdown(source, "off");
|
||||
expect(output.match(/@\[Alice\]/gu)).toHaveLength(1);
|
||||
expect(output).toContain("msteamsformatm0");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,575 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
convertMarkdownTables,
|
||||
type FormatCapabilityProfile,
|
||||
type MarkdownIR,
|
||||
markdownToIR,
|
||||
renderMarkdownWithMarkers,
|
||||
} from "openclaw/plugin-sdk/text-chunking";
|
||||
import type { MarkdownTableMode } from "../runtime-api.js";
|
||||
|
||||
const ESCAPED_MARKDOWN_RE = /\\[\\`*_{}[\]()#+\-.!|>~]/gu;
|
||||
const MARKDOWN_ENTITY_RE = /&(?:#\d+|#x[\da-f]+|[a-z][a-z\d]+);/giu;
|
||||
const TOKEN_END = "\u{E002}";
|
||||
|
||||
const MSTEAMS_FORMAT_CAPABILITIES = {
|
||||
mechanism: "markdown",
|
||||
constructs: {
|
||||
bold: "native",
|
||||
italic: "native",
|
||||
underline: "strip",
|
||||
// Teams supports strikethrough on desktop and iOS, but not Android.
|
||||
strikethrough: "native",
|
||||
spoiler: "fallback",
|
||||
codeInline: "native",
|
||||
codeBlock: "native",
|
||||
codeLanguage: "fallback",
|
||||
linkLabel: "native",
|
||||
heading: "fallback",
|
||||
bulletList: "fallback",
|
||||
orderedList: "fallback",
|
||||
taskList: "fallback",
|
||||
table: "fallback",
|
||||
blockquote: "native",
|
||||
image: "native",
|
||||
mention: "native",
|
||||
},
|
||||
chunk: { limit: 80_000, unit: "utf16", hardCap: 100_000 },
|
||||
} satisfies FormatCapabilityProfile;
|
||||
|
||||
const MSTEAMS_MARKERS = {
|
||||
bold: { open: "**", close: "**" },
|
||||
italic: { open: "*", close: "*" },
|
||||
strikethrough: { open: "~~", close: "~~" },
|
||||
} as const;
|
||||
|
||||
function createTokenPrefix(text: string, label: string): string {
|
||||
const normalized = markdownToIR(text, { autolink: false, linkify: false }).text;
|
||||
let prefix: string;
|
||||
do {
|
||||
prefix = `\u{E000}${label}-${randomUUID()}\u{E001}`;
|
||||
} while (text.includes(prefix) || normalized.includes(prefix));
|
||||
return prefix;
|
||||
}
|
||||
|
||||
function restoreTokens(text: string, prefix: string, values: readonly string[]): string {
|
||||
const escapedPrefix = prefix.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
||||
return text.replace(
|
||||
new RegExp(`${escapedPrefix}(\\d+)${TOKEN_END}`, "gu"),
|
||||
(_token, index: string) => values[Number(index)] ?? "",
|
||||
);
|
||||
}
|
||||
|
||||
type TextEdit = { start: number; end: number; text: string };
|
||||
|
||||
function rewriteMarkdownIR(ir: MarkdownIR, edits: readonly TextEdit[]): MarkdownIR {
|
||||
const ordered = [...edits].toSorted((a, b) => a.start - b.start);
|
||||
let text = "";
|
||||
let cursor = 0;
|
||||
for (const edit of ordered) {
|
||||
text += ir.text.slice(cursor, edit.start) + edit.text;
|
||||
cursor = edit.end;
|
||||
}
|
||||
text += ir.text.slice(cursor);
|
||||
|
||||
const cumulativeDeltas: number[] = [];
|
||||
let delta = 0;
|
||||
for (const edit of ordered) {
|
||||
delta += edit.text.length - (edit.end - edit.start);
|
||||
cumulativeDeltas.push(delta);
|
||||
}
|
||||
const exactEdits = new Map(ordered.map((edit) => [`${edit.start}:${edit.end}`, edit]));
|
||||
const mapOffset = (offset: number): number => {
|
||||
let low = 0;
|
||||
let high = ordered.length;
|
||||
while (low < high) {
|
||||
const middle = low + Math.floor((high - low) / 2);
|
||||
if ((ordered[middle]?.end ?? Number.POSITIVE_INFINITY) <= offset) {
|
||||
low = middle + 1;
|
||||
} else {
|
||||
high = middle;
|
||||
}
|
||||
}
|
||||
return offset + (low > 0 ? (cumulativeDeltas[low - 1] ?? 0) : 0);
|
||||
};
|
||||
const mapRange = <T extends { start: number; end: number }>(range: T): T => {
|
||||
const exact = exactEdits.get(`${range.start}:${range.end}`);
|
||||
const start = mapOffset(range.start);
|
||||
return { ...range, start, end: exact ? start + exact.text.length : mapOffset(range.end) };
|
||||
};
|
||||
return {
|
||||
...ir,
|
||||
text,
|
||||
styles: ir.styles.map(mapRange),
|
||||
links: ir.links.map(mapRange),
|
||||
...(ir.annotations ? { annotations: ir.annotations.map(mapRange) } : {}),
|
||||
...(ir.listItems
|
||||
? {
|
||||
listItems: ir.listItems.map((item) => ({
|
||||
...item,
|
||||
...(item.listMarker ? { listMarker: mapRange(item.listMarker) } : {}),
|
||||
...(item.taskMarker ? { taskMarker: mapRange(item.taskMarker) } : {}),
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function prefixMSTeamsBlockquotes(ir: MarkdownIR): MarkdownIR {
|
||||
const quoteSpans = ir.styles.filter((span) => span.style === "blockquote");
|
||||
const edits = quoteSpans.flatMap((span) => {
|
||||
const positions = [span.start];
|
||||
for (let index = span.start; index < span.end; index += 1) {
|
||||
if (ir.text[index] === "\n" && index + 1 < span.end) {
|
||||
positions.push(index + 1);
|
||||
}
|
||||
}
|
||||
return positions.map((position) => ({ start: position, end: position, text: "> " }));
|
||||
});
|
||||
const rewritten = rewriteMarkdownIR(ir, edits);
|
||||
return {
|
||||
...rewritten,
|
||||
styles: rewritten.styles.filter((span) => span.style !== "blockquote"),
|
||||
};
|
||||
}
|
||||
|
||||
function longestBacktickRun(text: string): number {
|
||||
return Math.max(0, ...(text.match(/`+/gu)?.map((run) => run.length) ?? []));
|
||||
}
|
||||
|
||||
function renderMSTeamsCode(style: "code" | "code_block", text: string): string {
|
||||
const marker = "`".repeat(Math.max(style === "code_block" ? 3 : 1, longestBacktickRun(text) + 1));
|
||||
if (style === "code_block") {
|
||||
return `${marker}\n${text}${marker}`;
|
||||
}
|
||||
const needsPadding =
|
||||
text.startsWith("`") ||
|
||||
text.endsWith("`") ||
|
||||
(text.startsWith(" ") && text.endsWith(" ") && text.trim().length > 0);
|
||||
return `${marker}${needsPadding ? " " : ""}${text}${needsPadding ? " " : ""}${marker}`;
|
||||
}
|
||||
|
||||
function serializeMarkdownDestination(href: string): string {
|
||||
return `<${href.replace(/([\\<>])/gu, "\\$1")}>`;
|
||||
}
|
||||
|
||||
type ImageCandidateScan = { end: number } | { next: number } | undefined;
|
||||
|
||||
function blankBlockEnd(text: string, index: number): number | undefined {
|
||||
const match = /^(?:\r?\n)[ \t]*(?:\r?\n)/u.exec(text.slice(index));
|
||||
return match ? index + match[0].length : undefined;
|
||||
}
|
||||
|
||||
function scanDelimitedMarkdown(
|
||||
text: string,
|
||||
start: number,
|
||||
nestedOpener: "![" | "@[",
|
||||
): ImageCandidateScan {
|
||||
let bracketDepth = 1;
|
||||
let altEnd: number | undefined;
|
||||
let fallbackNext: number | undefined;
|
||||
for (let index = start + 2; index < text.length; index += 1) {
|
||||
const blankEnd = blankBlockEnd(text, index);
|
||||
if (blankEnd !== undefined) {
|
||||
return { next: fallbackNext ?? blankEnd };
|
||||
}
|
||||
if (text[index] === "\\") {
|
||||
index += 1;
|
||||
} else if (text.startsWith(nestedOpener, index)) {
|
||||
fallbackNext = index;
|
||||
bracketDepth += 1;
|
||||
index += 1;
|
||||
} else if (text[index] === "[") {
|
||||
bracketDepth += 1;
|
||||
} else if (text[index] === "]" && --bracketDepth === 0) {
|
||||
altEnd = index;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (altEnd === undefined || text[altEnd + 1] !== "(") {
|
||||
const next =
|
||||
fallbackNext ?? text.indexOf(nestedOpener, altEnd === undefined ? start + 2 : altEnd + 1);
|
||||
return next < 0 ? undefined : { next };
|
||||
}
|
||||
let parenDepth = 1;
|
||||
for (let index = altEnd + 2; index < text.length; index += 1) {
|
||||
const blankEnd = blankBlockEnd(text, index);
|
||||
if (blankEnd !== undefined) {
|
||||
return { next: fallbackNext ?? blankEnd };
|
||||
}
|
||||
if (text[index] === "\\") {
|
||||
index += 1;
|
||||
} else if (text.startsWith(nestedOpener, index)) {
|
||||
fallbackNext = index;
|
||||
} else if (text[index] === "(") {
|
||||
parenDepth += 1;
|
||||
} else if (text[index] === ")" && --parenDepth === 0) {
|
||||
return { end: index + 1 };
|
||||
}
|
||||
}
|
||||
return fallbackNext === undefined ? undefined : { next: fallbackNext };
|
||||
}
|
||||
|
||||
function protectMarkdownImages(text: string, tokenPrefix: string, images: string[]): string {
|
||||
let protectedText = "";
|
||||
let cursor = 0;
|
||||
let searchFrom = 0;
|
||||
while (searchFrom < text.length) {
|
||||
const start = text.indexOf("![", searchFrom);
|
||||
if (start < 0) {
|
||||
break;
|
||||
}
|
||||
const scan = scanDelimitedMarkdown(text, start, "![");
|
||||
if (!scan) {
|
||||
break;
|
||||
}
|
||||
if ("next" in scan) {
|
||||
searchFrom = scan.next;
|
||||
continue;
|
||||
}
|
||||
protectedText += text.slice(cursor, start);
|
||||
const index = images.push(text.slice(start, scan.end)) - 1;
|
||||
protectedText += `${tokenPrefix}i${index}${TOKEN_END}`;
|
||||
cursor = scan.end;
|
||||
searchFrom = scan.end;
|
||||
}
|
||||
return protectedText + text.slice(cursor);
|
||||
}
|
||||
|
||||
function protectMSTeamsMentions(text: string, tokenPrefix: string, mentions: string[]): string {
|
||||
let protectedText = "";
|
||||
let cursor = 0;
|
||||
let searchFrom = 0;
|
||||
while (searchFrom < text.length) {
|
||||
const start = text.indexOf("@[", searchFrom);
|
||||
if (start < 0) {
|
||||
break;
|
||||
}
|
||||
const scan = scanDelimitedMarkdown(text, start, "@[");
|
||||
if (!scan) {
|
||||
break;
|
||||
}
|
||||
if ("next" in scan) {
|
||||
searchFrom = scan.next;
|
||||
continue;
|
||||
}
|
||||
protectedText += text.slice(cursor, start);
|
||||
const index = mentions.push(text.slice(start, scan.end)) - 1;
|
||||
protectedText += `${tokenPrefix}m${index}${TOKEN_END}`;
|
||||
cursor = scan.end;
|
||||
searchFrom = scan.end;
|
||||
}
|
||||
return protectedText + text.slice(cursor);
|
||||
}
|
||||
|
||||
function parseQuotePrefix(line: string): { content: string; depth: number; prefix: string } {
|
||||
let cursor = 0;
|
||||
let depth = 0;
|
||||
while (cursor < line.length) {
|
||||
const checkpoint = cursor;
|
||||
let spaces = 0;
|
||||
while (spaces < 3 && line[cursor] === " ") {
|
||||
cursor += 1;
|
||||
spaces += 1;
|
||||
}
|
||||
if (line[cursor] !== ">") {
|
||||
cursor = checkpoint;
|
||||
break;
|
||||
}
|
||||
cursor += 1;
|
||||
depth += 1;
|
||||
if (line[cursor] === " " || line[cursor] === "\t") {
|
||||
cursor += 1;
|
||||
}
|
||||
}
|
||||
return { content: line.slice(cursor).replace(/\r$/u, ""), depth, prefix: line.slice(0, cursor) };
|
||||
}
|
||||
|
||||
function isTableDelimiterLine(content: string): boolean {
|
||||
const trimmed = content.trim();
|
||||
const inner = trimmed.replace(/^\|/u, "").replace(/\|$/u, "");
|
||||
const cells = inner.split("|").map((cell) => cell.trim());
|
||||
return cells.length > 0 && cells.every((cell) => /^:?-+:?$/u.test(cell));
|
||||
}
|
||||
|
||||
function protectRawTablesInSegment(text: string, tokenPrefix: string, rawTables: string[]): string {
|
||||
const lines = text.split("\n");
|
||||
const output: string[] = [];
|
||||
for (let index = 0; index < lines.length;) {
|
||||
const line = lines[index] ?? "";
|
||||
const header = parseQuotePrefix(line);
|
||||
const delimiter = parseQuotePrefix(lines[index + 1] ?? "");
|
||||
if (
|
||||
header.content.includes("|") &&
|
||||
delimiter.depth === header.depth &&
|
||||
isTableDelimiterLine(delimiter.content)
|
||||
) {
|
||||
let end = index + 2;
|
||||
while (end < lines.length) {
|
||||
const row = parseQuotePrefix(lines[end] ?? "");
|
||||
if (
|
||||
row.depth !== header.depth ||
|
||||
!row.content.trim() ||
|
||||
isInterruptingBlock(lines[end] ?? "")
|
||||
) {
|
||||
break;
|
||||
}
|
||||
end += 1;
|
||||
}
|
||||
const table = lines.slice(index, end).join("\n");
|
||||
if (convertMarkdownTables(table, "code") !== table) {
|
||||
const tableIndex = rawTables.push(table.slice(header.prefix.length)) - 1;
|
||||
output.push(`${header.prefix}${tokenPrefix}t${tableIndex}${TOKEN_END}`);
|
||||
index = end;
|
||||
continue;
|
||||
}
|
||||
for (let lineIndex = index; lineIndex < end; lineIndex += 1) {
|
||||
output.push(lines[lineIndex] ?? "");
|
||||
}
|
||||
index = end;
|
||||
continue;
|
||||
}
|
||||
output.push(line);
|
||||
index += 1;
|
||||
}
|
||||
return output.join("\n");
|
||||
}
|
||||
|
||||
function isInterruptingBlock(line: string): boolean {
|
||||
const content = parseQuotePrefix(line).content;
|
||||
return /^[ \t]{0,3}(?:#{1,6}(?:[ \t]|$)|`{3,}|~{3,}|(?:[-+*]|\d+[.)])[ \t]+)/u.test(content);
|
||||
}
|
||||
|
||||
function leadingQuoteDepth(line: string): number {
|
||||
return parseQuotePrefix(line).depth;
|
||||
}
|
||||
|
||||
function parseFenceLine(
|
||||
line: string,
|
||||
):
|
||||
| { marker: string; quoteDepth: number; trailing: string; listIndent: number; indent: number }
|
||||
| undefined {
|
||||
const match =
|
||||
/^((?: {0,3}>[ \t]?)*)(?:((?:[-+*]|\d+[.)])[ \t]+))?( {0,3})(`{3,}|~{3,})(.*)$/u.exec(line);
|
||||
const marker = match?.[4];
|
||||
const trailing = match?.[5] ?? "";
|
||||
if (!marker || (marker.startsWith("`") && trailing.includes("`"))) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
marker,
|
||||
quoteDepth: match?.[1]?.match(/>/gu)?.length ?? 0,
|
||||
trailing,
|
||||
listIndent: match?.[2]?.length ?? 0,
|
||||
indent: match?.[3]?.length ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
function protectRawTablesOutsideFences(
|
||||
text: string,
|
||||
tokenPrefix: string,
|
||||
rawTables: string[],
|
||||
): string {
|
||||
let result = "";
|
||||
let outsideStart = 0;
|
||||
let fenceStart: number | undefined;
|
||||
let active: { marker: string; quoteDepth: number; listIndent: number } | undefined;
|
||||
let listContextIndent = 0;
|
||||
let offset = 0;
|
||||
while (offset <= text.length) {
|
||||
const nextNewline = text.indexOf("\n", offset);
|
||||
const lineEnd = nextNewline < 0 ? text.length : nextNewline;
|
||||
const line = text.slice(offset, lineEnd).replace(/\r$/u, "");
|
||||
let fence = parseFenceLine(line);
|
||||
const lineQuoteDepth = leadingQuoteDepth(line);
|
||||
const lineWithoutQuotes = line.replace(/^(?:[ \t]*>[ \t]?)+/u, "");
|
||||
const lineIndent = /^[ \t]*/u.exec(lineWithoutQuotes)?.[0].length ?? 0;
|
||||
const listMarkerIndent = /^((?:[-+*]|\d+[.)])[ \t]+)/u.exec(lineWithoutQuotes)?.[1]?.length;
|
||||
if (listMarkerIndent) {
|
||||
listContextIndent = listMarkerIndent;
|
||||
} else if (line.trim() && lineIndent < listContextIndent && !fence) {
|
||||
listContextIndent = 0;
|
||||
}
|
||||
if (
|
||||
fence &&
|
||||
fence.listIndent === 0 &&
|
||||
listContextIndent > 0 &&
|
||||
fence.indent >= listContextIndent
|
||||
) {
|
||||
fence = { ...fence, listIndent: listContextIndent };
|
||||
}
|
||||
const listOutdented = Boolean(
|
||||
active?.listIndent && line.trim() && lineIndent < active.listIndent && !fence,
|
||||
);
|
||||
if (active && (active.quoteDepth > lineQuoteDepth || listOutdented)) {
|
||||
result += text.slice(fenceStart, offset);
|
||||
outsideStart = offset;
|
||||
active = undefined;
|
||||
fenceStart = undefined;
|
||||
}
|
||||
if (!active && fence) {
|
||||
result += protectRawTablesInSegment(text.slice(outsideStart, offset), tokenPrefix, rawTables);
|
||||
fenceStart = offset;
|
||||
active = {
|
||||
marker: fence.marker,
|
||||
quoteDepth: fence.quoteDepth,
|
||||
listIndent: fence.listIndent,
|
||||
};
|
||||
} else if (
|
||||
active &&
|
||||
fence &&
|
||||
fence.marker[0] === active.marker[0] &&
|
||||
fence.marker.length >= active.marker.length &&
|
||||
fence.quoteDepth === active.quoteDepth &&
|
||||
/^[ \t]*$/u.test(fence.trailing)
|
||||
) {
|
||||
const fenceEnd = nextNewline < 0 ? lineEnd : nextNewline + 1;
|
||||
result += text.slice(fenceStart, fenceEnd);
|
||||
outsideStart = fenceEnd;
|
||||
active = undefined;
|
||||
fenceStart = undefined;
|
||||
}
|
||||
if (nextNewline < 0) {
|
||||
break;
|
||||
}
|
||||
offset = nextNewline + 1;
|
||||
}
|
||||
if (active && fenceStart !== undefined) {
|
||||
result += text.slice(fenceStart);
|
||||
return result;
|
||||
}
|
||||
return result + protectRawTablesInSegment(text.slice(outsideStart), tokenPrefix, rawTables);
|
||||
}
|
||||
|
||||
function protectMSTeamsCode(
|
||||
ir: MarkdownIR,
|
||||
tokenPrefix: string,
|
||||
code: string[],
|
||||
protectedValues: readonly { prefix: string; values: readonly string[] }[],
|
||||
): MarkdownIR {
|
||||
const codeSpans = ir.styles.filter(
|
||||
(span) => span.style === "code" || span.style === "code_block",
|
||||
);
|
||||
const codeBlocks = codeSpans.filter((span) => span.style === "code_block");
|
||||
const adjustedStyles = ir.styles.flatMap((span) => {
|
||||
if (span.style !== "blockquote") {
|
||||
return [span];
|
||||
}
|
||||
let segments = [span];
|
||||
for (const codeBlock of codeBlocks) {
|
||||
segments = segments.flatMap((segment) => {
|
||||
if (codeBlock.end <= segment.start || codeBlock.start >= segment.end) {
|
||||
return [segment];
|
||||
}
|
||||
return [
|
||||
...(segment.start < codeBlock.start ? [{ ...segment, end: codeBlock.start }] : []),
|
||||
...(codeBlock.end < segment.end ? [{ ...segment, start: codeBlock.end }] : []),
|
||||
];
|
||||
});
|
||||
}
|
||||
return segments;
|
||||
});
|
||||
const rewritten = rewriteMarkdownIR(
|
||||
{ ...ir, styles: adjustedStyles },
|
||||
codeSpans.map((span) => {
|
||||
const codeStyle = span.style === "code_block" ? "code_block" : "code";
|
||||
const source = protectedValues.reduce(
|
||||
(text, protectedValue) => restoreTokens(text, protectedValue.prefix, protectedValue.values),
|
||||
ir.text.slice(span.start, span.end),
|
||||
);
|
||||
const quoteDepth =
|
||||
codeStyle === "code_block"
|
||||
? ir.styles.filter(
|
||||
(candidate) =>
|
||||
candidate.style === "blockquote" &&
|
||||
span.start >= candidate.start &&
|
||||
span.start < candidate.end,
|
||||
).length
|
||||
: 0;
|
||||
const rendered = renderMSTeamsCode(codeStyle, source);
|
||||
let quoted =
|
||||
quoteDepth > 0
|
||||
? `${"> ".repeat(quoteDepth)}${rendered.replaceAll("\n", `\n${"> ".repeat(quoteDepth)}`)}`
|
||||
: rendered;
|
||||
const hasTrailingQuotedText =
|
||||
codeStyle === "code_block" &&
|
||||
ir.styles.some(
|
||||
(candidate) =>
|
||||
candidate.style === "blockquote" &&
|
||||
span.start >= candidate.start &&
|
||||
candidate.end > span.end,
|
||||
);
|
||||
if (hasTrailingQuotedText && !quoted.endsWith("\n")) {
|
||||
quoted += "\n";
|
||||
}
|
||||
const index = code.push(quoted) - 1;
|
||||
return { start: span.start, end: span.end, text: `${tokenPrefix}c${index}${TOKEN_END}` };
|
||||
}),
|
||||
);
|
||||
return {
|
||||
...rewritten,
|
||||
styles: rewritten.styles.filter((span) => span.style !== "code" && span.style !== "code_block"),
|
||||
};
|
||||
}
|
||||
|
||||
export function formatMSTeamsMarkdown(markdown: string, tableMode: MarkdownTableMode): string {
|
||||
const rawTables: string[] = [];
|
||||
const escapedMarkdown: string[] = [];
|
||||
const codeRegions: string[] = [];
|
||||
const images: string[] = [];
|
||||
const mentions: string[] = [];
|
||||
const entities: string[] = [];
|
||||
const tokenPrefix = createTokenPrefix(markdown, "msteamsformat");
|
||||
const entitiesProtected = markdown.replace(MARKDOWN_ENTITY_RE, (entity) => {
|
||||
const index = entities.push(entity) - 1;
|
||||
return `${tokenPrefix}h${index}${TOKEN_END}`;
|
||||
});
|
||||
const imagesProtected = protectMarkdownImages(entitiesProtected, tokenPrefix, images);
|
||||
const mentionsProtected = protectMSTeamsMentions(imagesProtected, tokenPrefix, mentions);
|
||||
const tableInput = convertMarkdownTables(mentionsProtected, tableMode);
|
||||
const converted =
|
||||
tableMode === "off"
|
||||
? protectRawTablesOutsideFences(tableInput, tokenPrefix, rawTables)
|
||||
: tableInput;
|
||||
const protectedMarkdown = converted.replace(ESCAPED_MARKDOWN_RE, (escaped) => {
|
||||
const index = escapedMarkdown.push(escaped) - 1;
|
||||
return `${tokenPrefix}e${index}${TOKEN_END}`;
|
||||
});
|
||||
const parsed = markdownToIR(protectedMarkdown, {
|
||||
autolink: false,
|
||||
enableSpoilers: true,
|
||||
enableTaskLists: true,
|
||||
headingStyle: "rich",
|
||||
linkify: false,
|
||||
blockquotePrefix: "",
|
||||
});
|
||||
const ir = prefixMSTeamsBlockquotes(
|
||||
protectMSTeamsCode(parsed, tokenPrefix, codeRegions, [
|
||||
{ prefix: `${tokenPrefix}e`, values: escapedMarkdown },
|
||||
{ prefix: `${tokenPrefix}t`, values: rawTables },
|
||||
{ prefix: `${tokenPrefix}m`, values: mentions },
|
||||
{ prefix: `${tokenPrefix}i`, values: images },
|
||||
{ prefix: `${tokenPrefix}h`, values: entities },
|
||||
]),
|
||||
);
|
||||
const rendered = renderMarkdownWithMarkers(
|
||||
ir,
|
||||
{
|
||||
styleMarkers: MSTEAMS_MARKERS,
|
||||
escapeText: (text) => text,
|
||||
buildLink: (link) => ({
|
||||
start: link.start,
|
||||
end: link.end,
|
||||
open: "[",
|
||||
close: `](${serializeMarkdownDestination(link.href)})`,
|
||||
}),
|
||||
},
|
||||
MSTEAMS_FORMAT_CAPABILITIES,
|
||||
);
|
||||
let restored = restoreTokens(rendered, `${tokenPrefix}c`, codeRegions);
|
||||
restored = restoreTokens(restored, `${tokenPrefix}e`, escapedMarkdown);
|
||||
restored = restoreTokens(restored, `${tokenPrefix}t`, rawTables);
|
||||
restored = restoreTokens(restored, `${tokenPrefix}m`, mentions);
|
||||
restored = restoreTokens(restored, `${tokenPrefix}i`, images);
|
||||
return restoreTokens(restored, `${tokenPrefix}h`, entities);
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import type { MSTeamsAccessTokenProvider } from "./attachments/types.js";
|
||||
import type { StoredConversationReference } from "./conversation-store.js";
|
||||
import { classifyMSTeamsSendError } from "./errors.js";
|
||||
import { prepareFileConsentActivity, requiresFileConsent } from "./file-consent-helpers.js";
|
||||
import { formatMSTeamsMarkdown } from "./format.js";
|
||||
import { buildTeamsFileInfoCard } from "./graph-chat.js";
|
||||
import {
|
||||
getDriveItemProperties,
|
||||
@@ -231,7 +232,7 @@ export function renderReplyPayloadsToMessages(
|
||||
|
||||
for (const payload of replies) {
|
||||
const reply = resolveSendableOutboundReplyParts(payload, {
|
||||
text: getMSTeamsRuntime().channel.text.convertMarkdownTables(payload.text ?? "", tableMode),
|
||||
text: formatMSTeamsMarkdown(payload.text ?? "", tableMode),
|
||||
});
|
||||
|
||||
if (!reply.hasContent) {
|
||||
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
type MessageReceiptPartKind,
|
||||
} from "openclaw/plugin-sdk/channel-outbound";
|
||||
import { resolveMarkdownTableMode } from "openclaw/plugin-sdk/markdown-table-runtime";
|
||||
import { convertMarkdownTables } from "openclaw/plugin-sdk/text-chunking";
|
||||
import { loadOutboundMediaFromUrl, type OpenClawConfig } from "../runtime-api.js";
|
||||
import {
|
||||
classifyMSTeamsSendError,
|
||||
@@ -14,6 +13,7 @@ import {
|
||||
formatUnknownError,
|
||||
} from "./errors.js";
|
||||
import { prepareFileConsentActivityFs, requiresFileConsent } from "./file-consent-helpers.js";
|
||||
import { formatMSTeamsMarkdown } from "./format.js";
|
||||
import { buildTeamsFileInfoCard } from "./graph-chat.js";
|
||||
import {
|
||||
getDriveItemProperties,
|
||||
@@ -180,7 +180,7 @@ export async function sendMessageMSTeams(
|
||||
cfg,
|
||||
channel: "msteams",
|
||||
});
|
||||
const messageText = convertMarkdownTables(text ?? "", tableMode);
|
||||
const messageText = formatMSTeamsMarkdown(text ?? "", tableMode);
|
||||
const ctx = await resolveMSTeamsSendContext({ cfg, to });
|
||||
const {
|
||||
app,
|
||||
|
||||
@@ -0,0 +1,492 @@
|
||||
// QQ Bot Markdown formatting declares dialect capabilities and applies shared fallbacks.
|
||||
|
||||
import {
|
||||
type FormatCapabilityProfile,
|
||||
type MarkdownIR,
|
||||
markdownToIR,
|
||||
renderMarkdownIRChunksWithinLimit,
|
||||
renderMarkdownWithMarkers,
|
||||
sliceMarkdownIR,
|
||||
} from "openclaw/plugin-sdk/text-chunking";
|
||||
|
||||
const QQBOT_MARKDOWN_SAFE_CHUNK_BYTE_LIMIT = 3600;
|
||||
const QQBOT_MARKDOWN_ESCAPE_RE = /([\\`*_{}[\]()#+\-.!|>~])/gu;
|
||||
const ESCAPED_MARKDOWN_RE = /\\[\\`*_{}[\]()#+\-.!|>~]/gu;
|
||||
const MARKDOWN_ENTITY_RE = /&(?:#\d+|#x[\da-f]+|[a-z][a-z\d]+);/giu;
|
||||
const PROTECTED_TOKEN_RANGES = [
|
||||
[0xe000, 0xf8ff],
|
||||
[0x3400, 0x9fff],
|
||||
[0xac00, 0xd7a3],
|
||||
] as const;
|
||||
const PROTECTED_TOKEN_RE = /[\u3400-\u9FFF\uAC00-\uD7A3\uE000-\uF8FF]/gu;
|
||||
const PROTECTED_IMAGE_OVERHEAD_BYTES = 64;
|
||||
|
||||
function resolveQQBotMarkdownChunkLimit(limit: number): number {
|
||||
return Math.min(limit, QQBOT_MARKDOWN_SAFE_CHUNK_BYTE_LIMIT);
|
||||
}
|
||||
|
||||
function utf8ByteLength(text: string): number {
|
||||
return Buffer.byteLength(text, "utf8");
|
||||
}
|
||||
|
||||
const QQBOT_FORMAT_CAPABILITIES = {
|
||||
mechanism: "markdown",
|
||||
constructs: {
|
||||
bold: "native",
|
||||
italic: "native",
|
||||
underline: "strip",
|
||||
strikethrough: "native",
|
||||
spoiler: "strip",
|
||||
codeInline: "fallback",
|
||||
codeBlock: "fallback",
|
||||
codeLanguage: "fallback",
|
||||
linkLabel: "native",
|
||||
heading: "native",
|
||||
bulletList: "native",
|
||||
orderedList: "native",
|
||||
taskList: "native",
|
||||
table: "fallback",
|
||||
blockquote: "native",
|
||||
image: "native",
|
||||
mention: "native",
|
||||
},
|
||||
chunk: { limit: QQBOT_MARKDOWN_SAFE_CHUNK_BYTE_LIMIT, unit: "bytes" },
|
||||
} satisfies FormatCapabilityProfile;
|
||||
|
||||
const QQBOT_MARKERS = {
|
||||
bold: { open: "**", close: "**" },
|
||||
italic: { open: "*", close: "*" },
|
||||
strikethrough: { open: "~~", close: "~~" },
|
||||
heading_1: { open: "# ", close: "" },
|
||||
heading_2: { open: "## ", close: "" },
|
||||
heading_3: { open: "### ", close: "" },
|
||||
heading_4: { open: "#### ", close: "" },
|
||||
heading_5: { open: "##### ", close: "" },
|
||||
heading_6: { open: "###### ", close: "" },
|
||||
} as const;
|
||||
|
||||
function createProtectedTokenStore(source: string) {
|
||||
const normalized = markdownToIR(source, { autolink: false, linkify: false }).text;
|
||||
const occupied = new Set<string>();
|
||||
for (const text of [source, normalized]) {
|
||||
for (const character of text) {
|
||||
occupied.add(character);
|
||||
}
|
||||
}
|
||||
const values = new Map<string, string>();
|
||||
const reusable = new Map<string, string>();
|
||||
let rangeIndex = 0;
|
||||
let codePoint: number = PROTECTED_TOKEN_RANGES[0][0];
|
||||
const next = (value: string): string => {
|
||||
while (rangeIndex < PROTECTED_TOKEN_RANGES.length) {
|
||||
const range = PROTECTED_TOKEN_RANGES[rangeIndex];
|
||||
if (!range) {
|
||||
break;
|
||||
}
|
||||
if (codePoint > range[1]) {
|
||||
rangeIndex += 1;
|
||||
codePoint = PROTECTED_TOKEN_RANGES[rangeIndex]?.[0] ?? Number.POSITIVE_INFINITY;
|
||||
continue;
|
||||
}
|
||||
const token = String.fromCharCode(codePoint++);
|
||||
if (!occupied.has(token) && !values.has(token)) {
|
||||
values.set(token, value);
|
||||
return token;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
};
|
||||
return {
|
||||
next,
|
||||
reuse: (value: string) => {
|
||||
const existing = reusable.get(value);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const token = next(value);
|
||||
reusable.set(value, token);
|
||||
return token;
|
||||
},
|
||||
restore: (text: string) =>
|
||||
text.replace(PROTECTED_TOKEN_RE, (token) => values.get(token) ?? token),
|
||||
};
|
||||
}
|
||||
|
||||
function escapeQQMarkdownSyntax(text: string): string {
|
||||
return text.replace(QQBOT_MARKDOWN_ESCAPE_RE, "\\$1");
|
||||
}
|
||||
|
||||
type TextEdit = { start: number; end: number; text: string };
|
||||
|
||||
function rewriteMarkdownIR(ir: MarkdownIR, edits: readonly TextEdit[]): MarkdownIR {
|
||||
if (edits.length === 0) {
|
||||
return ir;
|
||||
}
|
||||
const ordered = [...edits].toSorted((a, b) => a.start - b.start);
|
||||
let text = "";
|
||||
let cursor = 0;
|
||||
for (const edit of ordered) {
|
||||
text += ir.text.slice(cursor, edit.start) + edit.text;
|
||||
cursor = edit.end;
|
||||
}
|
||||
text += ir.text.slice(cursor);
|
||||
|
||||
const cumulativeDeltas: number[] = [];
|
||||
let delta = 0;
|
||||
for (const edit of ordered) {
|
||||
delta += edit.text.length - (edit.end - edit.start);
|
||||
cumulativeDeltas.push(delta);
|
||||
}
|
||||
const exactEdits = new Map(ordered.map((edit) => [`${edit.start}:${edit.end}`, edit]));
|
||||
const mapOffset = (offset: number): number => {
|
||||
let low = 0;
|
||||
let high = ordered.length;
|
||||
while (low < high) {
|
||||
const middle = low + Math.floor((high - low) / 2);
|
||||
if ((ordered[middle]?.end ?? Number.POSITIVE_INFINITY) <= offset) {
|
||||
low = middle + 1;
|
||||
} else {
|
||||
high = middle;
|
||||
}
|
||||
}
|
||||
return offset + (low > 0 ? (cumulativeDeltas[low - 1] ?? 0) : 0);
|
||||
};
|
||||
const mapRange = <T extends { start: number; end: number }>(range: T): T => {
|
||||
const exact = exactEdits.get(`${range.start}:${range.end}`);
|
||||
const start = mapOffset(range.start);
|
||||
return { ...range, start, end: exact ? start + exact.text.length : mapOffset(range.end) };
|
||||
};
|
||||
return {
|
||||
...ir,
|
||||
text,
|
||||
styles: ir.styles.map(mapRange),
|
||||
links: ir.links.map(mapRange),
|
||||
...(ir.annotations ? { annotations: ir.annotations.map(mapRange) } : {}),
|
||||
...(ir.listItems
|
||||
? {
|
||||
listItems: ir.listItems.map((item) => ({
|
||||
...item,
|
||||
...(item.listMarker ? { listMarker: mapRange(item.listMarker) } : {}),
|
||||
...(item.taskMarker ? { taskMarker: mapRange(item.taskMarker) } : {}),
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function prefixQQBotBlockquotes(ir: MarkdownIR): MarkdownIR {
|
||||
const quoteSpans = ir.styles.filter((span) => span.style === "blockquote");
|
||||
const edits = quoteSpans.flatMap((span) => {
|
||||
const positions: number[] = [];
|
||||
for (let index = span.start; index < span.end; index += 1) {
|
||||
if (ir.text[index] === "\n" && index + 1 < span.end) {
|
||||
positions.push(index + 1);
|
||||
}
|
||||
}
|
||||
return positions.map((position) => ({ start: position, end: position, text: "> " }));
|
||||
});
|
||||
return rewriteMarkdownIR(ir, edits);
|
||||
}
|
||||
|
||||
function escapeQQFallbackCode(
|
||||
ir: MarkdownIR,
|
||||
protectEscape: (escaped: string) => string,
|
||||
): MarkdownIR {
|
||||
return rewriteMarkdownIR(
|
||||
ir,
|
||||
ir.styles
|
||||
.filter((span) => span.style === "code" || span.style === "code_block")
|
||||
.map((span) => ({
|
||||
start: span.start,
|
||||
end: span.end,
|
||||
text: ir.text
|
||||
.slice(span.start, span.end)
|
||||
.replace(QQBOT_MARKDOWN_ESCAPE_RE, (char) => protectEscape(`\\${char}`)),
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
function specializeProtectedTokensInCode(
|
||||
ir: MarkdownIR,
|
||||
tokens: readonly string[],
|
||||
protectedTokens: ReturnType<typeof createProtectedTokenStore>,
|
||||
): MarkdownIR {
|
||||
const codeStyles = ir.styles.filter(
|
||||
(span) => span.style === "code" || span.style === "code_block",
|
||||
);
|
||||
const edits: TextEdit[] = [];
|
||||
const protectedSet = new Set(tokens);
|
||||
for (let start = 0; start < ir.text.length; start += 1) {
|
||||
const token = ir.text[start] ?? "";
|
||||
if (
|
||||
protectedSet.has(token) &&
|
||||
codeStyles.some((span) => start >= span.start && start + token.length <= span.end)
|
||||
) {
|
||||
const escaped = escapeQQMarkdownSyntax(protectedTokens.restore(token));
|
||||
edits.push({ start, end: start + token.length, text: protectedTokens.reuse(escaped) });
|
||||
}
|
||||
}
|
||||
return rewriteMarkdownIR(ir, edits);
|
||||
}
|
||||
|
||||
type ImageCandidateScan = { end: number } | { next: number } | undefined;
|
||||
|
||||
function blankBlockEnd(text: string, index: number): number | undefined {
|
||||
const match = /^(?:\r?\n)[ \t]*(?:\r?\n)/u.exec(text.slice(index));
|
||||
return match ? index + match[0].length : undefined;
|
||||
}
|
||||
|
||||
function scanQQBotMarkdownImage(text: string, start: number): ImageCandidateScan {
|
||||
let bracketDepth = 1;
|
||||
let altEnd: number | undefined;
|
||||
let fallbackNext: number | undefined;
|
||||
for (let index = start + 2; index < text.length; index += 1) {
|
||||
const blankEnd = blankBlockEnd(text, index);
|
||||
if (blankEnd !== undefined) {
|
||||
return { next: fallbackNext ?? blankEnd };
|
||||
}
|
||||
if (text[index] === "\\") {
|
||||
index += 1;
|
||||
} else if (text.startsWith("![", index)) {
|
||||
fallbackNext = index;
|
||||
bracketDepth += 1;
|
||||
index += 1;
|
||||
} else if (text[index] === "[") {
|
||||
bracketDepth += 1;
|
||||
} else if (text[index] === "]" && --bracketDepth === 0) {
|
||||
altEnd = index;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (altEnd === undefined || text[altEnd + 1] !== "(") {
|
||||
const next = fallbackNext ?? text.indexOf("![", altEnd === undefined ? start + 2 : altEnd + 1);
|
||||
return next < 0 ? undefined : { next };
|
||||
}
|
||||
|
||||
let parenDepth = 1;
|
||||
for (let index = altEnd + 2; index < text.length; index += 1) {
|
||||
const blankEnd = blankBlockEnd(text, index);
|
||||
if (blankEnd !== undefined) {
|
||||
return { next: fallbackNext ?? blankEnd };
|
||||
}
|
||||
if (text[index] === "\\") {
|
||||
index += 1;
|
||||
} else if (text.startsWith("![", index)) {
|
||||
fallbackNext = index;
|
||||
} else if (text[index] === "(") {
|
||||
parenDepth += 1;
|
||||
} else if (text[index] === ")" && --parenDepth === 0) {
|
||||
return { end: index + 1 };
|
||||
}
|
||||
}
|
||||
return fallbackNext === undefined ? undefined : { next: fallbackNext };
|
||||
}
|
||||
|
||||
function protectQQBotMarkdownImages(
|
||||
text: string,
|
||||
createToken: (image: string) => string,
|
||||
byteLimit: number,
|
||||
): { text: string; tokens: string[] } {
|
||||
let protectedText = "";
|
||||
let cursor = 0;
|
||||
let searchFrom = 0;
|
||||
const tokens: string[] = [];
|
||||
while (searchFrom < text.length) {
|
||||
const start = text.indexOf("![", searchFrom);
|
||||
if (start < 0) {
|
||||
break;
|
||||
}
|
||||
const scan = scanQQBotMarkdownImage(text, start);
|
||||
if (!scan) {
|
||||
break;
|
||||
}
|
||||
if ("next" in scan) {
|
||||
searchFrom = scan.next;
|
||||
continue;
|
||||
}
|
||||
let slashStart = start;
|
||||
while (slashStart > cursor && text[slashStart - 1] === "\\") {
|
||||
slashStart -= 1;
|
||||
}
|
||||
const escaped = (start - slashStart) % 2 === 1;
|
||||
const image = text.slice(start, scan.end);
|
||||
const protectedSize = Math.max(
|
||||
utf8ByteLength(image),
|
||||
utf8ByteLength(escapeQQMarkdownSyntax(image)),
|
||||
);
|
||||
if (protectedSize + PROTECTED_IMAGE_OVERHEAD_BYTES > byteLimit) {
|
||||
searchFrom = scan.end;
|
||||
continue;
|
||||
}
|
||||
protectedText += text.slice(cursor, escaped ? start - 1 : start);
|
||||
const token = createToken(escaped ? `\\${image}` : image);
|
||||
tokens.push(token);
|
||||
protectedText += token;
|
||||
cursor = scan.end;
|
||||
searchFrom = scan.end;
|
||||
}
|
||||
return { text: protectedText + text.slice(cursor), tokens };
|
||||
}
|
||||
|
||||
function serializeMarkdownDestination(href: string): string {
|
||||
return `<${href.replace(/([\\<>])/gu, "\\$1")}>`;
|
||||
}
|
||||
|
||||
function fallbackOversizedQQLinks(
|
||||
ir: MarkdownIR,
|
||||
byteLimit: number,
|
||||
render: (ir: MarkdownIR) => string,
|
||||
protectEscape: (escaped: string) => string,
|
||||
): MarkdownIR {
|
||||
const oversizedIndexes = new Set<number>();
|
||||
for (const [index, link] of ir.links.entries()) {
|
||||
const rendered = render(sliceMarkdownIR(ir, link.start, link.end));
|
||||
if (utf8ByteLength(rendered) > byteLimit) {
|
||||
oversizedIndexes.add(index);
|
||||
}
|
||||
}
|
||||
if (oversizedIndexes.size === 0) {
|
||||
return ir;
|
||||
}
|
||||
const oversized = ir.links.filter((_link, index) => oversizedIndexes.has(index));
|
||||
const rewritten = rewriteMarkdownIR(
|
||||
ir,
|
||||
oversized.map((link) => ({
|
||||
start: link.end,
|
||||
end: link.end,
|
||||
text: ` (${link.href.replace(QQBOT_MARKDOWN_ESCAPE_RE, (char) => protectEscape(`\\${char}`))})`,
|
||||
})),
|
||||
);
|
||||
return {
|
||||
...rewritten,
|
||||
links: rewritten.links.filter((_link, index) => !oversizedIndexes.has(index)),
|
||||
};
|
||||
}
|
||||
|
||||
function fallbackOversizedProtectedImages(
|
||||
ir: MarkdownIR,
|
||||
byteLimit: number,
|
||||
render: (ir: MarkdownIR) => string,
|
||||
protectedTokens: ReturnType<typeof createProtectedTokenStore>,
|
||||
): MarkdownIR {
|
||||
const edits: TextEdit[] = [];
|
||||
for (let start = 0; start < ir.text.length; start += 1) {
|
||||
const token = ir.text[start] ?? "";
|
||||
const protectedValue = protectedTokens.restore(token);
|
||||
const escapedLiteral = protectedValue.startsWith("\\![");
|
||||
const unescaped = protectedValue.replace(/\\(.)/gu, "$1");
|
||||
if (
|
||||
/^!?\[[\s\S]*\]\([\s\S]*\)$/u.test(unescaped) &&
|
||||
utf8ByteLength(render(sliceMarkdownIR(ir, start, start + token.length))) > byteLimit
|
||||
) {
|
||||
if (escapedLiteral) {
|
||||
const literal = protectedValue.replace(QQBOT_MARKDOWN_ESCAPE_RE, (char) =>
|
||||
protectedTokens.reuse(`\\${char}`),
|
||||
);
|
||||
edits.push({ start, end: start + token.length, text: literal });
|
||||
continue;
|
||||
}
|
||||
const altStart = protectedValue.startsWith("![") ? 2 : 1;
|
||||
let depth = 1;
|
||||
let altEnd = altStart;
|
||||
let alt = "";
|
||||
for (; altEnd < protectedValue.length; altEnd += 1) {
|
||||
if (protectedValue[altEnd] === "\\" && protectedValue[altEnd + 1]) {
|
||||
alt += protectedValue[++altEnd];
|
||||
} else if (protectedValue[altEnd] === "[") {
|
||||
depth += 1;
|
||||
alt += "[";
|
||||
} else if (protectedValue[altEnd] === "]" && --depth === 0) {
|
||||
break;
|
||||
} else {
|
||||
alt += protectedValue[altEnd] ?? "";
|
||||
}
|
||||
}
|
||||
edits.push({ start, end: start + token.length, text: alt });
|
||||
}
|
||||
}
|
||||
return rewriteMarkdownIR(ir, edits);
|
||||
}
|
||||
|
||||
export function formatQQBotMarkdown(markdown: string, limit: number): string[] {
|
||||
const protectedTokens = createProtectedTokenStore(markdown);
|
||||
const chunkLimit = resolveQQBotMarkdownChunkLimit(limit);
|
||||
const images = protectQQBotMarkdownImages(markdown, protectedTokens.reuse, chunkLimit);
|
||||
const entityTokens: string[] = [];
|
||||
const entitiesProtected = images.text.replace(MARKDOWN_ENTITY_RE, (entity) => {
|
||||
const protectedSize = Math.max(
|
||||
utf8ByteLength(entity),
|
||||
utf8ByteLength(escapeQQMarkdownSyntax(entity)),
|
||||
);
|
||||
if (protectedSize + PROTECTED_IMAGE_OVERHEAD_BYTES > chunkLimit) {
|
||||
return entity;
|
||||
}
|
||||
const token = protectedTokens.reuse(entity);
|
||||
entityTokens.push(token);
|
||||
return token;
|
||||
});
|
||||
const escapeTokens: string[] = [];
|
||||
const protectedMarkdown = entitiesProtected.replace(ESCAPED_MARKDOWN_RE, (escaped) => {
|
||||
const token = protectedTokens.reuse(escaped);
|
||||
escapeTokens.push(token);
|
||||
return token;
|
||||
});
|
||||
const parsed = markdownToIR(protectedMarkdown, {
|
||||
autolink: false,
|
||||
enableSpoilers: true,
|
||||
enableTaskLists: true,
|
||||
headingStyle: "rich",
|
||||
linkify: false,
|
||||
blockquotePrefix: "",
|
||||
});
|
||||
const specialized = specializeProtectedTokensInCode(
|
||||
specializeProtectedTokensInCode(parsed, images.tokens, protectedTokens),
|
||||
[...escapeTokens, ...entityTokens],
|
||||
protectedTokens,
|
||||
);
|
||||
const renderChunk = (chunk: MarkdownIR): string =>
|
||||
protectedTokens.restore(
|
||||
renderMarkdownWithMarkers(
|
||||
chunk,
|
||||
{
|
||||
styleMarkers: {
|
||||
...QQBOT_MARKERS,
|
||||
blockquote: {
|
||||
open: (span: { start: number }) =>
|
||||
chunk.text.slice(span.start, span.start + 2) === "> " ? "" : "> ",
|
||||
close: "",
|
||||
},
|
||||
},
|
||||
escapeText: (text) => text,
|
||||
buildLink: (link) => ({
|
||||
start: link.start,
|
||||
end: link.end,
|
||||
open: "[",
|
||||
close: `](${serializeMarkdownDestination(link.href)})`,
|
||||
}),
|
||||
},
|
||||
QQBOT_FORMAT_CAPABILITIES,
|
||||
),
|
||||
);
|
||||
const formatted = prefixQQBotBlockquotes(
|
||||
escapeQQFallbackCode(specialized, protectedTokens.reuse),
|
||||
);
|
||||
const imagesSized = fallbackOversizedProtectedImages(
|
||||
formatted,
|
||||
chunkLimit,
|
||||
renderChunk,
|
||||
protectedTokens,
|
||||
);
|
||||
const ir = fallbackOversizedQQLinks(imagesSized, chunkLimit, renderChunk, protectedTokens.reuse);
|
||||
const chunks = renderMarkdownIRChunksWithinLimit({
|
||||
ir,
|
||||
limit: chunkLimit,
|
||||
measureRendered: utf8ByteLength,
|
||||
renderChunk,
|
||||
}).map((chunk) => chunk.rendered);
|
||||
const last = chunks.length - 1;
|
||||
if (last >= 0) {
|
||||
chunks[last] = chunks[last]?.trimEnd() ?? "";
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
@@ -6,6 +6,194 @@ const baseChunker = (text: string, limit: number): string[] =>
|
||||
text.length <= limit ? [text] : [text.slice(0, limit), text.slice(limit)];
|
||||
|
||||
describe("chunkQQBotMarkdownText", () => {
|
||||
it("falls unsupported inline code back to plain text", () => {
|
||||
expect(chunkQQBotMarkdownText("Run `openclaw status` now.", 120, baseChunker)).toEqual([
|
||||
"Run openclaw status now.",
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves transport-owned markdown images beside fallback code", () => {
|
||||
const image = "";
|
||||
expect(chunkQQBotMarkdownText(`Run \`status\`.\n\n${image}`, 200, baseChunker)).toEqual([
|
||||
`Run status.\n\n${image}`,
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves transport-owned image URLs with balanced parentheses", () => {
|
||||
const image = ".png)";
|
||||
expect(chunkQQBotMarkdownText(`Run \`status\`.\n\n${image}`, 200, baseChunker)).toEqual([
|
||||
`Run status.\n\n${image}`,
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves images containing nested opener text", () => {
|
||||
const image = "";
|
||||
expect(chunkQQBotMarkdownText(image, 200, baseChunker)).toEqual([image]);
|
||||
});
|
||||
|
||||
it("keeps BMP protected image tokens atomic at the chunk boundary", () => {
|
||||
const image = "";
|
||||
const output = chunkQQBotMarkdownText(`${"A".repeat(3_597)}${image}`, 3_600, baseChunker);
|
||||
expect(output.join("")).toBe(`${"A".repeat(3_597)}${image}`);
|
||||
expect(output.every((chunk) => !chunk.includes("�"))).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps escaped images atomic at the chunk boundary", () => {
|
||||
const image = String.raw`\`;
|
||||
const chunks = chunkQQBotMarkdownText(`${"A".repeat(3_599)}${image}`, 3_600, baseChunker);
|
||||
expect(chunks.join("")).toBe(`${"A".repeat(3_599)}${image}`);
|
||||
expect(chunks.some((chunk) => chunk.startsWith("!["))).toBe(false);
|
||||
});
|
||||
|
||||
it("falls protected images back when final quote context exceeds the limit", () => {
|
||||
const image = `![x](https://example.com/${"a".repeat(3_400)}.png)`;
|
||||
const chunks = chunkQQBotMarkdownText(`${"> ".repeat(40)}${image}`, 3_600, baseChunker);
|
||||
expect(chunks.every((chunk) => Buffer.byteLength(chunk, "utf8") <= 3_600)).toBe(true);
|
||||
expect(chunks.join("")).toContain("![x]");
|
||||
});
|
||||
|
||||
it("does not hide later code behind malformed images", () => {
|
||||
const output = chunkQQBotMarkdownText("", 200, baseChunker).join("");
|
||||
expect(output).toContain("code");
|
||||
expect(output).not.toContain("`code`");
|
||||
});
|
||||
|
||||
it("does not let nested images complete malformed outer candidates", () => {
|
||||
const output = chunkQQBotMarkdownText(
|
||||
"",
|
||||
200,
|
||||
baseChunker,
|
||||
).join("");
|
||||
expect(output).not.toContain("`code`");
|
||||
expect(output).toContain("");
|
||||
});
|
||||
|
||||
it("continues image protection after many malformed candidates", () => {
|
||||
const image = "";
|
||||
const chunks = chunkQQBotMarkdownText(`${"";
|
||||
const output = chunkQQBotMarkdownText(`󰀀 ${image}`, 200, baseChunker).join("");
|
||||
expect(output.startsWith("󰀀 ")).toBe(true);
|
||||
expect(output.match(/!\[x\]/gu)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("preserves entity-encoded markdown literals", () => {
|
||||
const source = "**literal**";
|
||||
expect(chunkQQBotMarkdownText(source, 200, baseChunker)).toEqual([source]);
|
||||
});
|
||||
|
||||
it("restores entities nested inside protected images", () => {
|
||||
const image = "";
|
||||
expect(chunkQQBotMarkdownText(image, 200, baseChunker)).toEqual([image]);
|
||||
});
|
||||
|
||||
it("keeps oversized entities chunkable", () => {
|
||||
const source = `&#${"1".repeat(300)};`;
|
||||
const chunks = chunkQQBotMarkdownText(source, 100, baseChunker);
|
||||
expect(chunks.every((chunk) => Buffer.byteLength(chunk, "utf8") <= 100)).toBe(true);
|
||||
});
|
||||
|
||||
it("falls oversized images back to chunkable plain content", () => {
|
||||
const image = `![x](https://example.com/${"a".repeat(4_000)}.png)`;
|
||||
const chunks = chunkQQBotMarkdownText(image, 3_600, baseChunker);
|
||||
expect(chunks.every((chunk) => Buffer.byteLength(chunk, "utf8") <= 3_600)).toBe(true);
|
||||
expect(chunks.join("")).toBe("x");
|
||||
});
|
||||
|
||||
it("falls oversized links back to chunkable plain content", () => {
|
||||
const href = `https://example.com/${"a".repeat(4_000)}`;
|
||||
const escapedHref = href.replaceAll(".", "\\.");
|
||||
const chunks = chunkQQBotMarkdownText(`[x](${href})`, 3_600, baseChunker);
|
||||
expect(chunks.every((chunk) => Buffer.byteLength(chunk, "utf8") <= 3_600)).toBe(true);
|
||||
expect(chunks.join("")).toBe(`x (${escapedHref})`);
|
||||
});
|
||||
|
||||
it("removes only the oversized occurrence when link destinations repeat", () => {
|
||||
const href = `https://e.co/${"a".repeat(3_575)}`;
|
||||
const escapedHref = href.replaceAll(".", "\\.");
|
||||
const source = `[x](${href})\n[${"long".repeat(8)}](${href})`;
|
||||
const output = chunkQQBotMarkdownText(source, 3_600, baseChunker).join("");
|
||||
expect(output).toContain(`[x](<${href}>)`);
|
||||
expect(output).toContain(`${"long".repeat(8)} (${escapedHref})`);
|
||||
});
|
||||
|
||||
it("supports more authored escapes than the BMP private-use block", () => {
|
||||
const source = "\\*".repeat(6_401);
|
||||
expect(chunkQQBotMarkdownText(source, 3_600, baseChunker).join("")).toBe(source);
|
||||
});
|
||||
|
||||
it("escapes markdown-looking inline code after removing code markers", () => {
|
||||
expect(chunkQQBotMarkdownText("``", 200, baseChunker)).toEqual([
|
||||
String.raw`\!\[x\]\(https://example\.com/x\.png\)`,
|
||||
]);
|
||||
});
|
||||
|
||||
it("matches equal-length inline delimiters around shorter backtick runs", () => {
|
||||
expect(
|
||||
chunkQQBotMarkdownText("``a `` b``", 200, baseChunker),
|
||||
).toEqual([String.raw`a \`\!\[x\]\(https://example\.com/x\.png\)\` b`]);
|
||||
});
|
||||
|
||||
it("preserves escaped literal backticks", () => {
|
||||
expect(chunkQQBotMarkdownText(String.raw`\`literal\``, 200, baseChunker)).toEqual([
|
||||
String.raw`\`literal\``,
|
||||
]);
|
||||
});
|
||||
|
||||
it("re-escapes protected backslashes inside fallback code", () => {
|
||||
expect(chunkQQBotMarkdownText("`\\*`", 200, baseChunker)).toEqual([String.raw`\\\*`]);
|
||||
});
|
||||
|
||||
it("serializes link destinations with angle brackets", () => {
|
||||
expect(chunkQQBotMarkdownText("[x](https://host/a)", 200, baseChunker)).toEqual([
|
||||
"[x](<https://host/a>)",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps every paragraph inside a blockquote", () => {
|
||||
expect(chunkQQBotMarkdownText("> one\n>\n> two", 200, baseChunker)).toEqual([
|
||||
"> one\n> \n> two",
|
||||
]);
|
||||
});
|
||||
|
||||
it("stops blockquote prefixes before following text", () => {
|
||||
expect(chunkQQBotMarkdownText("> quoted\n\noutside", 200, baseChunker)).toEqual([
|
||||
"> quoted\n\noutside",
|
||||
]);
|
||||
});
|
||||
|
||||
it("prefixes every chunk of a long blockquote", () => {
|
||||
const chunks = chunkQQBotMarkdownText(`> ${"a".repeat(5_000)}`, 200, baseChunker);
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
expect(chunks.every((chunk) => chunk.startsWith("> "))).toBe(true);
|
||||
});
|
||||
|
||||
it("does not duplicate blockquote prefixes at continuation boundaries", () => {
|
||||
const chunks = chunkQQBotMarkdownText(`> ${"a".repeat(3_597)}\n> second`, 3_600, baseChunker);
|
||||
expect(chunks.some((chunk) => chunk.startsWith("> > "))).toBe(false);
|
||||
expect(chunks.join("")).toContain("> second");
|
||||
});
|
||||
|
||||
it("keeps fallback code lines inside a blockquote", () => {
|
||||
expect(chunkQQBotMarkdownText("> ```\n> one\n> two\n> ```", 200, baseChunker)).toEqual([
|
||||
"> one\n> two",
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not linkify plain filenames", () => {
|
||||
expect(chunkQQBotMarkdownText("See README.md", 200, baseChunker)).toEqual(["See README.md"]);
|
||||
});
|
||||
|
||||
it("keeps nested list indentation out of code fallback", () => {
|
||||
expect(chunkQQBotMarkdownText("- parent\n - child", 200, baseChunker)).toEqual([
|
||||
"• parent\n • child",
|
||||
]);
|
||||
});
|
||||
|
||||
it("prefixes continuation chunks with the active table header", () => {
|
||||
const text = [
|
||||
"| Id | Value |",
|
||||
@@ -162,12 +350,39 @@ describe("chunkQQBotMarkdownText", () => {
|
||||
expect(chunker.flushPendingText(160)).toEqual(["5 reportbuilder.ts generatemonthly_sales"]);
|
||||
});
|
||||
|
||||
it("keeps fenced code blocks self-contained across streaming block flushes", () => {
|
||||
it("falls fenced code blocks back to plain text across streaming block flushes", () => {
|
||||
const chunker = createQQBotMarkdownChunker((text) => [text]);
|
||||
|
||||
expect(chunker.chunkText(["```ts", "const a = 1;"].join("\n"), 200)).toEqual([]);
|
||||
expect(chunker.chunkText(["const b = 2;", "```"].join("\n"), 200)).toEqual([
|
||||
["```ts", "const a = 1;", "const b = 2;", "```"].join("\n"),
|
||||
["const a = 1;", "const b = 2;"].join("\n"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps streamed template-literal backticks as escaped plain text", () => {
|
||||
const chunker = createQQBotMarkdownChunker((text) => [text]);
|
||||
|
||||
expect(chunker.chunkText(["```ts", "const value = `hello`;"].join("\n"), 200)).toEqual([]);
|
||||
expect(chunker.chunkText("```", 200)).toEqual([String.raw`const value = \`hello\`;`]);
|
||||
});
|
||||
|
||||
it("keeps markdown-looking streamed fence bodies in code fallback", () => {
|
||||
const chunker = createQQBotMarkdownChunker((text) => [text]);
|
||||
expect(chunker.chunkText(["```", "**literal**"].join("\n"), 200)).toEqual([]);
|
||||
expect(chunker.chunkText("```", 200)).toEqual([String.raw`\*\*literal\*\*`]);
|
||||
});
|
||||
|
||||
it("handles longer fences containing shorter fence examples", () => {
|
||||
const markdown = ["````md", "```", "inside", "```", "```` "].join("\n");
|
||||
expect(chunkQQBotMarkdownText(markdown, 200, baseChunker)).toEqual([
|
||||
[String.raw`\`\`\``, "inside", String.raw`\`\`\``].join("\n"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("escapes markdown-looking indented code after fallback", () => {
|
||||
const markdown = [" **literal**", " "].join("\n");
|
||||
expect(chunkQQBotMarkdownText(markdown, 200, baseChunker)).toEqual([
|
||||
[String.raw`\*\*literal\*\*`, String.raw`\!\[x\]\(https://example\.com/x\.png\)`].join("\n"),
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -180,13 +395,14 @@ describe("chunkQQBotMarkdownText", () => {
|
||||
expect(
|
||||
chunker.chunkText(["0", " def get_dsn(self) -> str:", "```"].join("\n"), 200),
|
||||
).toEqual([
|
||||
["```python", " pool_timeout: float = 30.0", " def get_dsn(self) -> str:", "```"].join(
|
||||
"\n",
|
||||
),
|
||||
[
|
||||
String.raw` pool\_timeout: float = 30\.0`,
|
||||
String.raw` def get\_dsn\(self\) \-\> str:`,
|
||||
].join("\n"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps long fenced chunks under the QQ markdown byte safety limit", () => {
|
||||
it("keeps long fallback code chunks under the QQ markdown byte safety limit", () => {
|
||||
const lines = Array.from(
|
||||
{ length: 90 },
|
||||
(_, index) =>
|
||||
@@ -198,11 +414,26 @@ describe("chunkQQBotMarkdownText", () => {
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const chunk of chunks) {
|
||||
expect(Buffer.byteLength(chunk, "utf8")).toBeLessThanOrEqual(3600);
|
||||
expect(chunk.startsWith("```python\n")).toBe(true);
|
||||
expect(chunk.endsWith("\n```")).toBe(true);
|
||||
expect(chunk).not.toContain("```");
|
||||
}
|
||||
});
|
||||
|
||||
it("does not split generated markdown escape pairs across byte chunks", () => {
|
||||
const chunks = chunkQQBotMarkdownText(
|
||||
["```", "*".repeat(5_000), "```"].join("\n"),
|
||||
3_600,
|
||||
baseChunker,
|
||||
);
|
||||
expect(chunks.join("")).toBe("\\*".repeat(5_000));
|
||||
expect(chunks.every((chunk) => !/(^|[^\\])(?:\\\\)*\\$/u.test(chunk))).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps generated markdown escape pairs atomic at odd byte limits", () => {
|
||||
const chunks = chunkQQBotMarkdownText(["```", "***", "```"].join("\n"), 3, baseChunker);
|
||||
expect(chunks.join("")).toBe("\\*".repeat(3));
|
||||
expect(chunks.every((chunk) => !chunk.endsWith("\\"))).toBe(true);
|
||||
});
|
||||
|
||||
it("allows ASCII fenced chunks past the old 1800 character fallback", () => {
|
||||
const lines = Array.from(
|
||||
{ length: 90 },
|
||||
@@ -214,17 +445,16 @@ describe("chunkQQBotMarkdownText", () => {
|
||||
expect(chunks.some((chunk) => chunk.length > 1800)).toBe(true);
|
||||
for (const chunk of chunks) {
|
||||
expect(Buffer.byteLength(chunk, "utf8")).toBeLessThanOrEqual(3600);
|
||||
expect(chunk.startsWith("```python\n")).toBe(true);
|
||||
expect(chunk.endsWith("\n```")).toBe(true);
|
||||
expect(chunk).not.toContain("```");
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps fenced formula blocks self-contained across streaming block flushes", () => {
|
||||
it("falls fenced formula blocks back to plain text across streaming block flushes", () => {
|
||||
const chunker = createQQBotMarkdownChunker((text) => [text]);
|
||||
|
||||
expect(chunker.chunkText(["```math", "E = mc^2"].join("\n"), 200)).toEqual([]);
|
||||
expect(chunker.chunkText(["a^2 + b^2 = c^2", "```"].join("\n"), 200)).toEqual([
|
||||
["```math", "E = mc^2", "a^2 + b^2 = c^2", "```"].join("\n"),
|
||||
["E = mc^2", String.raw`a^2 \+ b^2 = c^2`].join("\n"),
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -241,10 +471,7 @@ describe("chunkQQBotMarkdownText", () => {
|
||||
...chunker.flushPendingText(limit),
|
||||
];
|
||||
|
||||
expect(chunks).toEqual([
|
||||
["```ts", firstLine, "```"].join("\n"),
|
||||
["```ts", secondLine, "```"].join("\n"),
|
||||
]);
|
||||
expect(chunks).toEqual([firstLine, secondLine]);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
// QQ Bot Markdown chunking keeps each sent message self-contained.
|
||||
|
||||
import { formatQQBotMarkdown } from "./markdown-format.js";
|
||||
|
||||
type QQBotBaseMarkdownChunker = (text: string, limit: number) => string[];
|
||||
|
||||
const QQBOT_MARKDOWN_SAFE_CHUNK_BYTE_LIMIT = 3600;
|
||||
@@ -233,9 +235,7 @@ class QQBotMarkdownChunkingState {
|
||||
if (this.textLines.length === 0) {
|
||||
return;
|
||||
}
|
||||
if (this.flushFenceText(chunks, limit)) {
|
||||
return;
|
||||
}
|
||||
const continuedFenceOpenLine = this.activeFence?.openLine;
|
||||
let text = this.textLines.join("\n");
|
||||
this.textLines = [];
|
||||
if (this.pendingTextFenceOpenLine) {
|
||||
@@ -248,44 +248,23 @@ class QQBotMarkdownChunkingState {
|
||||
if (!text) {
|
||||
return;
|
||||
}
|
||||
pushBaseChunks(chunks, text, limit, this.baseChunker);
|
||||
}
|
||||
|
||||
private flushFenceText(chunks: string[], limit: number): boolean {
|
||||
const pendingFenceOpenLine = this.pendingTextFenceOpenLine;
|
||||
const firstLineFence = pendingFenceOpenLine ? null : parseFenceLine(this.textLines[0] ?? "");
|
||||
const fence = pendingFenceOpenLine ? parseFenceLine(pendingFenceOpenLine) : firstLineFence;
|
||||
if (!fence) {
|
||||
return false;
|
||||
chunks.push(...formatQQBotMarkdown(text, limit));
|
||||
if (continuedFenceOpenLine) {
|
||||
this.pendingTextFenceOpenLine = continuedFenceOpenLine;
|
||||
}
|
||||
|
||||
const bodyLines = pendingFenceOpenLine ? [...this.textLines] : this.textLines.slice(1);
|
||||
this.textLines = [];
|
||||
this.pendingTextFenceOpenLine = null;
|
||||
const lastBodyLine = bodyLines.at(-1);
|
||||
if (lastBodyLine !== undefined && isClosingFenceLine(lastBodyLine, fence)) {
|
||||
bodyLines.pop();
|
||||
}
|
||||
if (this.activeFence && bodyLines.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
pushFenceLineChunks({
|
||||
chunks,
|
||||
openLine: fence.openLine,
|
||||
closeLine: fence.closeLine,
|
||||
bodyLines,
|
||||
limit,
|
||||
baseChunker: this.baseChunker,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
private consumePendingFenceLinePrefix(text: string): string {
|
||||
if (!this.pendingFenceLineFragment) {
|
||||
return text;
|
||||
}
|
||||
const separator = shouldJoinFenceLineFragments(this.pendingFenceLineFragment, text) ? "" : "\n";
|
||||
const firstLine = text.split("\n", 1)[0] ?? "";
|
||||
const startsWithClosingFence =
|
||||
this.activeFence && isClosingFenceLine(firstLine, this.activeFence);
|
||||
const separator =
|
||||
!startsWithClosingFence && shouldJoinFenceLineFragments(this.pendingFenceLineFragment, text)
|
||||
? ""
|
||||
: "\n";
|
||||
const merged = `${this.pendingFenceLineFragment}${separator}${text}`;
|
||||
this.pendingFenceLineFragment = null;
|
||||
return merged;
|
||||
@@ -376,7 +355,24 @@ function pushBaseChunks(
|
||||
byteLimit: number,
|
||||
baseChunker: QQBotBaseMarkdownChunker,
|
||||
): void {
|
||||
for (const chunk of baseChunker(text, byteLimit)) {
|
||||
const baseChunks = baseChunker(text, byteLimit).filter(Boolean);
|
||||
for (let index = 0; index + 1 < baseChunks.length; index += 1) {
|
||||
const chunk = baseChunks[index] ?? "";
|
||||
if (!/(^|[^\\])(?:\\\\)*\\$/u.test(chunk)) {
|
||||
continue;
|
||||
}
|
||||
const next = baseChunks[index + 1] ?? "";
|
||||
const firstCodePoint = next.codePointAt(0);
|
||||
const first = firstCodePoint === undefined ? "" : String.fromCodePoint(firstCodePoint);
|
||||
if (first && utf8ByteLength(chunk + first) <= byteLimit) {
|
||||
baseChunks[index] = chunk + first;
|
||||
baseChunks[index + 1] = next.slice(first.length);
|
||||
} else {
|
||||
baseChunks[index] = chunk.slice(0, -1);
|
||||
baseChunks[index + 1] = `\\${next}`;
|
||||
}
|
||||
}
|
||||
for (const chunk of baseChunks) {
|
||||
if (!chunk) {
|
||||
continue;
|
||||
}
|
||||
@@ -395,15 +391,20 @@ function splitByUtf8ByteLimit(text: string, byteLimit: number): string[] {
|
||||
const chunks: string[] = [];
|
||||
let current = "";
|
||||
let currentBytes = 0;
|
||||
for (const char of text) {
|
||||
const charBytes = utf8ByteLength(char);
|
||||
if (current && currentBytes + charBytes > byteLimit) {
|
||||
const chars = Array.from(text);
|
||||
for (let index = 0; index < chars.length; index += 1) {
|
||||
const char = chars[index] ?? "";
|
||||
const escapedUnit = char === "\\" && chars[index + 1] ? `${char}${chars[index + 1]}` : "";
|
||||
const unit =
|
||||
escapedUnit && utf8ByteLength(escapedUnit) <= byteLimit ? `${char}${chars[++index]}` : char;
|
||||
const unitBytes = utf8ByteLength(unit);
|
||||
if (current && currentBytes + unitBytes > byteLimit) {
|
||||
chunks.push(current);
|
||||
current = "";
|
||||
currentBytes = 0;
|
||||
}
|
||||
current += char;
|
||||
currentBytes += charBytes;
|
||||
current += unit;
|
||||
currentBytes += unitBytes;
|
||||
}
|
||||
if (current) {
|
||||
chunks.push(current);
|
||||
@@ -559,45 +560,6 @@ function renderTableRowAsFields(headers: string[], cells: string[]): string {
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function pushFenceLineChunks(params: {
|
||||
chunks: string[];
|
||||
openLine: string;
|
||||
closeLine: string;
|
||||
bodyLines: string[];
|
||||
limit: number;
|
||||
baseChunker: QQBotBaseMarkdownChunker;
|
||||
}): void {
|
||||
const { chunks, openLine, closeLine, bodyLines, limit, baseChunker } = params;
|
||||
let currentLines: string[] = [];
|
||||
const render = (lines: string[]) => [openLine, ...lines, closeLine].join("\n");
|
||||
const flushCurrent = (): void => {
|
||||
if (currentLines.length === 0) {
|
||||
return;
|
||||
}
|
||||
chunks.push(render(currentLines));
|
||||
currentLines = [];
|
||||
};
|
||||
|
||||
for (const line of bodyLines) {
|
||||
const candidate = [...currentLines, line];
|
||||
if (utf8ByteLength(render(candidate)) <= limit) {
|
||||
currentLines = candidate;
|
||||
continue;
|
||||
}
|
||||
flushCurrent();
|
||||
const singleLineChunk = render([line]);
|
||||
if (utf8ByteLength(singleLineChunk) <= limit) {
|
||||
currentLines = [line];
|
||||
continue;
|
||||
}
|
||||
pushBaseChunks(chunks, singleLineChunk, limit, baseChunker);
|
||||
}
|
||||
|
||||
if (currentLines.length > 0 || bodyLines.length === 0) {
|
||||
chunks.push(render(currentLines));
|
||||
}
|
||||
}
|
||||
|
||||
function parseFenceLine(line: string): ActiveFence | null {
|
||||
const match = line.match(/^(\s*)(`{3,}|~{3,})/);
|
||||
if (!match?.[2]) {
|
||||
|
||||
Reference in New Issue
Block a user