mirror of
https://github.com/open-webui/open-webui.git
synced 2026-08-13 01:02:25 -06:00
feat: parse :::writing block metadata and use the subject as the block title (#28280)
Newer OpenAI chat models put metadata on the opening line of a colon fence block, like :::writing{variant="email" id="48173" subject="Short question" recipient="mail@example.com"}. The tokenizer matched that line and discarded it, so every block rendered under the same generic "Writing" heading no matter what it contained.
The opening line is now parsed into an attributes map on the token and the header uses it: the subject becomes the title, the recipient follows it and the full string is reachable on hover when the row is too narrow for it. Blocks without metadata render exactly as before, and the other fence types get the parsed attributes for free.
Attributes are read only from inside the {...} braces, not from the whole opening line. Scanning the whole line turned ordinary prose containing key="value" into metadata, and it backtracked quadratically: a 40k character opening line took 586ms to parse, and that runs again on every re-lex while the message streams. Anchored to the braces it is 0.0ms.
Nothing here turns the recipient into a link or a send action. That metadata is model output and can be steered by whatever is in the context, so a prefilled mail action is a separate decision rather than a side effect of parsing.
This commit is contained in:
@@ -18,9 +18,12 @@
|
||||
export let onTaskClick: Function = () => {};
|
||||
export let onSourceClick: Function = () => {};
|
||||
|
||||
const fenceType: string = token.fenceType ?? 'default';
|
||||
$: fenceType = token.fenceType ?? 'default';
|
||||
$: attributes = token.attributes ?? {};
|
||||
|
||||
const label = fenceType.replace(/_/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase());
|
||||
$: label =
|
||||
attributes.subject || fenceType.replace(/_/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase());
|
||||
$: header = attributes.recipient ? `${label} · ${attributes.recipient}` : label;
|
||||
|
||||
let copied = false;
|
||||
|
||||
@@ -34,13 +37,16 @@
|
||||
</script>
|
||||
|
||||
<div class="relative group my-2 rounded-2xl border border-gray-100 dark:border-gray-800 px-4 py-3">
|
||||
<!-- Header row: type badge + copy button -->
|
||||
<div class="flex items-center justify-between mb-2">
|
||||
<span class="text-xs font-normal text-gray-500 dark:text-gray-400">
|
||||
{label}
|
||||
<div class="flex items-center justify-between gap-2 mb-2">
|
||||
<span
|
||||
class="text-xs font-normal text-gray-500 dark:text-gray-400 truncate"
|
||||
dir="auto"
|
||||
title={header}
|
||||
>
|
||||
{header}
|
||||
</span>
|
||||
|
||||
<div class="invisible group-hover:visible flex gap-0.5">
|
||||
<div class="invisible group-hover:visible flex gap-0.5 shrink-0">
|
||||
<Tooltip content={copied ? $i18n.t('Copied') : $i18n.t('Copy')}>
|
||||
<button
|
||||
class="p-1 rounded-lg bg-transparent hover:bg-black/5 dark:hover:bg-white/5 transition"
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { Marked } from 'marked';
|
||||
|
||||
import colonFenceExtension from './colon-fence-extension';
|
||||
|
||||
type ColonFenceToken = {
|
||||
type: string;
|
||||
fenceType: string;
|
||||
attributes: Record<string, string>;
|
||||
text: string;
|
||||
};
|
||||
|
||||
const firstFence = (src: string) => {
|
||||
const marked = new Marked();
|
||||
marked.use(colonFenceExtension());
|
||||
const tokens = marked.lexer(src) as unknown as ColonFenceToken[];
|
||||
return tokens.find((token) => token.type === 'colonFence') as ColonFenceToken;
|
||||
};
|
||||
|
||||
describe('colon fence extension', () => {
|
||||
it('tokenizes a block without attributes', () => {
|
||||
const token = firstFence(':::writing\n\nHello\n:::\n');
|
||||
|
||||
expect(token.fenceType).toBe('writing');
|
||||
expect(token.attributes).toEqual({});
|
||||
expect(token.text).toBe('Hello');
|
||||
});
|
||||
|
||||
it('parses the attributes on the opening line', () => {
|
||||
const token = firstFence(
|
||||
':::writing{variant="email" id="48173" subject="Short question" recipient="mail@example.com"}\n\nHello\n:::\n'
|
||||
);
|
||||
|
||||
expect(token.attributes).toEqual({
|
||||
variant: 'email',
|
||||
id: '48173',
|
||||
subject: 'Short question',
|
||||
recipient: 'mail@example.com'
|
||||
});
|
||||
expect(token.text).toBe('Hello');
|
||||
});
|
||||
|
||||
it('does not read attributes from unbraced trailing text', () => {
|
||||
const token = firstFence(':::writing{variant="document"} draft subject="junk"\n\nHello\n:::\n');
|
||||
|
||||
expect(token.attributes).toEqual({ variant: 'document' });
|
||||
});
|
||||
|
||||
it('ignores an opening line that has no braces at all', () => {
|
||||
const token = firstFence(
|
||||
':::note prose mentioning recipient="evil@example.com"\n\nHello\n:::\n'
|
||||
);
|
||||
|
||||
expect(token.fenceType).toBe('note');
|
||||
expect(token.attributes).toEqual({});
|
||||
});
|
||||
|
||||
it('ignores an unclosed brace', () => {
|
||||
const token = firstFence(':::writing{subject="Short question"\n\nHello\n:::\n');
|
||||
|
||||
expect(token.attributes).toEqual({});
|
||||
});
|
||||
|
||||
it('allows braces inside an attribute value', () => {
|
||||
const token = firstFence(':::writing{subject="use {x} here"}\n\nHello\n:::\n');
|
||||
|
||||
expect(token.attributes.subject).toBe('use {x} here');
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* Marked extension for colon-fence blocks (:::type ... :::)
|
||||
* Marked extension for colon-fence blocks (:::type{key="value"} ... :::)
|
||||
*
|
||||
* Used by newer OpenAI chat models to wrap semantically distinct content:
|
||||
* :::writing – reusable prose (letters, articles, docs)
|
||||
@@ -9,12 +9,21 @@
|
||||
* The extension is generic and will tokenize any :::<identifier> block.
|
||||
*/
|
||||
|
||||
function parseAttributes(attributeList: string): Record<string, string> {
|
||||
const attributes: Record<string, string> = {};
|
||||
for (const [, key, value] of attributeList.matchAll(/([\w-]+)="([^"]*)"/g)) {
|
||||
attributes[key] = value;
|
||||
}
|
||||
return attributes;
|
||||
}
|
||||
|
||||
function colonFenceTokenizer(this: any, src: string) {
|
||||
// Match :::type at the start of a line, optionally followed by content, then closing :::
|
||||
const match = /^:::([\w-]+)[^\n]*\n([\s\S]*?)(?:\n:::(?:\s*(?:\n|$)))/.exec(src);
|
||||
const match = /^:::([\w-]+)(?:\{([^\n]*)\})?[^\n]*\n([\s\S]*?)(?:\n:::(?:\s*(?:\n|$)))/.exec(src);
|
||||
if (match) {
|
||||
const fenceType = match[1];
|
||||
const text = match[2].trim();
|
||||
const attributes = parseAttributes(match[2] ?? '');
|
||||
const text = match[3].trim();
|
||||
const raw = match[0];
|
||||
|
||||
const tokens: any[] = [];
|
||||
@@ -24,6 +33,7 @@ function colonFenceTokenizer(this: any, src: string) {
|
||||
type: 'colonFence',
|
||||
raw,
|
||||
fenceType,
|
||||
attributes,
|
||||
text,
|
||||
tokens
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user