feat(gateway): define system-agent QR contract

This commit is contained in:
jesse-merhi
2026-08-12 00:15:23 +10:00
parent 2a1eba7713
commit 6bc9a7be7e
14 changed files with 390 additions and 34 deletions
@@ -10988,6 +10988,8 @@ public struct WizardStep: Codable, Sendable {
public let executor: AnyCodable?
public let externalurl: String?
public let devicecode: [String: AnyCodable]?
public let qrdataurl: String?
public let expiresinms: Int?
public init(
id: String,
@@ -11001,7 +11003,9 @@ public struct WizardStep: Codable, Sendable {
sensitive: Bool? = nil,
executor: AnyCodable? = nil,
externalurl: String? = nil,
devicecode: [String: AnyCodable]? = nil)
devicecode: [String: AnyCodable]? = nil,
qrdataurl: String? = nil,
expiresinms: Int? = nil)
{
self.id = id
self.type = type
@@ -11015,6 +11019,8 @@ public struct WizardStep: Codable, Sendable {
self.executor = executor
self.externalurl = externalurl
self.devicecode = devicecode
self.qrdataurl = qrdataurl
self.expiresinms = expiresinms
}
private enum CodingKeys: String, CodingKey {
@@ -11030,6 +11036,8 @@ public struct WizardStep: Codable, Sendable {
case executor
case externalurl = "externalUrl"
case devicecode = "deviceCode"
case qrdataurl = "qrDataUrl"
case expiresinms = "expiresInMs"
}
}
@@ -87,6 +87,38 @@ struct SystemAgentChatQuestionTests {
}
}
@Test
func `valid QR wizard step decodes and round trips`() throws {
let json =
"""
{
"sessionId": "test-session",
"reply": "Scan the code.",
"action": "none",
"step": {
"id": "setup-qr",
"type": "qr",
"title": "Scan QR code",
"message": "Scan, then continue.",
"qrDataUrl": "data:image/png;base64,AAAA",
"expiresInMs": 1,
"executor": "client"
}
}
"""
let decoded = try JSONDecoder().decode(SystemAgentChatResult.self, from: Data(json.utf8))
let step = try #require(decoded.step)
#expect(step.id == "setup-qr")
#expect(step.qrdataurl == "data:image/png;base64,AAAA")
#expect(step.expiresinms == 1)
let roundTripped = try JSONDecoder().decode(
SystemAgentChatResult.self,
from: JSONEncoder().encode(decoded))
#expect(roundTripped.step?.qrdataurl == step.qrdataurl)
}
private static func parse(_ questionJSON: String) throws -> SystemAgentChatQuestion? {
let result = try JSONDecoder().decode(
SystemAgentChatResult.self,
+9 -2
View File
@@ -97,8 +97,8 @@ const caps = [GATEWAY_CLIENT_CAPS.TOOL_EVENTS];
The current registry contains `approvals`, `exec-approvals`, `inline-widgets`,
`run-tool-bindings`, `session-scoped-events`, `plugin-approvals`,
`task-suggestions`, `terminal-offset-seq`, `tool-events`, and `ui-commands`.
Advertise only capabilities the client actually implements.
`system-agent-qr-code`, `task-suggestions`, `terminal-offset-seq`, `tool-events`,
and `ui-commands`. Advertise only capabilities the client actually implements.
<Warning>
`tool-events` gates live tool-execution streaming. The Gateway registers only
@@ -134,6 +134,13 @@ rejection, while text-only model runs can omit additional images after their
offload cap and still complete the request. The values are a connection-time
snapshot, so re-read them on every reconnect.
### Present system-agent QR codes
`GATEWAY_CLIENT_CAPS.SYSTEM_AGENT_QR_CODE` and the QR wizard-step shape are
reserved until system-agent QR production and Gateway projection are both
available. Clients should not advertise this capability yet; the contract alone
does not make existing Gateway methods emit QR steps.
## Recover state after reconnect
Treat every successful reconnect as a new projection over durable history and
@@ -84,6 +84,7 @@ export const GATEWAY_CLIENT_CAPS = {
RUN_TOOL_BINDINGS: "run-tool-bindings",
SESSION_SCOPED_EVENTS: "session-scoped-events",
PLUGIN_APPROVALS: "plugin-approvals",
SYSTEM_AGENT_QR_CODE: "system-agent-qr-code",
TASK_SUGGESTIONS: "task-suggestions",
TERMINAL_OFFSET_SEQ: "terminal-offset-seq",
TOOL_EVENTS: "tool-events",
@@ -12,6 +12,8 @@ import type { WizardStep } from "./schema/wizard.js";
describe("SystemAgentChatResultSchema", () => {
const validate = Compile(SystemAgentChatResultSchema);
const base = { sessionId: "chat-1", reply: "Bot token", action: "none" };
const qrDataUrl =
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=";
const steps: Array<{ name: string; step: WizardStep }> = [
{
@@ -115,6 +117,18 @@ describe("SystemAgentChatResultSchema", () => {
executor: "client",
},
},
{
name: "bounded QR image with a client acknowledgement",
step: {
id: "step-qr",
type: "qr",
title: "Scan QR code",
message: "Scan the code, then continue.",
qrDataUrl,
expiresInMs: 120_000,
executor: "client",
},
},
];
it.each(steps)("accepts a chat result carrying a $name step", ({ step }) => {
@@ -128,4 +142,79 @@ describe("SystemAgentChatResultSchema", () => {
it("rejects a step outside the wizard step contract", () => {
expect(validate.Check({ ...base, step: { id: "step-bogus", type: "freeform" } })).toBe(false);
});
it("rejects an incomplete or gateway-executed QR step", () => {
expect(validate.Check({ ...base, step: { id: "step-qr", type: "qr" } })).toBe(false);
expect(
validate.Check({
...base,
step: {
...steps.at(-1)?.step,
executor: "gateway",
},
}),
).toBe(false);
});
it("rejects malformed QR payloads and QR fields on other step types", () => {
const oversizedQrDataUrl = `data:image/png;base64,${"A".repeat(16_384)}`;
const invalidQrPngDataUrls = [
// Zero-width and zero-height IHDR chunks with matching CRCs.
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAAAAAABCAQAAABa3mc8AAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAACAQAAAB+QN+nAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
// The valid sample with its IDAT removed, and with a corrupted IHDR CRC.
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAAElFTkSuQmCC",
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwDAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
];
expect(
validate.Check({
...base,
step: { ...steps.at(-1)?.step, qrDataUrl: "data:image/png;base64,not-base64" },
}),
).toBe(false);
expect(
validate.Check({
...base,
step: { ...steps.at(-1)?.step, qrDataUrl: "data:image/png;base64,SGVsbG8=" },
}),
).toBe(false);
expect(
validate.Check({
...base,
step: { ...steps.at(-1)?.step, qrDataUrl: "data:image/png;base64,iVBORw0KGgp=" },
}),
).toBe(false);
expect(
validate.Check({
...base,
step: { ...steps.at(-1)?.step, qrDataUrl: qrDataUrl.slice(0, -1) },
}),
).toBe(false);
expect(
validate.Check({
...base,
step: { ...steps.at(-1)?.step, qrDataUrl: oversizedQrDataUrl },
}),
).toBe(false);
for (const invalidQrPngDataUrl of invalidQrPngDataUrls) {
expect(
validate.Check({
...base,
step: { ...steps.at(-1)?.step, qrDataUrl: invalidQrPngDataUrl },
}),
).toBe(false);
}
expect(
validate.Check({
...base,
step: {
id: "step-text",
type: "text",
executor: "client",
qrDataUrl,
expiresInMs: 120_000,
},
}),
).toBe(false);
});
});
@@ -1,5 +1,6 @@
/** Canonical owner-module barrel for gateway protocol schemas. */
export * from "./schema/primitives.js";
export * from "./schema/qr.js";
export * from "./schema/agent.js";
export * from "./schema/agents-models-skills.js";
export * from "./schema/agents-workspace.js";
+105
View File
@@ -0,0 +1,105 @@
// Gateway Protocol QR schemas share the established PNG data-URL contract.
import { Type } from "typebox";
export const QR_PNG_DATA_URL_MAX_LENGTH = 16_384;
export const QR_PNG_DATA_URL_PREFIX = "data:image/png;base64,";
// The first ten characters plus `o-r` encode the eight-byte PNG signature. If
// the payload ends there, only `o=` has canonical zero pad bits. Longer values
// complete that quartet before using the canonical padded Base64 tail grammar.
const QR_PNG_BASE64_SIGNATURE_PATTERN = "iVBORw0KGg";
const QR_PNG_BASE64_CANONICAL_TAIL_PATTERN =
"(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/][AQgw]==|[A-Za-z0-9+/]{2}[AEIMQUYcgkosw048]=)?";
const QR_PNG_DATA_URL_PATTERN = `^${QR_PNG_DATA_URL_PREFIX}${QR_PNG_BASE64_SIGNATURE_PATTERN}(?:o=|[o-r][A-Za-z0-9+/]${QR_PNG_BASE64_CANONICAL_TAIL_PATTERN})$`;
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a] as const;
const PNG_IHDR = 0x49484452;
const PNG_IDAT = 0x49444154;
const PNG_IEND = 0x49454e44;
function readUint32Be(bytes: Uint8Array, offset: number): number {
return (
(((bytes[offset] ?? 0) << 24) |
((bytes[offset + 1] ?? 0) << 16) |
((bytes[offset + 2] ?? 0) << 8) |
(bytes[offset + 3] ?? 0)) >>>
0
);
}
function pngCrc32(bytes: Uint8Array, start: number, end: number): number {
let crc = 0xffffffff;
for (let index = start; index < end; index += 1) {
crc ^= bytes[index] ?? 0;
for (let bit = 0; bit < 8; bit += 1) {
crc = crc & 1 ? 0xedb88320 ^ (crc >>> 1) : crc >>> 1;
}
}
return (crc ^ 0xffffffff) >>> 0;
}
function decodeQrPngDataUrl(value: string): Uint8Array | undefined {
if (!value.startsWith(QR_PNG_DATA_URL_PREFIX)) {
return undefined;
}
try {
const decoded = atob(value.slice(QR_PNG_DATA_URL_PREFIX.length));
return Uint8Array.from(decoded, (character) => character.charCodeAt(0));
} catch {
return undefined;
}
}
/** Validates the bounded PNG structure that QR-capable protocol clients consume. */
function isValidQrPngDataUrl(value: string): boolean {
const bytes = decodeQrPngDataUrl(value);
if (!bytes || !PNG_SIGNATURE.every((byte, index) => bytes[index] === byte)) {
return false;
}
let offset: number = PNG_SIGNATURE.length;
let sawIdat = false;
while (offset + 12 <= bytes.length) {
const length = readUint32Be(bytes, offset);
const typeOffset = offset + 4;
const dataOffset = typeOffset + 4;
const crcOffset = dataOffset + length;
const nextOffset = crcOffset + 4;
if (crcOffset > bytes.length - 4) {
return false;
}
const type = readUint32Be(bytes, typeOffset);
if (
pngCrc32(bytes, typeOffset, crcOffset) !== readUint32Be(bytes, crcOffset) ||
(offset === PNG_SIGNATURE.length &&
(type !== PNG_IHDR ||
length !== 13 ||
readUint32Be(bytes, dataOffset) === 0 ||
readUint32Be(bytes, dataOffset + 4) === 0))
) {
return false;
}
if (type === PNG_IHDR && offset !== PNG_SIGNATURE.length) {
return false;
}
if (type === PNG_IDAT) {
sawIdat = true;
}
if (type === PNG_IEND) {
return length === 0 && sawIdat && nextOffset === bytes.length;
}
offset = nextOffset;
}
return false;
}
export const QrPngDataUrlSchema = Type.Refine(
Type.String({
maxLength: QR_PNG_DATA_URL_MAX_LENGTH,
pattern: QR_PNG_DATA_URL_PATTERN,
}),
isValidQrPngDataUrl,
() => "Expected a structurally valid PNG QR data URL",
);
+62 -23
View File
@@ -3,6 +3,7 @@ import type { Static } from "typebox";
import { Type } from "typebox";
import { closedObject } from "./closed-object.js";
import { NonEmptyString } from "./primitives.js";
import { QrPngDataUrlSchema } from "./qr.js";
/** Runtime state reported for gateway-driven setup wizard sessions. */
const WizardRunStatusSchema = Type.Union([
@@ -61,28 +62,67 @@ const WizardDeviceCodeSchema = closedObject({
});
/** UI contract for one wizard step rendered by gateway clients. */
export const WizardStepSchema = closedObject({
id: NonEmptyString,
type: Type.Union([
Type.Literal("note"),
Type.Literal("select"),
Type.Literal("text"),
Type.Literal("confirm"),
Type.Literal("multiselect"),
Type.Literal("progress"),
Type.Literal("action"),
]),
title: Type.Optional(Type.String()),
message: Type.Optional(Type.String()),
format: Type.Optional(Type.Union([Type.Literal("plain")])),
options: Type.Optional(Type.Array(WizardStepOptionSchema)),
initialValue: Type.Optional(Type.Unknown()),
placeholder: Type.Optional(Type.String()),
sensitive: Type.Optional(Type.Boolean()),
executor: Type.Optional(Type.Union([Type.Literal("gateway"), Type.Literal("client")])),
externalUrl: Type.Optional(Type.String()),
deviceCode: Type.Optional(WizardDeviceCodeSchema),
});
const WizardStepObjectSchema = Type.Object(
{
id: NonEmptyString,
type: Type.Union([
Type.Literal("note"),
Type.Literal("select"),
Type.Literal("text"),
Type.Literal("confirm"),
Type.Literal("multiselect"),
Type.Literal("progress"),
Type.Literal("action"),
Type.Literal("qr"),
]),
title: Type.Optional(Type.String()),
message: Type.Optional(Type.String()),
format: Type.Optional(Type.Union([Type.Literal("plain")])),
options: Type.Optional(Type.Array(WizardStepOptionSchema)),
initialValue: Type.Optional(Type.Unknown()),
placeholder: Type.Optional(Type.String()),
sensitive: Type.Optional(Type.Boolean()),
executor: Type.Optional(Type.Union([Type.Literal("gateway"), Type.Literal("client")])),
externalUrl: Type.Optional(Type.String()),
deviceCode: Type.Optional(WizardDeviceCodeSchema),
/** PNG QR image rendered by clients that negotiated QR support. */
qrDataUrl: Type.Optional(QrPngDataUrlSchema),
/** Remaining lifetime when the Gateway emits this step. */
expiresInMs: Type.Optional(Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER })),
},
{
additionalProperties: false,
if: Type.Object({ type: Type.Literal("qr") }),
// oxlint-disable-next-line unicorn/no-thenable -- `then` is the JSON Schema conditional keyword.
then: Type.Object({
qrDataUrl: QrPngDataUrlSchema,
expiresInMs: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
executor: Type.Literal("client"),
}),
else: {
not: {
anyOf: [{ required: ["qrDataUrl"] }, { required: ["expiresInMs"] }],
},
},
},
);
type WizardStepWire = Static<typeof WizardStepObjectSchema>;
type WizardStepBase = Omit<WizardStepWire, "type" | "executor" | "qrDataUrl" | "expiresInMs">;
export type WizardStep = WizardStepBase &
(
| {
type: Exclude<WizardStepWire["type"], "qr">;
executor?: "gateway" | "client";
qrDataUrl?: never;
expiresInMs?: never;
}
| { type: "qr"; executor: "client"; qrDataUrl: string; expiresInMs: number }
);
// Preserve the object schema for native code generation while giving TypeBox's
// static type the same QR requirements enforced by the JSON Schema conditional.
export const WizardStepSchema = Type.Unsafe<WizardStep>(WizardStepObjectSchema);
/** Channel/account pair the channels flow actually configured. */
const WizardConfiguredAccountSchema = closedObject({
@@ -129,7 +169,6 @@ export type WizardAnswer = Static<typeof WizardAnswerSchema>;
export type WizardNextParams = Static<typeof WizardNextParamsSchema>;
export type WizardCancelParams = Static<typeof WizardCancelParamsSchema>;
export type WizardStatusParams = Static<typeof WizardStatusParamsSchema>;
export type WizardStep = Static<typeof WizardStepSchema>;
export type WizardNextResult = Static<typeof WizardNextResultSchema>;
export type WizardStartResult = Static<typeof WizardStartResultSchema>;
export type WizardStatusResult = Static<typeof WizardStatusResultSchema>;
+2 -2
View File
@@ -113,8 +113,8 @@ const ownerModules = [
...schemaModulesSource.matchAll(/^export \* from "\.\/schema\/([^"]+)\.js";$/gmu),
].map(([, moduleName = ""]) => moduleName);
check(
ownerModules.length === 55 && new Set(ownerModules).size === ownerModules.length,
"schema-modules.ts must contain one unique 55-module owner list",
ownerModules.length === 56 && new Set(ownerModules).size === ownerModules.length,
"schema-modules.ts must contain one unique 56-module owner list",
);
check(
schemaModulesSource.split("\n").filter(Boolean).length === ownerModules.length,
+25 -2
View File
@@ -9,7 +9,9 @@ const { MOCK_PNG_BASE64, MOCK_PNG_BUFFER, toBuffer } = vi.hoisted(() => {
return {
MOCK_PNG_BASE64: MOCK_PNG_BUFFERLocal.toString("base64"),
MOCK_PNG_BUFFER: MOCK_PNG_BUFFERLocal,
toBuffer: vi.fn(async () => MOCK_PNG_BUFFERLocal),
toBuffer: vi.fn(
async (_input: string, _opts: { margin: number; scale: number }) => MOCK_PNG_BUFFERLocal,
),
};
});
@@ -26,11 +28,13 @@ vi.mock("qrcode", async (importOriginal) => {
let renderQrPngBase64: typeof import("./qr-image.ts").renderQrPngBase64;
let renderQrPngDataUrl: typeof import("./qr-image.ts").renderQrPngDataUrl;
let renderQrPngDataUrlWithinLimit: typeof import("./qr-image.ts").renderQrPngDataUrlWithinLimit;
let writeQrPngTempFile: typeof import("./qr-image.ts").writeQrPngTempFile;
beforeAll(async () => {
vi.resetModules();
({ renderQrPngBase64, renderQrPngDataUrl, writeQrPngTempFile } = await import("./qr-image.ts"));
({ renderQrPngBase64, renderQrPngDataUrl, renderQrPngDataUrlWithinLimit, writeQrPngTempFile } =
await import("./qr-image.ts"));
});
describe("renderQrPngBase64", () => {
@@ -89,6 +93,25 @@ describe("renderQrPngBase64", () => {
);
});
it("reduces the scale until the QR data URL fits its presentation limit", async () => {
toBuffer
.mockResolvedValueOnce(Buffer.alloc(30))
.mockResolvedValueOnce(Buffer.alloc(20))
.mockResolvedValueOnce(Buffer.alloc(10));
const result = await renderQrPngDataUrlWithinLimit("openclaw", 40);
expect(result.length).toBeLessThanOrEqual(40);
});
it("rejects QR data URLs that exceed the limit at minimum scale", async () => {
toBuffer.mockResolvedValue(Buffer.alloc(30));
await expect(renderQrPngDataUrlWithinLimit("openclaw", 40)).rejects.toThrow(
"exceeds the presentation limit",
);
});
it("writes QR PNGs to a scoped temp file", async () => {
await fs.mkdir(tmpRoot, { recursive: true });
+28
View File
@@ -96,6 +96,34 @@ export async function renderQrPngDataUrl(
return formatQrPngDataUrl(await renderQrPngBase64(input, opts));
}
/** Renders the highest-scale QR PNG data URL that fits a presentation limit. */
export async function renderQrPngDataUrlWithinLimit(
input: string,
maxDataUrlLength: number,
opts: QrPngRenderOptions = {},
): Promise<string> {
if (
!Number.isSafeInteger(maxDataUrlLength) ||
maxDataUrlLength <= QR_PNG_DATA_URL_PREFIX.length
) {
throw new RangeError("maxDataUrlLength must be a safe positive data URL length.");
}
const initialScale = resolveQrPngIntegerOption({
name: "scale",
value: opts.scale,
defaultValue: DEFAULT_QR_PNG_SCALE,
min: MIN_QR_PNG_SCALE,
max: MAX_QR_PNG_SCALE,
});
for (let scale = initialScale; scale >= MIN_QR_PNG_SCALE; scale -= 1) {
const dataUrl = await renderQrPngDataUrl(input, { ...opts, scale });
if (dataUrl.length <= maxDataUrlLength) {
return dataUrl;
}
}
throw new RangeError("QR PNG data URL exceeds the presentation limit at minimum scale.");
}
/** Writes QR PNG output into a scoped temp directory and returns that directory as a media root. */
export async function writeQrPngTempFile(
input: string,
+5 -2
View File
@@ -7,6 +7,8 @@ import { WizardCancelledError, type WizardProgress, type WizardPrompter } from "
// WizardSession exposes interactive setup as a step/answer protocol for remote
// clients while reusing the same WizardPrompter contract as the local CLI.
export type WizardStep = ProtocolWizardStep;
type WizardNonQrStep = Exclude<WizardStep, { type: "qr" }>;
type WizardNonQrStepInput = Omit<WizardNonQrStep, "id">;
type WizardStepInputRequirement = "always" | "never" | "client-executor";
@@ -18,6 +20,7 @@ const WIZARD_STEP_INPUT_REQUIREMENT_BY_TYPE = {
multiselect: "always",
progress: "never",
action: "client-executor",
qr: "client-executor",
} as const satisfies Record<WizardStep["type"], WizardStepInputRequirement>;
/** Whether a step needs a user answer instead of client or gateway acknowledgement. */
@@ -237,11 +240,11 @@ class WizardSessionPrompter implements WizardPrompter {
this.session.queueExternalUrl(url);
}
private async prompt(step: Omit<WizardStep, "id">): Promise<unknown> {
private async prompt(step: WizardNonQrStepInput): Promise<unknown> {
return await this.session.awaitAnswer(this.createStep(step));
}
private createStep(step: Omit<WizardStep, "id">): WizardStep {
private createStep(step: WizardNonQrStepInput): WizardNonQrStep {
// Each emitted step receives an id so remote clients can answer the exact
// pending prompt and stale answers can be rejected. Explicit browser
// destinations bind to the very next step regardless of its input type.
@@ -326,6 +326,10 @@ export function renderWizardStepControls(
return props.step.executor === "gateway"
? renderProgressStep(props)
: renderContinueStep(props);
// QR steps need a client-owned presentation; clients without one must not
// turn them into an answerable generic step.
case "qr":
return nothing;
// These show whatever the step supplies behind a single Continue.
case "note":
case "action":
@@ -8,8 +8,24 @@ const options = [
{ label: "Twitch", value: "twitch" },
];
function step(patch: Partial<WizardStep>): WizardStep {
return { id: "step", type: "select", options, ...patch };
type TestStepPatch =
| { type?: "select"; initialValue?: unknown }
| { type: "multiselect"; initialValue?: unknown }
| { type: "confirm"; initialValue?: unknown }
| { type: "text"; initialValue?: unknown }
| { type: "action"; initialValue?: unknown };
function step(patch: TestStepPatch): WizardStep {
switch (patch.type) {
case "multiselect":
return { id: "step", options, ...patch };
case "confirm":
case "text":
case "action":
return { id: "step", ...patch };
default:
return { id: "step", type: "select", options, ...patch };
}
}
describe("Custodian rich wizard answers", () => {