diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b11b9c830c4..31301292c8e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -406,6 +406,7 @@ Docs: https://docs.openclaw.ai - **Media-store remote downloads:** bound response-header waits and stalled bodies, close abandoned redirect and error responses, and remove partial temp files so hung sources cannot pin callers. (#104624) Thanks @hugenshen. - **Cron llama.cpp tool schemas:** keep the model-facing cron declaration schema compatible with llama.cpp while retaining gateway and runtime nonblank validation. Fixes #107449. (#108360) Thanks @lee-xydt. - **System-agent recovery guidance:** direct browser and app users to Settings or the OpenClaw host instead of terminal-only exit guidance while preserving the required stop, onboard, and restart lifecycle. (#114633) Thanks @jesse-merhi. +- **Browser keyboard aliases:** accept `Esc`, `Return`, `Del`, `Ctrl`, and `Cmd` in browser actions and shortcuts, and preserve keyboard guidance in compact tool schemas. (#130401) Thanks @geekforlife. ## 2026.7.1 diff --git a/docs/cli/browser.md b/docs/cli/browser.md index fa303de3e53b..dd67897fc8ed 100644 --- a/docs/cli/browser.md +++ b/docs/cli/browser.md @@ -254,6 +254,8 @@ openclaw browser evaluate --fn 'const title = document.title; return title;' openclaw browser evaluate --timeout-ms 30000 --fn 'async () => { await window.ready; return true; }' ``` +`press` accepts named keys and shortcuts such as `Escape`, `Control+Shift+T`, and `Control++`; common `Esc`, `Return`, `Del`, `Ctrl`, and `Cmd` aliases are normalized. + For managed browser profiles, `select` preserves option values exactly. Quote empty or whitespace-sensitive values, such as `openclaw browser select ""` or `openclaw browser select " padded "`. `evaluate --fn` accepts a function source, an expression, or a statement body. Statement bodies are wrapped as async functions, so use `return` for the value you want back. Use `--timeout-ms` when the page-side function may need longer than the default evaluate timeout. `browser.evaluateEnabled=false` (default: `true`) disables both `evaluate` and `wait --fn`. diff --git a/extensions/browser/src/browser-tool.schema.test.ts b/extensions/browser/src/browser-tool.schema.test.ts index 181fcbb078d1..daddcafcb3c2 100644 --- a/extensions/browser/src/browser-tool.schema.test.ts +++ b/extensions/browser/src/browser-tool.schema.test.ts @@ -1,5 +1,7 @@ // Browser tests cover browser tool.schema plugin behavior. import { expectDefined } from "@openclaw/normalization-core"; +import { projectRuntimeToolInputSchema } from "openclaw/plugin-sdk/agent-harness-runtime"; +import { normalizeOpenAIToolSchemas } from "openclaw/plugin-sdk/provider-tools"; import { describe, expect, it } from "vitest"; import { createBrowserToolSchema, resolveBrowserToolCapabilities } from "./browser-tool.schema.js"; import { ACT_MAX_VIEWPORT_DIMENSION } from "./browser/act-policy.js"; @@ -54,6 +56,54 @@ describe("browser tool schema", () => { expect(requestTargetId.description).toBe(targetId.description); }); + it("describes canonical keyboard keys and aliases on nested and flattened act params", () => { + const properties = BrowserToolSchema.properties as BrowserSchemaRecord; + const requestProperties = requireSchemaProperty(properties, "request", "browser request schema") + .properties as BrowserSchemaRecord; + const key = requireSchemaProperty(properties, "key", "browser key schema"); + const requestKey = requireSchemaProperty( + requestProperties, + "key", + "browser request key schema", + ); + + expect(key.description).toContain("Escape"); + expect(key.description).toContain("aliases Esc, Return, Del, Ctrl, Cmd"); + expect(key.description).toContain("Control+Shift+T"); + expect(requestKey.description).toBe(key.description); + }); + + it.each([false, true])( + "preserves key guidance within the Codex schema budget (bound=%s)", + (tabBound) => { + const schema = createBrowserToolSchema(resolveBrowserToolCapabilities({ tabBound })); + const properties = schema.properties as BrowserSchemaRecord; + const key = requireSchemaProperty(properties, "key", "browser key schema"); + const normalized = normalizeOpenAIToolSchemas({ + provider: "openai", + modelApi: "openai-chatgpt-responses", + tools: [ + { + name: "browser", + label: "Browser", + description: "Browser", + parameters: schema, + execute: async () => ({ content: [], details: {} }), + }, + ], + }); + const projection = projectRuntimeToolInputSchema(normalized[0]?.parameters); + expect(projection.violations).toEqual([]); + // Codex strips parameter descriptions above 5,000 bytes after schema normalization. + expect(Buffer.byteLength(JSON.stringify(projection.schema))).toBeLessThanOrEqual(5_000); + expect(projection.schema).toHaveProperty("properties.key.description", key.description); + expect(projection.schema).toHaveProperty( + "properties.request.properties.key.description", + key.description, + ); + }, + ); + it("exposes explicit download actions and their output path", () => { const properties = BrowserToolSchema.properties as BrowserSchemaRecord; diff --git a/extensions/browser/src/browser-tool.schema.ts b/extensions/browser/src/browser-tool.schema.ts index e1d20574e4ad..3b28452c7bc1 100644 --- a/extensions/browser/src/browser-tool.schema.ts +++ b/extensions/browser/src/browser-tool.schema.ts @@ -65,7 +65,7 @@ const BROWSER_SNAPSHOT_REFS = ["role", "aria"] as const; const BROWSER_IMAGE_TYPES = ["png", "jpeg"] as const; const TAB_REFERENCE_DESCRIPTION = - "Tab reference. Prefer suggestedTargetId, tabId, or label from tabs output; raw CDP targetId and unique raw prefixes remain supported for compatibility."; + "Prefer suggestedTargetId/tabId/label; raw CDP targetId or unique prefix works."; // NOTE: Using a flattened object schema instead of Type.Union([Type.Object(...), ...]) // because Claude API on Vertex AI rejects nested anyOf schemas as invalid JSON Schema. @@ -136,7 +136,11 @@ function createBrowserActProperties(capabilities: BrowserToolCapabilities) { submit: Type.Optional(Type.Boolean()), slowly: Type.Optional(Type.Boolean()), // press - key: Type.Optional(Type.String()), + key: Type.Optional( + Type.String({ + description: "Escape, Enter, Control+Shift+T; aliases Esc, Return, Del, Ctrl, Cmd.", + }), + ), delayMs: optionalNonNegativeIntegerSchema(), // drag startRef: Type.Optional(Type.String()), diff --git a/extensions/browser/src/browser/routes/agent.act.existing-session-navigation-guard.test.ts b/extensions/browser/src/browser/routes/agent.act.existing-session-navigation-guard.test.ts index cb56e75bfbde..bef5f6a6b5b3 100644 --- a/extensions/browser/src/browser/routes/agent.act.existing-session-navigation-guard.test.ts +++ b/extensions/browser/src/browser/routes/agent.act.existing-session-navigation-guard.test.ts @@ -593,6 +593,15 @@ describe("existing-session interaction navigation guard", () => { expect(navigationGuardMocks.assertBrowserNavigationResultAllowed).not.toHaveBeenCalled(); }); + it("normalizes keyboard aliases before existing-session Chrome MCP dispatch", async () => { + const response = await runAction({ kind: "press", key: "Ctrl+Shift+Esc" }, null); + + expect(response.statusCode).toBe(200); + expect(chromeMcpMocks.pressChromeMcpKey).toHaveBeenCalledWith( + expect.objectContaining({ key: "Control+Shift+Escape" }), + ); + }); + it("still probes navigation when the interaction command throws", async () => { chromeMcpMocks.clickChromeMcpElement.mockImplementationOnce(() => { throw new Error("stale element"); diff --git a/extensions/browser/src/browser/routes/agent.act.normalize.test.ts b/extensions/browser/src/browser/routes/agent.act.normalize.test.ts index bec711b23d05..5c7e5344c9bd 100644 --- a/extensions/browser/src/browser/routes/agent.act.normalize.test.ts +++ b/extensions/browser/src/browser/routes/agent.act.normalize.test.ts @@ -65,6 +65,40 @@ describe("canonicalizeActTargetIds", () => { }); }); +describe("normalizeActRequest keyboard keys", () => { + it.each([ + ["Esc", "Escape"], + ["ESC", "Escape"], + ["Return", "Enter"], + ["Del", "Delete"], + ["Ctrl+a", "Control+a"], + ["Cmd+A", "Meta+A"], + ["Ctrl+Shift+Esc", "Control+Shift+Escape"], + ["Ctrl++", "Control++"], + ])("normalizes the keyboard alias %s", (key, expected) => { + expect(normalizeActRequest({ kind: "press", key })).toMatchObject({ key: expected }); + }); + + it.each(["a", "A", "+", "Control++", "ControlOrMeta+a", "__proto__", "constructor"])( + "preserves the existing keyboard contract for %s", + (key) => { + expect(normalizeActRequest({ kind: "press", key })).toMatchObject({ key }); + }, + ); + + it("normalizes keyboard aliases inside nested batch actions", () => { + expect( + normalizeActRequest({ + kind: "batch", + actions: [{ kind: "batch", actions: [{ kind: "press", key: "Ctrl+Esc" }] }], + }), + ).toMatchObject({ + kind: "batch", + actions: [{ kind: "batch", actions: [{ kind: "press", key: "Control+Escape" }] }], + }); + }); +}); + describe("normalizeActRequest numeric fields", () => { it("keeps structured numeric action options", () => { expect( diff --git a/extensions/browser/src/browser/routes/agent.act.normalize.ts b/extensions/browser/src/browser/routes/agent.act.normalize.ts index ba9778a25f56..41b0325da9c9 100644 --- a/extensions/browser/src/browser/routes/agent.act.normalize.ts +++ b/extensions/browser/src/browser/routes/agent.act.normalize.ts @@ -25,6 +25,14 @@ import { } from "./route-numeric.js"; import { toBoolean, toStringArray, toStringOrEmpty } from "./utils.js"; +const KEY_ALIASES = new Map([ + ["esc", "Escape"], + ["return", "Enter"], + ["del", "Delete"], + ["ctrl", "Control"], + ["cmd", "Meta"], +]); + function countBatchActions(actions: BrowserActRequest[]): number { let count = 0; for (const action of actions) { @@ -224,7 +232,11 @@ export function normalizeActRequest( }; } case "press": { - const key = toStringOrEmpty(body.key); + // Empty chord segments represent a literal plus key and must survive normalization. + const key = toStringOrEmpty(body.key) + .split("+") + .map((part) => KEY_ALIASES.get(part.toLowerCase()) ?? part) + .join("+"); if (!key) { throw new Error("press requires key"); } diff --git a/extensions/browser/src/browser/server.agent-contract-core.test.ts b/extensions/browser/src/browser/server.agent-contract-core.test.ts index b0404cc1e171..e63c3dd2e9ec 100644 --- a/extensions/browser/src/browser/server.agent-contract-core.test.ts +++ b/extensions/browser/src/browser/server.agent-contract-core.test.ts @@ -841,21 +841,20 @@ describe("browser control server", () => { expect((typeArgs as { submit?: boolean }).submit).toBeUndefined(); expect((typeArgs as { slowly?: boolean }).slowly).toBeUndefined(); - const press = await postJson<{ ok: boolean }>(`${base}/act`, { - kind: "press", - key: "Enter", - }); - expect(press.ok).toBe(true); - const pressArgs = mockFirstArg(requirePwMock("pressKeyViaPlaywright"), 0, "press"); - expectRecordFields(pressArgs, { - cdpUrl: state.cdpBaseUrl, - targetId: "abcd1234", - key: "Enter", - ssrfPolicy: { - dangerouslyAllowPrivateNetwork: true, - }, - }); - expect((pressArgs as { delayMs?: number }).delayMs).toBeUndefined(); + for (const [index, key] of ["Enter", "Ctrl+Shift+Esc"].entries()) { + const press = await postJson<{ ok: boolean }>(`${base}/act`, { kind: "press", key }); + expect(press.ok).toBe(true); + const pressArgs = mockFirstArg(requirePwMock("pressKeyViaPlaywright"), index, "press"); + expectRecordFields(pressArgs, { + cdpUrl: state.cdpBaseUrl, + targetId: "abcd1234", + key: key === "Enter" ? "Enter" : "Control+Shift+Escape", + ssrfPolicy: { + dangerouslyAllowPrivateNetwork: true, + }, + }); + expect((pressArgs as { delayMs?: number }).delayMs).toBeUndefined(); + } const hover = await postJson<{ ok: boolean }>(`${base}/act`, { kind: "hover",