mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 21:07:01 -06:00
feat(mantis): let proof agents extend desktop control (#127271)
* feat(mantis): let proof agents extend desktop control * fix(mantis): constrain desktop extensions * docs(mantis): document desktop action authority * fix(mantis): bind actions to recorded window
This commit is contained in:
Ayaan Zaidi
@@ -63,15 +63,22 @@ Use `$OPENCLAW_TELEGRAM_MANTIS_LANE_CMD` with `--lane baseline|candidate`:
|
||||
- `requests` (redacted provider requests; zero is a valid recorded fact)
|
||||
- `press --message-id ID --button INDEX`
|
||||
- `delete --message-id ID` (only user messages sent in this session)
|
||||
- `desktop --actions-file <public-json> [--timeout-seconds N]` (run an
|
||||
agent-authored click/key/type/sleep action sequence in the recorded desktop)
|
||||
- `view --message-id ID` (scroll Desktop to the exact Telegram server message)
|
||||
- `screenshot` (returns a public inspection PNG)
|
||||
- `finish [--focus-message-id ID]` (focus the named message or the latest sent message, stop, capture, publish facts)
|
||||
- `block --reason TEXT [--missing-primitive NAME]` (clean stop-report)
|
||||
- `abort` (cleanup after scenario failure)
|
||||
|
||||
`start` returns the exact command/budget list. No generic exec/eval or raw
|
||||
Telegram API exists. If the comparison cannot prove the PR's visible behavior,
|
||||
use `block` and say why.
|
||||
`start` returns the exact command/budget list. When the listed primitives cannot
|
||||
exercise the behavior, extend the harness: write a focused JSON action sequence
|
||||
under `MANTIS_OUTPUT_DIR` and run it with `desktop`. Actions use Telegram-window
|
||||
coordinates: `{"command":"click","x":N,"y":N,"button":1}`,
|
||||
`{"command":"key","keys":["ctrl+a"]}`, `{"command":"type","text":"..."}`,
|
||||
or `{"command":"sleep","milliseconds":N}`. Inspect a screenshot, adjust the
|
||||
sequence, and continue the proof. Use `block` only when the ephemeral desktop
|
||||
itself cannot exercise the behavior.
|
||||
Raw response events must form a complete provider response; deltas alone do not
|
||||
produce a final answer. Copy the terminal item and completed-response structure
|
||||
from `responseEvents` in `scripts/e2e/mock-openai-server.mjs`, and use
|
||||
|
||||
@@ -202,10 +202,13 @@ in.
|
||||
### Telegram Desktop recorder
|
||||
|
||||
The Telegram Desktop recorder is a standalone operator utility, invoked
|
||||
directly through `pnpm qa:telegram-desktop-recorder`. It records native
|
||||
Telegram Desktop and nothing else: it never drives OpenClaw or sends Telegram
|
||||
messages. Whoever runs it owns the turn — start the SUT, send through a real
|
||||
Telegram user, then tell the recorder which message to show — and supplies
|
||||
directly through `pnpm qa:telegram-desktop-recorder`. It never drives OpenClaw.
|
||||
Its normal recording commands do not send Telegram messages. The optional
|
||||
`actions` command drives only the measured Telegram window through bounded
|
||||
`click`, `key`, `type`, and `sleep` actions; those actions can send as the
|
||||
signed-in Telegram user. Whoever runs it owns the turn and those side effects —
|
||||
start the SUT, send through a real Telegram user, then tell the recorder which
|
||||
message to show — and supplies
|
||||
`--user-driver`, the command the recorder shells out to for the TDLib calls it
|
||||
cannot make itself (`confirm-qr`, `terminate-session`). Any driver exposing
|
||||
those two verbs works, including this repo's
|
||||
|
||||
@@ -27,6 +27,7 @@ const recorderSessionBaseSchema = z.object({
|
||||
/** Telegram Desktop window as placed on the recorded desktop; the crop uses it. */
|
||||
window: z.object({
|
||||
height: z.number().int().positive(),
|
||||
id: z.string().regex(/^0x[0-9a-f]+$/iu),
|
||||
width: z.number().int().positive(),
|
||||
x: z.number().int().nonnegative(),
|
||||
y: z.number().int().nonnegative(),
|
||||
@@ -70,6 +71,13 @@ export type ViewOptions = {
|
||||
sessionPath: string;
|
||||
};
|
||||
|
||||
export type ActionsOptions = {
|
||||
actionsFile: string;
|
||||
command: "actions";
|
||||
sessionPath: string;
|
||||
timeoutSeconds: number;
|
||||
};
|
||||
|
||||
export type ScreenshotOptions = {
|
||||
command: "screenshot";
|
||||
output?: string;
|
||||
@@ -100,6 +108,7 @@ export type ArtifactsOptions = {
|
||||
|
||||
type RecorderOptions =
|
||||
| ArtifactsOptions
|
||||
| ActionsOptions
|
||||
| RecoverOptions
|
||||
| ScreenshotOptions
|
||||
| StartOptions
|
||||
@@ -113,6 +122,7 @@ export function recorderUsageText(): string {
|
||||
" pnpm qa:telegram-desktop-recorder artifacts --session <recorder.json>",
|
||||
' pnpm qa:telegram-desktop-recorder start --output-dir <dir> --chat <-100groupId> --user-driver "<space-separated cmd prefix>" [options]',
|
||||
" pnpm qa:telegram-desktop-recorder view --session <recorder.json> --message-id <id>",
|
||||
" pnpm qa:telegram-desktop-recorder actions --session <recorder.json> --actions-file <json> [--timeout-seconds <seconds>]",
|
||||
" pnpm qa:telegram-desktop-recorder screenshot --session <recorder.json> [--output <png>]",
|
||||
" pnpm qa:telegram-desktop-recorder recover --session <recorder.json>",
|
||||
" pnpm qa:telegram-desktop-recorder stop --session <recorder.json> [--crop telegram-window] [--keep-box]",
|
||||
@@ -163,7 +173,7 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
|
||||
throw new Error(recorderUsageText());
|
||||
}
|
||||
const parsedCommand = z
|
||||
.enum(["artifacts", "recover", "screenshot", "start", "status", "stop", "view"])
|
||||
.enum(["actions", "artifacts", "recover", "screenshot", "start", "status", "stop", "view"])
|
||||
.safeParse(rawCommand);
|
||||
if (!parsedCommand.success) {
|
||||
throw new Error(`Unknown command: ${rawCommand}\n\n${recorderUsageText()}`);
|
||||
@@ -205,11 +215,13 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
|
||||
])
|
||||
: command === "view"
|
||||
? new Set(["--message-id", "--session"])
|
||||
: command === "screenshot"
|
||||
? new Set(["--output", "--session"])
|
||||
: command === "stop"
|
||||
? new Set(["--crop", "--session"])
|
||||
: new Set(["--session"]);
|
||||
: command === "actions"
|
||||
? new Set(["--actions-file", "--session", "--timeout-seconds"])
|
||||
: command === "screenshot"
|
||||
? new Set(["--output", "--session"])
|
||||
: command === "stop"
|
||||
? new Set(["--crop", "--session"])
|
||||
: new Set(["--session"]);
|
||||
for (const flag of values.keys()) {
|
||||
if (!allowed.has(flag)) {
|
||||
throw new Error(`${flag} is not available for ${command}.`);
|
||||
@@ -266,6 +278,14 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
|
||||
positiveInteger(messageId, "--message-id");
|
||||
return { command, messageId, sessionPath };
|
||||
}
|
||||
if (command === "actions") {
|
||||
return {
|
||||
actionsFile: requiredString(values, "--actions-file"),
|
||||
command,
|
||||
sessionPath,
|
||||
timeoutSeconds: positiveInteger(values.get("--timeout-seconds") ?? "60", "--timeout-seconds"),
|
||||
};
|
||||
}
|
||||
if (command === "screenshot") {
|
||||
return { command, output: values.get("--output"), sessionPath };
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
import {
|
||||
parseRecorderArgs,
|
||||
readRecorderSession,
|
||||
type ActionsOptions,
|
||||
type ArtifactsOptions,
|
||||
type RecoverOptions,
|
||||
recorderUsageText,
|
||||
@@ -174,14 +175,13 @@ export DISPLAY=:99
|
||||
win="$(wmctrl -lx | awk 'tolower($0) ~ /telegramdesktop/ {print $1; exit}')"
|
||||
test -n "$win"
|
||||
eval "$(xdotool getwindowgeometry --shell "$win")"
|
||||
printf '%s %s %s %s\n' "$X" "$Y" "$WIDTH" "$HEIGHT"`;
|
||||
printf '%s %s %s %s %s\n' "$win" "$X" "$Y" "$WIDTH" "$HEIGHT"`;
|
||||
}
|
||||
|
||||
function renderHideTelegramWindow(): string {
|
||||
function renderHideTelegramWindow(windowId: string): string {
|
||||
return `set -euo pipefail
|
||||
export DISPLAY=:99
|
||||
win="$(wmctrl -lx | awk 'tolower($0) ~ /telegramdesktop/ {print $1; exit}')"
|
||||
test -n "$win"
|
||||
win=${shellQuote(windowId)}
|
||||
xdotool windowminimize "$win"
|
||||
sleep 0.2`;
|
||||
}
|
||||
@@ -236,19 +236,26 @@ exit 1`;
|
||||
|
||||
export function parseWindowGeometry(raw: string): {
|
||||
height: number;
|
||||
id: string;
|
||||
width: number;
|
||||
x: number;
|
||||
y: number;
|
||||
} {
|
||||
const parts = raw.trim().split(/\s+/u).map(Number);
|
||||
if (parts.length !== 4 || parts.some((value) => !Number.isFinite(value) || value < 0)) {
|
||||
const [id, ...rawGeometry] = raw.trim().split(/\s+/u);
|
||||
const parts = rawGeometry.map(Number);
|
||||
if (
|
||||
!id ||
|
||||
!/^0x[0-9a-f]+$/iu.test(id) ||
|
||||
parts.length !== 4 ||
|
||||
parts.some((value) => !Number.isFinite(value) || value < 0)
|
||||
) {
|
||||
throw new Error(`Telegram Desktop window geometry was not readable: ${raw.trim()}`);
|
||||
}
|
||||
const [x, y, width, height] = parts as [number, number, number, number];
|
||||
if (width < 200 || height < 200) {
|
||||
throw new Error(`Telegram Desktop window is too small to crop: ${width}x${height}`);
|
||||
}
|
||||
return { height, width, x, y };
|
||||
return { height, id, width, x, y };
|
||||
}
|
||||
|
||||
function driverCommand(userDriver: string[], args: string[]) {
|
||||
@@ -611,7 +618,7 @@ async function startRecorderAttempt(
|
||||
// The lane clears prior history before recorder start. Keep the empty chat hidden until
|
||||
// the first session-owned send is ready, so setup frames reveal neither account UI nor chat.
|
||||
await operations.sshRun({
|
||||
command: renderHideTelegramWindow(),
|
||||
command: renderHideTelegramWindow(windowGeometry.id),
|
||||
cwd,
|
||||
inspect,
|
||||
run: operations.runCommand,
|
||||
@@ -815,6 +822,93 @@ export async function viewRecorder(
|
||||
});
|
||||
}
|
||||
|
||||
const desktopActionsSchema = z
|
||||
.array(
|
||||
z.discriminatedUnion("command", [
|
||||
z.object({
|
||||
button: z.number().int().min(1).max(5).default(1),
|
||||
command: z.literal("click"),
|
||||
x: z.number().int().nonnegative(),
|
||||
y: z.number().int().nonnegative(),
|
||||
}),
|
||||
z.object({
|
||||
command: z.literal("key"),
|
||||
keys: z
|
||||
.array(z.string().regex(/^[A-Za-z0-9_+:-]+$/u))
|
||||
.min(1)
|
||||
.max(20),
|
||||
}),
|
||||
z.object({ command: z.literal("sleep"), milliseconds: z.number().int().min(1).max(30_000) }),
|
||||
z.object({
|
||||
command: z.literal("type"),
|
||||
delayMs: z.number().int().min(0).max(1_000).default(5),
|
||||
text: z.string().min(1).max(10_000),
|
||||
}),
|
||||
]),
|
||||
)
|
||||
.min(1)
|
||||
.max(100);
|
||||
|
||||
export async function runRecorderActions(
|
||||
cwd: string,
|
||||
opts: ActionsOptions,
|
||||
operations: RecorderOperations = defaultOperations,
|
||||
): Promise<{ results: Array<{ command: string; stderr: string; stdout: string }> }> {
|
||||
const sessionPath = resolveRecorderPath(cwd, opts.sessionPath, "--session");
|
||||
const actionsPath = resolveRecorderPath(cwd, opts.actionsFile, "--actions-file");
|
||||
const actionsStat = fs.lstatSync(actionsPath);
|
||||
if (!actionsStat.isFile() || actionsStat.isSymbolicLink() || actionsStat.size > 64 * 1024) {
|
||||
throw new Error("--actions-file must be a regular file no larger than 64 KiB.");
|
||||
}
|
||||
const actions = desktopActionsSchema.parse(JSON.parse(fs.readFileSync(actionsPath, "utf8")));
|
||||
const session = readRecorderSession(sessionPath);
|
||||
for (const action of actions) {
|
||||
if (
|
||||
action.command === "click" &&
|
||||
(action.x >= session.window.width || action.y >= session.window.height)
|
||||
) {
|
||||
throw new Error("click coordinates must stay inside the Telegram window.");
|
||||
}
|
||||
}
|
||||
const crabboxBin = process.env.OPENCLAW_TELEGRAM_USER_CRABBOX_BIN?.trim() || "crabbox";
|
||||
const inspect = await sessionInspect({ crabboxBin, cwd, operations, session });
|
||||
const results: Array<{ command: string; stderr: string; stdout: string }> = [];
|
||||
for (const action of actions) {
|
||||
if (action.command === "sleep") {
|
||||
await sleep(action.milliseconds);
|
||||
results.push({ command: "sleep", stderr: "", stdout: "" });
|
||||
continue;
|
||||
}
|
||||
const telegramWindow = `win=${shellQuote(session.window.id)}
|
||||
if ! wmctrl -lx | awk -v win="$win" 'tolower($1) == tolower(win) && tolower($0) ~ /telegramdesktop/ {found=1} END {exit !found}'; then
|
||||
echo "Recorded Telegram window $win no longer exists." >&2
|
||||
exit 1
|
||||
fi
|
||||
eval "$(xdotool getwindowgeometry --shell "$win")"
|
||||
if [ "$X" -ne ${session.window.x} ] || [ "$Y" -ne ${session.window.y} ] || [ "$WIDTH" -ne ${session.window.width} ] || [ "$HEIGHT" -ne ${session.window.height} ]; then
|
||||
echo "Recorded Telegram window $win moved or resized." >&2
|
||||
exit 1
|
||||
fi
|
||||
`;
|
||||
const actionCommand =
|
||||
action.command === "click"
|
||||
? `xdotool windowactivate --sync "$win" mousemove --window "$win" ${action.x} ${action.y} click ${action.button}`
|
||||
: action.command === "key"
|
||||
? `xdotool key --window "$win" ${action.keys.map(shellQuote).join(" ")}`
|
||||
: `xdotool type --window "$win" --delay ${action.delayMs} -- ${shellQuote(action.text)}`;
|
||||
const result = await operations.sshRun({
|
||||
command: `export DISPLAY=:99\n${telegramWindow}${actionCommand}`,
|
||||
cwd,
|
||||
inspect,
|
||||
run: operations.runCommand,
|
||||
stdio: "pipe",
|
||||
timeoutMs: opts.timeoutSeconds * 1000,
|
||||
});
|
||||
results.push({ command: action.command, ...result });
|
||||
}
|
||||
return { results };
|
||||
}
|
||||
|
||||
async function captureScreenshot(params: {
|
||||
crop: ReturnType<typeof proofViewport>;
|
||||
cwd: string;
|
||||
@@ -1067,6 +1161,10 @@ async function main(): Promise<void> {
|
||||
console.log(`Telegram Desktop opened message ${opts.messageId}.`);
|
||||
return;
|
||||
}
|
||||
if (opts.command === "actions") {
|
||||
console.log(JSON.stringify(await runRecorderActions(cwd, opts), null, 2));
|
||||
return;
|
||||
}
|
||||
if (opts.command === "screenshot") {
|
||||
console.log(await screenshotRecorder(cwd, opts));
|
||||
return;
|
||||
|
||||
@@ -112,6 +112,7 @@ const commandOptions: Record<string, readonly string[]> = {
|
||||
abort: ["--lane"],
|
||||
block: ["--lane", "--missing-primitive", "--reason"],
|
||||
delete: ["--lane", "--message-id"],
|
||||
desktop: ["--lane", "--actions-file", "--timeout-seconds"],
|
||||
finish: ["--lane", "--focus-message-id"],
|
||||
mock: ["--lane", "--response-file", "--response-events-file", "--chunk-delay-ms"],
|
||||
observe: ["--lane", "--seconds", "--since"],
|
||||
@@ -1033,6 +1034,56 @@ async function observerAction(
|
||||
return response;
|
||||
}
|
||||
|
||||
async function runDesktopActions(
|
||||
state: ActiveSession,
|
||||
values: Map<string, string>,
|
||||
roots: Roots,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const actions = readPublicFile(
|
||||
roots.outputRoot,
|
||||
required(values, "--actions-file"),
|
||||
"--actions-file",
|
||||
64 * 1024,
|
||||
);
|
||||
if (!actions.text.trim()) {
|
||||
throw new Error("--actions-file must not be empty.");
|
||||
}
|
||||
const timeoutSeconds = values.has("--timeout-seconds")
|
||||
? numberOption(values, "--timeout-seconds", 300, 1)
|
||||
: 60;
|
||||
const privateActions = path.join(
|
||||
state.privateDir,
|
||||
`desktop-actions-${state.invocations.length + 1}.json`,
|
||||
);
|
||||
fs.mkdirSync(state.privateDir, { recursive: true });
|
||||
fs.writeFileSync(privateActions, actions.text, { mode: 0o640 });
|
||||
const actionsSha256 = createHash("sha256").update(actions.text).digest("hex");
|
||||
appendInvocation(state, "desktop", {
|
||||
actionsFile: actions.relative,
|
||||
actionsSha256,
|
||||
timeoutSeconds,
|
||||
});
|
||||
saveActive(roots.sessionRoot, state);
|
||||
const result = z
|
||||
.object({
|
||||
results: z.array(z.object({ command: z.string(), stderr: z.string(), stdout: z.string() })),
|
||||
})
|
||||
.parse(
|
||||
JSON.parse(
|
||||
await runCommandOutput(requiredEnv("OPENCLAW_TELEGRAM_DESKTOP_RECORDER_CMD"), [
|
||||
"actions",
|
||||
"--session",
|
||||
recorderRelativePath(state.recorderSession),
|
||||
"--actions-file",
|
||||
recorderRelativePath(privateActions),
|
||||
"--timeout-seconds",
|
||||
String(timeoutSeconds),
|
||||
]),
|
||||
),
|
||||
);
|
||||
return { ...result, actionsSha256 };
|
||||
}
|
||||
|
||||
async function focusMessage(state: ActiveSession, messageId: string): Promise<void> {
|
||||
if (!/^\d+$/u.test(messageId) || BigInt(messageId) < 1n) {
|
||||
throw new Error("--message-id must be a positive Telegram server message id.");
|
||||
@@ -1445,6 +1496,8 @@ async function main(): Promise<void> {
|
||||
const credential = credentialSchema.parse(readJson(roots.credentialFile));
|
||||
if (cli.command === "mock") {
|
||||
outputJson(updateMockResponse(state, cli.values, roots.outputRoot));
|
||||
} else if (cli.command === "desktop") {
|
||||
outputJson(await runDesktopActions(state, cli.values, roots));
|
||||
} else if (cli.command === "send") {
|
||||
const sent = await sendVisibleMessage(state, cli.values, roots, credential.sutToken);
|
||||
outputJson({ ...sent.response, revealedMessageId: sent.revealedMessageId });
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
readRecorderSession,
|
||||
recoverRecorderStartup,
|
||||
recorderArtifacts,
|
||||
runRecorderActions,
|
||||
screenshotRecorder,
|
||||
type RecorderOperations,
|
||||
type RecorderSession,
|
||||
@@ -59,7 +60,7 @@ function testSession(): RecorderSession {
|
||||
},
|
||||
schemaVersion: 1,
|
||||
startedAt: "2026-08-15T12:00:00.000Z",
|
||||
window: { height: 1000, width: 650, x: 635, y: 40 },
|
||||
window: { height: 1000, id: "0x04600007", width: 650, x: 635, y: 40 },
|
||||
userDriver: ["python3", "driver.py", "--account", "qa shared"],
|
||||
};
|
||||
}
|
||||
@@ -133,6 +134,85 @@ describe("Telegram Desktop recorder CLI", () => {
|
||||
command: "artifacts",
|
||||
sessionPath: "recorder.json",
|
||||
});
|
||||
expect(
|
||||
parseRecorderArgs([
|
||||
"actions",
|
||||
"--session",
|
||||
"recorder.json",
|
||||
"--actions-file",
|
||||
"click.json",
|
||||
"--timeout-seconds",
|
||||
"90",
|
||||
]),
|
||||
).toEqual({
|
||||
actionsFile: "click.json",
|
||||
command: "actions",
|
||||
sessionPath: "recorder.json",
|
||||
timeoutSeconds: 90,
|
||||
});
|
||||
});
|
||||
|
||||
it("targets the recorded Telegram window instead of the first matching window", async () => {
|
||||
const root = makeTempDir();
|
||||
const sessionPath = path.join(root, "recorder.json");
|
||||
const actionsPath = path.join(root, "click.json");
|
||||
writeRecorderSession(sessionPath, testSession());
|
||||
fs.writeFileSync(
|
||||
actionsPath,
|
||||
JSON.stringify([
|
||||
{ command: "click", x: 120, y: 240 },
|
||||
{ command: "sleep", milliseconds: 5 },
|
||||
]),
|
||||
);
|
||||
const sshRun = vi.fn<RecorderOperations["sshRun"]>(async () => ({
|
||||
stderr: "",
|
||||
stdout: "clicked\n",
|
||||
}));
|
||||
const operations = {
|
||||
createCroppedMotionPreview: vi.fn(async () => ({ crop: "", fps: 24, outputWidth: 650 })),
|
||||
createMotionPreview: vi.fn(async () => ({})),
|
||||
inspectCrabbox: vi.fn(async () => ({
|
||||
sshHost: "host",
|
||||
sshKey: "/tmp/key",
|
||||
sshPort: "22",
|
||||
sshUser: "user",
|
||||
})),
|
||||
runCommand: vi.fn(async () => ({ stderr: "", stdout: "" })),
|
||||
scpFromRemote: vi.fn(async () => undefined),
|
||||
sshRun,
|
||||
} satisfies RecorderOperations;
|
||||
|
||||
await expect(
|
||||
runRecorderActions(
|
||||
root,
|
||||
{
|
||||
actionsFile: "click.json",
|
||||
command: "actions",
|
||||
sessionPath: "recorder.json",
|
||||
timeoutSeconds: 90,
|
||||
},
|
||||
operations,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
results: [
|
||||
{ command: "click", stderr: "", stdout: "clicked\n" },
|
||||
{ command: "sleep", stderr: "", stdout: "" },
|
||||
],
|
||||
});
|
||||
expect(sshRun).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
command: expect.stringContaining(
|
||||
'xdotool windowactivate --sync "$win" mousemove --window "$win" 120 240 click 1',
|
||||
),
|
||||
stdio: "pipe",
|
||||
timeoutMs: 90_000,
|
||||
}),
|
||||
);
|
||||
const actionCommand = sshRun.mock.calls[0]?.[0].command;
|
||||
expect(actionCommand).toContain("win='0x04600007'");
|
||||
expect(actionCommand).toContain("tolower($1) == tolower(win)");
|
||||
expect(actionCommand).toContain('[ "$WIDTH" -ne 650 ]');
|
||||
expect(actionCommand).not.toContain("{print $1; exit}");
|
||||
});
|
||||
|
||||
it("requires start inputs and a -100 private-group chat id", () => {
|
||||
@@ -465,7 +545,7 @@ describe("Telegram Desktop recorder remote contract", () => {
|
||||
throw new Error("first desktop stayed on QR");
|
||||
}
|
||||
if (command.includes("getwindowgeometry")) {
|
||||
return { stderr: "", stdout: "635 40 650 1000" };
|
||||
return { stderr: "", stdout: "0x04600007 635 40 650 1000" };
|
||||
}
|
||||
return { stderr: "", stdout: "" };
|
||||
}),
|
||||
@@ -504,7 +584,7 @@ describe("Telegram Desktop recorder remote contract", () => {
|
||||
return { stderr: "", stdout: "tg://login?token=open-target-chat" };
|
||||
}
|
||||
if (command.includes("getwindowgeometry")) {
|
||||
return { stderr: "", stdout: "635 40 650 1000" };
|
||||
return { stderr: "", stdout: "0x04600007 635 40 650 1000" };
|
||||
}
|
||||
return { stderr: "", stdout: "" };
|
||||
});
|
||||
@@ -785,14 +865,15 @@ describe("Telegram Desktop recorder window geometry", () => {
|
||||
});
|
||||
|
||||
it("parses the measured window and rejects unusable geometry", () => {
|
||||
expect(parseWindowGeometry(" 636 45 648 995 \n")).toEqual({
|
||||
expect(parseWindowGeometry(" 0x04600007 636 45 648 995 \n")).toEqual({
|
||||
height: 995,
|
||||
id: "0x04600007",
|
||||
width: 648,
|
||||
x: 636,
|
||||
y: 45,
|
||||
});
|
||||
expect(() => parseWindowGeometry("636 45 648")).toThrow("was not readable");
|
||||
expect(() => parseWindowGeometry("636 45 10 10")).toThrow("too small to crop");
|
||||
expect(() => parseWindowGeometry("0x04600007 636 45 648")).toThrow("was not readable");
|
||||
expect(() => parseWindowGeometry("0x04600007 636 45 10 10")).toThrow("too small to crop");
|
||||
});
|
||||
|
||||
it("crops the recorded window instead of a fixed rectangle", async () => {
|
||||
@@ -800,7 +881,7 @@ describe("Telegram Desktop recorder window geometry", () => {
|
||||
const sessionPath = path.join(root, "recorder.json");
|
||||
writeRecorderSession(sessionPath, {
|
||||
...testSession(),
|
||||
window: { height: 995, width: 648, x: 636, y: 45 },
|
||||
window: { height: 995, id: "0x04600007", width: 648, x: 636, y: 45 },
|
||||
});
|
||||
const cropped = vi.fn(async () => ({ crop: "", fps: 24, outputWidth: 648 }));
|
||||
const sshRun = vi.fn<RecorderOperations["sshRun"]>(async () => ({ stderr: "", stdout: "" }));
|
||||
|
||||
@@ -52,7 +52,7 @@ async function setupHarness(
|
||||
fs.writeFileSync(trimmedVideo, Buffer.alloc(10_001));
|
||||
fs.writeFileSync(
|
||||
recorderCommand,
|
||||
`#!/bin/sh\nprintf '%s\\n' "$*" >> ${JSON.stringify(recorderLog)}\ncp ${JSON.stringify(path.join(root, "mock-response.json"))} ${JSON.stringify(recorderControlLog)}\n${options.failRecorder ? "exit 1\n" : ""}if [ "$1" = artifacts ]; then\n printf '%s\\n' ${JSON.stringify(JSON.stringify({ artifacts: { previewGifCropped: previewGif, screenshot, trimmedVideoCropped: trimmedVideo } }))}\nfi\n`,
|
||||
`#!/bin/sh\nprintf '%s\\n' "$*" >> ${JSON.stringify(recorderLog)}\ncp ${JSON.stringify(path.join(root, "mock-response.json"))} ${JSON.stringify(recorderControlLog)}\n${options.failRecorder ? "exit 1\n" : ""}if [ "$1" = artifacts ]; then\n printf '%s\\n' ${JSON.stringify(JSON.stringify({ artifacts: { previewGifCropped: previewGif, screenshot, trimmedVideoCropped: trimmedVideo } }))}\nelif [ "$1" = actions ]; then\n printf '%s\\n' ${JSON.stringify(JSON.stringify({ results: [{ command: "click", stderr: "", stdout: "clicked\n" }] }))}\nfi\n`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
fs.writeFileSync(userDriverCommand, "#!/bin/sh\nexit 0\n", { mode: 0o755 });
|
||||
@@ -340,6 +340,63 @@ describe("Telegram Mantis free-form lane", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("runs scenario-authored actions inside the ephemeral desktop", async () => {
|
||||
const harness = await setupHarness();
|
||||
const actions = path.join(harness.outputRoot, "click-picker.json");
|
||||
fs.writeFileSync(actions, JSON.stringify([{ command: "click", x: 120, y: 240 }]));
|
||||
try {
|
||||
const result = await runLane(harness.env, [
|
||||
"desktop",
|
||||
"--lane",
|
||||
"candidate",
|
||||
"--actions-file",
|
||||
actions,
|
||||
"--timeout-seconds",
|
||||
"90",
|
||||
]);
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({
|
||||
actionsSha256: expect.stringMatching(/^[0-9a-f]{64}$/u),
|
||||
results: [{ command: "click", stdout: "clicked\n" }],
|
||||
});
|
||||
expect(fs.readFileSync(harness.recorderLog, "utf8")).toContain(
|
||||
"actions --session attempt/recorder.json --actions-file attempt/desktop-actions-2.json --timeout-seconds 90",
|
||||
);
|
||||
const state = JSON.parse(
|
||||
fs.readFileSync(path.join(harness.sessionRoot, "candidate.active.json"), "utf8"),
|
||||
);
|
||||
expect(state.invocations.at(-1)).toMatchObject({
|
||||
args: { actionsFile: "click-picker.json", timeoutSeconds: 90 },
|
||||
command: "desktop",
|
||||
});
|
||||
} finally {
|
||||
await harness.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("records desktop actions before a timeout or failure", async () => {
|
||||
const harness = await setupHarness({ failRecorder: true });
|
||||
const actions = path.join(harness.outputRoot, "failed-actions.json");
|
||||
fs.writeFileSync(actions, JSON.stringify([{ command: "click", x: 120, y: 240 }]));
|
||||
try {
|
||||
await expect(
|
||||
runLane(harness.env, ["desktop", "--lane", "candidate", "--actions-file", actions]),
|
||||
).rejects.toThrow();
|
||||
const state = JSON.parse(
|
||||
fs.readFileSync(path.join(harness.sessionRoot, "candidate.active.json"), "utf8"),
|
||||
);
|
||||
expect(state.invocations.at(-1)).toMatchObject({
|
||||
args: {
|
||||
actionsFile: "failed-actions.json",
|
||||
actionsSha256: expect.stringMatching(/^[0-9a-f]{64}$/u),
|
||||
timeoutSeconds: 60,
|
||||
},
|
||||
command: "desktop",
|
||||
});
|
||||
} finally {
|
||||
await harness.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("retains the sent message when revealing it fails", async () => {
|
||||
const harness = await setupHarness({ failRecorder: true });
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user