perf(e2e): reuse one authorized Telegram Desktop session per Mantis run (#127835)

* refactor(mantis): reuse authorized desktop captures

* fix(mantis): budget desktop authorization failures

* chore(mantis): bound desktop proof retries

* fix(e2e): drop unused recorder failure fact type export

* fix(ci): route Mantis desktop teardown through the recorder wrapper

Cleanup invoked the internal recorder executable as mantis-sut, which is
deliberately kept out of the docker group and cannot read the
recorder-owned session file; teardown therefore failed and blocked
safe_to_release. The cleanup step already runs as the recorder user, so
call the public wrapper whose exec shim cds into the session root.

* fix(e2e): make recorder failure fact lane-readable; document v2 lifecycle

The Mantis workflow runs the recorder as the desktop user while the lane
reads the authorization-failure fact as mantis-sut; 0600 made that read
fail EACCES and silently disabled the two-attempt retry budget. Write the
fact 0644 — the 0770 attempt directory bounds visibility.

Update the mantis doc's recorder section for the v2 session lifecycle:
required --session handle with healthy-session reuse, capture-only stop,
and teardown owning authorization termination and lease release.
This commit is contained in:
Ayaan Zaidi
2026-08-22 15:58:12 +05:30
committed by GitHub
parent 2d0fff4ac5
commit da0cb592dd
9 changed files with 797 additions and 253 deletions
@@ -4,16 +4,33 @@ import { z } from "zod";
export const TELEGRAM_DESKTOP_VERSION = "7.0.9";
export const TELEGRAM_DESKTOP_AWS_IMAGE = "telegram-desktop=7.0.9";
export const TELEGRAM_DESKTOP_DOCKER_IMAGE = "openclaw-telegram-desktop:7.0.9";
export const RECORDER_AUTHORIZATION_FAILURE_FILENAME =
"telegram-desktop-authorization-failure.json";
export const recorderAuthorizationFailureSchema = z.object({
acceptedTokenCount: z.number().int().nonnegative(),
classification: z.enum(["main-window-timeout", "qr-unreadable", "token-accepted-no-transition"]),
failedAt: z.string(),
loginScreenshotPath: z.string().optional(),
qrAttemptCount: z.number().int().positive(),
});
export const recorderAuthorizationFailureFactSchema = z.object({
failures: z.array(recorderAuthorizationFailureSchema).min(1),
schemaVersion: z.literal(1),
});
export type RecorderAuthorizationFailure = z.infer<typeof recorderAuthorizationFailureSchema>;
const recorderSessionBaseSchema = z.object({
artifacts: z.record(z.string(), z.string()).optional(),
chat: z.string().regex(/^-100\d+$/u),
cleanupErrors: z.array(z.string()).optional(),
desktopSessionId: z.string().min(1),
keepBox: z.boolean(),
leaseId: z.string().min(1),
/** False when `--lease-id` borrowed an existing box: the recorder never stops it. */
leaseOwned: z.boolean(),
outputDir: z.string().min(1),
recordFps: z.number().int().positive(),
remotePaths: z.object({
desktopLog: z.string(),
@@ -22,7 +39,7 @@ const recorderSessionBaseSchema = z.object({
finalScreenshot: z.string(),
video: z.string(),
}),
schemaVersion: z.literal(1),
schemaVersion: z.literal(2),
startedAt: z.string(),
/** Telegram Desktop window as placed on the recorded desktop; the crop uses it. */
window: z.object({
@@ -61,6 +78,7 @@ export type StartOptions = {
outputDir: string;
provider: RecorderProvider;
recordFps: number;
sessionPath: string;
ttl: string;
userDriver: string[];
};
@@ -87,7 +105,11 @@ export type ScreenshotOptions = {
export type StopOptions = {
command: "stop";
crop?: "telegram-window";
keepBox: boolean;
sessionPath: string;
};
export type TeardownOptions = {
command: "teardown";
sessionPath: string;
};
@@ -114,18 +136,20 @@ type RecorderOptions =
| StartOptions
| StatusOptions
| StopOptions
| TeardownOptions
| ViewOptions;
export function recorderUsageText(): string {
return [
"Usage:",
" pnpm qa:telegram-desktop-recorder artifacts --session <recorder.json>",
' pnpm qa:telegram-desktop-recorder start --output-dir <dir> --chat <-100groupId> --user-driver "<space-separated cmd prefix>" [options]',
' pnpm qa:telegram-desktop-recorder start --session <recorder.json> --output-dir <dir> --chat <-100groupId> --user-driver "<space-separated cmd prefix>" [options]',
" pnpm qa:telegram-desktop-recorder view --session <recorder.json> --message-id <id>",
" pnpm qa:telegram-desktop-recorder actions --session <recorder.json> --actions-file <json> [--timeout-seconds <seconds>]",
" pnpm qa:telegram-desktop-recorder screenshot --session <recorder.json> [--output <png>]",
" pnpm qa:telegram-desktop-recorder recover --session <recorder.json>",
" pnpm qa:telegram-desktop-recorder stop --session <recorder.json> [--crop telegram-window] [--keep-box]",
" pnpm qa:telegram-desktop-recorder stop --session <recorder.json> [--crop telegram-window]",
" pnpm qa:telegram-desktop-recorder teardown --session <recorder.json>",
" pnpm qa:telegram-desktop-recorder status --session <recorder.json>",
"",
"Start options:",
@@ -173,7 +197,17 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
throw new Error(recorderUsageText());
}
const parsedCommand = z
.enum(["actions", "artifacts", "recover", "screenshot", "start", "status", "stop", "view"])
.enum([
"actions",
"artifacts",
"recover",
"screenshot",
"start",
"status",
"stop",
"teardown",
"view",
])
.safeParse(rawCommand);
if (!parsedCommand.success) {
throw new Error(`Unknown command: ${rawCommand}\n\n${recorderUsageText()}`);
@@ -186,7 +220,7 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
if (!flag) {
break;
}
if (flag === "--json" || flag === "--keep-box") {
if (flag === "--json") {
switches.add(flag);
continue;
}
@@ -210,6 +244,7 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
"--output-dir",
"--provider",
"--record-fps",
"--session",
"--ttl",
"--user-driver",
])
@@ -230,9 +265,6 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
if (switches.has("--json") && command !== "start") {
throw new Error(`--json is not available for ${command}.`);
}
if (switches.has("--keep-box") && command !== "stop") {
throw new Error(`--keep-box is not available for ${command}.`);
}
if (command === "start") {
const chat = requiredString(values, "--chat");
if (!/^-100\d+$/u.test(chat)) {
@@ -268,6 +300,7 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
outputDir: requiredString(values, "--output-dir"),
provider,
recordFps: positiveInteger(values.get("--record-fps") ?? "24", "--record-fps"),
sessionPath: requiredString(values, "--session"),
ttl: values.get("--ttl") ?? "2h",
userDriver,
};
@@ -292,12 +325,12 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
if (command === "stop") {
const crop = values.get("--crop");
if (crop === undefined) {
return { command, keepBox: switches.has("--keep-box"), sessionPath };
return { command, sessionPath };
}
if (crop !== "telegram-window") {
throw new Error("--crop must be telegram-window.");
}
return { command, crop, keepBox: switches.has("--keep-box"), sessionPath };
return { command, crop, sessionPath };
}
return { command, sessionPath };
}
+280 -115
View File
@@ -26,6 +26,9 @@ import {
import {
parseRecorderArgs,
readRecorderSession,
RECORDER_AUTHORIZATION_FAILURE_FILENAME,
recorderAuthorizationFailureFactSchema,
type RecorderAuthorizationFailure,
type ActionsOptions,
type ArtifactsOptions,
type RecoverOptions,
@@ -39,6 +42,7 @@ import {
type StartOptions,
type StatusOptions,
type StopOptions,
type TeardownOptions,
type ViewOptions,
writeRecorderSession,
} from "./telegram-desktop-recorder-contract.ts";
@@ -90,7 +94,14 @@ const confirmedQrSchema = z.object({
}),
});
class FreshDesktopRequiredError extends Error {}
class DesktopAuthorizationError extends Error {
readonly failure: RecorderAuthorizationFailure;
constructor(message: string, failure: RecorderAuthorizationFailure, options?: ErrorOptions) {
super(`${failure.classification}: ${message}`, options);
this.failure = failure;
}
}
const recorderStartupSchema = z.object({
desktopSessionId: z.string().min(1).optional(),
@@ -331,7 +342,9 @@ async function authorizeDesktop(params: {
let lastFailure: unknown;
let lastLink = "";
let acceptedWithoutTransition = 0;
let qrAttemptCount = 0;
for (let attempt = 1; attempt <= 6; attempt += 1) {
qrAttemptCount = attempt;
let link: string;
try {
const qr = await params.operations.sshRun({
@@ -390,6 +403,7 @@ async function authorizeDesktop(params: {
// an absent screenshot.
const evidencePath = path.join(params.outputDir, "telegram-login-screen.png");
let evidence: string;
let loginScreenshotPath: string | undefined;
try {
await params.operations.scpFromRemote({
cwd: params.cwd,
@@ -398,18 +412,34 @@ async function authorizeDesktop(params: {
remote: `${REMOTE_ROOT}/telegram-login-qr.png`,
run: params.operations.runCommand,
});
loginScreenshotPath = evidencePath;
evidence = ` Login screen: ${evidencePath}`;
} catch (error) {
evidence = ` Login screen could not be fetched: ${coerceErrorMessage(error)}`;
}
const message =
const classification =
acceptedWithoutTransition >= 2
? "token-accepted-no-transition"
: acceptedWithoutTransition === 1
? "main-window-timeout"
: "qr-unreadable";
const message =
classification === "token-accepted-no-transition"
? `Telegram server accepted ${acceptedWithoutTransition} login tokens, but Telegram Desktop stayed on the QR screen${detail}.${evidence}`
: `Telegram Desktop did not leave the login screen after 6 attempts${detail}.${evidence}`;
if (acceptedWithoutTransition >= 2) {
throw new FreshDesktopRequiredError(message, { cause: lastFailure });
}
throw new Error(message, { cause: lastFailure });
: classification === "main-window-timeout"
? `Telegram Desktop did not reach the main window after an accepted login token${detail}.${evidence}`
: `Telegram Desktop did not leave the login screen after 6 attempts${detail}.${evidence}`;
throw new DesktopAuthorizationError(
message,
{
acceptedTokenCount: acceptedWithoutTransition,
classification,
failedAt: new Date().toISOString(),
loginScreenshotPath,
qrAttemptCount,
},
{ cause: lastFailure },
);
}
// The recorder runs as a different user than the agent that drives it, so an output dir
@@ -448,6 +478,27 @@ function resolveOutputDir(cwd: string, outputDir: string): string {
return resolveRecorderPath(cwd, outputDir, "--output-dir");
}
function appendAuthorizationFailure(
outputDir: string,
failure: RecorderAuthorizationFailure,
): void {
const file = path.join(outputDir, RECORDER_AUTHORIZATION_FAILURE_FILENAME);
const current = fs.existsSync(file)
? recorderAuthorizationFailureFactSchema.parse(JSON.parse(fs.readFileSync(file, "utf8")))
: { failures: [], schemaVersion: 1 as const };
const fact = recorderAuthorizationFailureFactSchema.parse({
failures: [...current.failures, failure],
schemaVersion: 1,
});
const temporary = `${file}.${process.pid}.tmp`;
// 0644: cross-identity evidence — the Mantis workflow runs the recorder as the
// desktop user while the lane reads this fact as mantis-sut to enforce its
// retry budget; the 0770 attempt directory bounds visibility. 0600 would make
// the lane's read fail EACCES and silently disable the budget.
fs.writeFileSync(temporary, `${JSON.stringify(fact, null, 2)}\n`, { mode: 0o644 });
fs.renameSync(temporary, file);
}
async function stopBox(params: {
crabboxBin: string;
cwd: string;
@@ -489,6 +540,40 @@ async function terminateDesktopSessions(params: {
z.object({ ok: z.literal(true) }).parse(JSON.parse(result.stdout));
}
async function destroyRecorderSessionResources(
cwd: string,
session: RecorderSession,
operations: Pick<RecorderOperations, "runCommand">,
): Promise<string[]> {
const errors: string[] = [];
try {
await terminateDesktopSession({
cwd,
desktopSessionId: session.desktopSessionId,
run: operations.runCommand,
userDriver: session.userDriver,
});
} catch (error) {
errors.push(`terminate Telegram Desktop session: ${coerceErrorMessage(error)}`);
}
if (session.leaseOwned) {
try {
await stopBox({
crabboxBin: process.env.OPENCLAW_TELEGRAM_USER_CRABBOX_BIN?.trim() || "crabbox",
cwd,
leaseId: session.leaseId,
provider: session.provider,
run: operations.runCommand,
});
} catch (error) {
if (!coerceErrorMessage(error).includes("lease not found")) {
errors.push(`stop Crabbox: ${coerceErrorMessage(error)}`);
}
}
}
return errors;
}
async function assertLocalTelegramImage(params: { cwd: string; run: RunCommand }): Promise<void> {
try {
await params.run({
@@ -507,17 +592,109 @@ async function assertLocalTelegramImage(params: { cwd: string; run: RunCommand }
}
}
async function startRecorderAttempt(
type RecorderResource = Pick<
RecorderSession,
"desktopSessionId" | "leaseId" | "leaseOwned" | "provider" | "userDriver"
>;
async function beginRecorderCapture(params: {
cwd: string;
inspect: CrabboxInspect;
operations: RecorderOperations;
opts: StartOptions;
outputDir: string;
resource: RecorderResource;
sessionPath: string;
}): Promise<{ session: RecorderSession; sessionPath: string }> {
await params.operations.sshRun({
command: renderTelegramViewCommand({
binary: TELEGRAM_BINARY,
link: telegramPrivatePostLink(params.opts.chat, params.opts.messageId),
workdir: TELEGRAM_WORKDIR,
}),
cwd: params.cwd,
inspect: params.inspect,
run: params.operations.runCommand,
});
const geometry = await params.operations.sshRun({
command: renderReadWindowGeometry(),
cwd: params.cwd,
inspect: params.inspect,
run: params.operations.runCommand,
stdio: "pipe",
});
const windowGeometry = parseWindowGeometry(geometry.stdout);
await params.operations.sshRun({
command: renderHideTelegramWindow(windowGeometry.id),
cwd: params.cwd,
inspect: params.inspect,
run: params.operations.runCommand,
});
const sessionBase = {
chat: params.opts.chat,
desktopSessionId: params.resource.desktopSessionId,
leaseId: params.resource.leaseId,
leaseOwned: params.resource.leaseOwned,
outputDir: params.outputDir,
recordFps: params.opts.recordFps,
remotePaths,
schemaVersion: 2 as const,
startedAt: new Date().toISOString(),
userDriver: params.resource.userDriver,
window: windowGeometry,
};
const session: RecorderSession =
params.resource.provider === "docker"
? {
...sessionBase,
imageSource: TELEGRAM_DESKTOP_DOCKER_IMAGE,
provider: "docker",
}
: {
...sessionBase,
imageSource: TELEGRAM_DESKTOP_AWS_IMAGE,
provider: "aws",
};
// Publish the current capture destination before ffmpeg starts so crash recovery
// exports or stops only this attempt, never an earlier attempt's artifacts.
writeRecorderSession(params.sessionPath, session);
await params.operations.sshRun({
command: renderStartRemoteRecording({ paths: remotePaths, recordFps: params.opts.recordFps }),
cwd: params.cwd,
inspect: params.inspect,
run: params.operations.runCommand,
});
return { session, sessionPath: params.sessionPath };
}
async function inspectHealthySession(params: {
crabboxBin: string;
cwd: string;
operations: RecorderOperations;
session: RecorderSession;
}): Promise<CrabboxInspect | undefined> {
try {
const inspect = await sessionInspect(params);
const mainWindow = await desktopReachedMainWindow({
cwd: params.cwd,
inspect,
operations: params.operations,
seconds: 5,
});
return mainWindow.reached ? inspect : undefined;
} catch {
return undefined;
}
}
async function provisionRecorder(
cwd: string,
opts: StartOptions,
operations: RecorderOperations,
freshContainerAttempt: number,
outputDir: string,
sessionPath: string,
): Promise<{ session: RecorderSession; sessionPath: string }> {
const crabboxBin = process.env.OPENCLAW_TELEGRAM_USER_CRABBOX_BIN?.trim() || "crabbox";
const outputDir = resolveOutputDir(cwd, opts.outputDir);
fs.mkdirSync(outputDir, { recursive: true });
assertOutputDirWritable(outputDir);
const sessionPath = path.join(outputDir, "recorder.json");
const startupPath = recorderStartupPath(sessionPath);
let leaseId = opts.leaseId;
const leaseOwned = !opts.leaseId;
@@ -532,7 +709,7 @@ async function startRecorderAttempt(
};
// Provisioning crosses process and provider boundaries. Persist each acquired
// handle so a later workflow step can reclaim it after cancellation or SIGKILL.
writeRecorderStartup(startupPath, startup, freshContainerAttempt === 1);
writeRecorderStartup(startupPath, startup, true);
try {
if (!leaseId) {
if (opts.provider === "docker") {
@@ -593,71 +770,27 @@ async function startRecorderAttempt(
outputDir,
userDriver: opts.userDriver,
});
// Always open the target chat before recording: at the recorder's width Telegram shows
// either the chat list or one conversation, and the list is the QA account's own.
await operations.sshRun({
command: renderTelegramViewCommand({
binary: TELEGRAM_BINARY,
link: telegramPrivatePostLink(opts.chat, opts.messageId),
workdir: TELEGRAM_WORKDIR,
}),
const result = await beginRecorderCapture({
cwd,
inspect,
run: operations.runCommand,
operations,
opts,
outputDir,
resource: {
desktopSessionId,
leaseId,
leaseOwned,
provider: opts.provider,
userDriver: opts.userDriver,
},
sessionPath,
});
// Crop from the window Telegram actually got: window managers and providers
// place it differently, and a fixed crop silently cuts the chat pane.
const geometry = await operations.sshRun({
command: renderReadWindowGeometry(),
cwd,
inspect,
run: operations.runCommand,
stdio: "pipe",
});
const windowGeometry = parseWindowGeometry(geometry.stdout);
// The lane clears prior history before recorder start. Keep the empty chat hidden until
// the first session-owned send is ready, so setup frames reveal neither account UI nor chat.
await operations.sshRun({
command: renderHideTelegramWindow(windowGeometry.id),
cwd,
inspect,
run: operations.runCommand,
});
await operations.sshRun({
command: renderStartRemoteRecording({ paths: remotePaths, recordFps: opts.recordFps }),
cwd,
inspect,
run: operations.runCommand,
});
const sessionBase: Omit<RecorderSession, "imageSource" | "provider"> = {
chat: opts.chat,
desktopSessionId,
keepBox: false,
leaseId,
leaseOwned,
recordFps: opts.recordFps,
remotePaths,
schemaVersion: 1,
window: windowGeometry,
startedAt: new Date().toISOString(),
userDriver: opts.userDriver,
};
const session: RecorderSession =
opts.provider === "docker"
? {
...sessionBase,
imageSource: TELEGRAM_DESKTOP_DOCKER_IMAGE,
provider: opts.provider,
}
: {
...sessionBase,
imageSource: TELEGRAM_DESKTOP_AWS_IMAGE,
provider: opts.provider,
};
writeRecorderSession(sessionPath, session);
fs.rmSync(startupPath);
return { session, sessionPath };
return result;
} catch (error) {
if (error instanceof DesktopAuthorizationError) {
appendAuthorizationFailure(outputDir, error.failure);
}
const cleanupErrors: string[] = [];
if (desktopAuthorizationRequested) {
try {
@@ -688,17 +821,9 @@ async function startRecorderAttempt(
cleanupErrors.push(coerceErrorMessage(cleanupError));
}
}
if (
error instanceof FreshDesktopRequiredError &&
cleanupErrors.length === 0 &&
leaseOwned &&
opts.provider === "docker" &&
freshContainerAttempt === 1
) {
return await startRecorderAttempt(cwd, opts, operations, freshContainerAttempt + 1);
}
if (cleanupErrors.length === 0) {
fs.rmSync(startupPath, { force: true });
fs.rmSync(sessionPath, { force: true });
}
const suffix = cleanupErrors.length ? ` Cleanup also failed: ${cleanupErrors.join("; ")}` : "";
throw new Error(`${coerceErrorMessage(error)}${suffix}`, { cause: error });
@@ -710,7 +835,40 @@ export async function startRecorder(
opts: StartOptions,
operations: RecorderOperations = defaultOperations,
): Promise<{ session: RecorderSession; sessionPath: string }> {
return await startRecorderAttempt(cwd, opts, operations, 1);
const outputDir = resolveOutputDir(cwd, opts.outputDir);
fs.mkdirSync(outputDir, { recursive: true });
assertOutputDirWritable(outputDir);
const sessionPath = resolveRecorderPath(cwd, opts.sessionPath, "--session");
if (!fs.existsSync(sessionPath)) {
return await provisionRecorder(cwd, opts, operations, outputDir, sessionPath);
}
const session = readRecorderSession(sessionPath);
if (
session.chat !== opts.chat ||
session.provider !== opts.provider ||
session.userDriver.join("\0") !== opts.userDriver.join("\0")
) {
throw new Error("Existing recorder session does not match this start request.");
}
const crabboxBin = process.env.OPENCLAW_TELEGRAM_USER_CRABBOX_BIN?.trim() || "crabbox";
const inspect = await inspectHealthySession({ crabboxBin, cwd, operations, session });
if (inspect) {
return await beginRecorderCapture({
cwd,
inspect,
operations,
opts,
outputDir,
resource: session,
sessionPath,
});
}
const cleanupErrors = await destroyRecorderSessionResources(cwd, session, operations);
if (cleanupErrors.length) {
throw new Error(`Unhealthy recorder session cleanup failed:\n${cleanupErrors.join("\n")}`);
}
fs.rmSync(sessionPath);
return await provisionRecorder(cwd, opts, operations, outputDir, sessionPath);
}
export async function recoverRecorderStartup(
@@ -767,8 +925,8 @@ export function recorderArtifacts(
opts: ArtifactsOptions,
): { artifacts: Record<string, string> } {
const sessionPath = resolveRecorderPath(cwd, opts.sessionPath, "--session");
const outputDir = path.dirname(sessionPath);
const session = readRecorderSession(sessionPath);
const outputDir = session.outputDir;
const artifacts: Record<string, string> = {};
for (const [name, file] of Object.entries(session.artifacts ?? {})) {
const resolved = path.resolve(file);
@@ -942,7 +1100,7 @@ export async function screenshotRecorder(
const output =
opts.output ??
path.join(
path.dirname(opts.sessionPath),
path.relative(cwd, session.outputDir),
`telegram-desktop-recorder-screenshot-${new Date().toISOString().replace(/[:.]/gu, "-")}.png`,
);
const outputPath = resolveRecorderPath(cwd, output, "--output");
@@ -967,7 +1125,7 @@ export async function stopRecorder(
const sessionPath = resolveRecorderPath(cwd, opts.sessionPath, "--session");
const session = readRecorderSession(sessionPath);
const crabboxBin = process.env.OPENCLAW_TELEGRAM_USER_CRABBOX_BIN?.trim() || "crabbox";
const outputDir = path.dirname(sessionPath);
const outputDir = session.outputDir;
const errors: string[] = [];
const artifacts: Record<string, string> = {};
const attempt = async (label: string, action: () => Promise<void>) => {
@@ -978,14 +1136,12 @@ export async function stopRecorder(
}
};
let inspect: CrabboxInspect | undefined;
let leaseGone = false;
await attempt("inspect", async () => {
try {
inspect = await sessionInspect({ crabboxBin, cwd, operations, session });
} catch (error) {
// A lease that no longer exists is the desired end state, not a failure.
if (coerceErrorMessage(error).includes("lease not found")) {
leaseGone = true;
return;
}
throw error;
@@ -1077,36 +1233,12 @@ export async function stopRecorder(
});
}
}
// --keep-box keeps the whole debugging surface: the Desktop authorization stays
// valid for WebVNC until the operator finishes; a later `stop` without it revokes.
if (!opts.keepBox) {
await attempt("terminate Telegram Desktop session", async () => {
await terminateDesktopSession({
cwd,
desktopSessionId: session.desktopSessionId,
run: operations.runCommand,
userDriver: session.userDriver,
});
});
}
if (!opts.keepBox && session.leaseOwned && !leaseGone) {
await attempt("stop Crabbox", async () => {
await stopBox({
crabboxBin,
cwd,
leaseId: session.leaseId,
provider: session.provider,
run: operations.runCommand,
});
});
}
const stopped: RecorderSession = {
...session,
// Keep paths recorded by an earlier stop (--keep-box, then a later stop once
// the lease expired); fresh copies overwrite their own entries.
// Keep paths recorded by an earlier stop if a later cleanup sees an expired lease;
// fresh copies overwrite their own entries.
artifacts: { ...session.artifacts, ...artifacts },
cleanupErrors: errors.length ? errors : undefined,
keepBox: opts.keepBox,
stoppedAt: new Date().toISOString(),
};
writeRecorderSession(sessionPath, stopped);
@@ -1116,6 +1248,35 @@ export async function stopRecorder(
return stopped;
}
export async function teardownRecorder(
cwd: string,
opts: TeardownOptions,
operations: Pick<RecorderOperations, "runCommand"> = defaultOperations,
): Promise<{ tornDown: boolean }> {
const sessionPath = resolveRecorderPath(cwd, opts.sessionPath, "--session");
const startupPath = recorderStartupPath(sessionPath);
if (fs.existsSync(startupPath)) {
await recoverRecorderStartup(
cwd,
{ command: "recover", sessionPath: opts.sessionPath },
operations,
);
fs.rmSync(sessionPath, { force: true });
return { tornDown: true };
}
if (!fs.existsSync(sessionPath)) {
return { tornDown: false };
}
const session = readRecorderSession(sessionPath);
const errors = await destroyRecorderSessionResources(cwd, session, operations);
if (errors.length) {
writeRecorderSession(sessionPath, { ...session, cleanupErrors: errors });
throw new Error(`Recorder teardown completed with errors:\n${errors.join("\n")}`);
}
fs.rmSync(sessionPath);
return { tornDown: true };
}
async function statusRecorder(
cwd: string,
opts: StatusOptions,
@@ -1173,6 +1334,10 @@ async function main(): Promise<void> {
console.log(JSON.stringify(await stopRecorder(cwd, opts), null, 2));
return;
}
if (opts.command === "teardown") {
console.log(JSON.stringify(await teardownRecorder(cwd, opts), null, 2));
return;
}
console.log(JSON.stringify(await statusRecorder(cwd, opts, defaultOperations), null, 2));
}
+116 -13
View File
@@ -11,6 +11,12 @@ import { z } from "zod";
import { coerceErrorMessage } from "../lib/error-format.mts";
import { sleep } from "../lib/sleep.mjs";
import { telegramBotApi } from "./telegram-bot-api.ts";
import {
RECORDER_AUTHORIZATION_FAILURE_FILENAME,
recorderAuthorizationFailureSchema,
recorderAuthorizationFailureFactSchema,
type RecorderAuthorizationFailure,
} from "./telegram-desktop-recorder-contract.ts";
import {
destroyMantisSut,
type MantisSutRecovery,
@@ -119,6 +125,13 @@ const invocationSchema = z.object({
const recorderArtifactsSchema = z.object({
artifacts: z.record(z.string(), z.string()),
});
const desktopRecorderFailureBudgetSchema = z.object({
attemptCount: z.number().int().positive(),
classification: recorderAuthorizationFailureSchema.shape.classification,
loginScreenshotPath: z.string().optional(),
schemaVersion: z.literal(1),
unavailable: z.boolean(),
});
const activeSessionSchema = z.object({
attempt: z.number().int().positive(),
config: configSchema,
@@ -151,6 +164,7 @@ type ObserverResponse = {
ok: boolean;
truncated?: boolean;
} & Record<string, unknown>;
type DesktopRecorderFailureBudget = z.infer<typeof desktopRecorderFailureBudgetSchema>;
const MAX_SENDS = 12;
const MAX_RPC_BYTES = 4 * 1024 * 1024;
@@ -315,6 +329,98 @@ function writeJsonAtomic(file: string, value: unknown, mode = 0o600): void {
fs.chmodSync(file, mode);
}
function desktopRecorderFailureBudgetFile(sessionRoot: string): string {
return path.join(sessionRoot, "desktop-recorder-failures.json");
}
function readDesktopRecorderFailureBudget(
sessionRoot: string,
): DesktopRecorderFailureBudget | undefined {
const file = desktopRecorderFailureBudgetFile(sessionRoot);
return fs.existsSync(file) ? desktopRecorderFailureBudgetSchema.parse(readJson(file)) : undefined;
}
function desktopUnavailableMessage(fact: DesktopRecorderFailureBudget, factFile: string): string {
const screenshotDetail = fact.loginScreenshotPath
? `, loginScreenshotPath=${fact.loginScreenshotPath}`
: "";
return (
`desktop-unavailable: stop retrying; this run's desktop is unavailable ` +
`(attemptCount=${fact.attemptCount}, classification=${fact.classification}${screenshotDetail}, ` +
`fact=${factFile})`
);
}
function assertDesktopRecorderAvailable(sessionRoot: string): void {
const fact = readDesktopRecorderFailureBudget(sessionRoot);
if (fact?.unavailable) {
throw new Error(desktopUnavailableMessage(fact, desktopRecorderFailureBudgetFile(sessionRoot)));
}
}
function recordDesktopRecorderFailures(
sessionRoot: string,
failures: RecorderAuthorizationFailure[],
): DesktopRecorderFailureBudget | undefined {
if (failures.length === 0) {
return readDesktopRecorderFailureBudget(sessionRoot);
}
const prior = readDesktopRecorderFailureBudget(sessionRoot);
const latest = failures.at(-1);
if (!latest) {
return prior;
}
const attemptCount = (prior?.attemptCount ?? 0) + failures.length;
const fact = desktopRecorderFailureBudgetSchema.parse({
attemptCount,
classification: latest.classification,
loginScreenshotPath: latest.loginScreenshotPath,
schemaVersion: 1,
unavailable: attemptCount >= 2,
});
writeJsonAtomic(desktopRecorderFailureBudgetFile(sessionRoot), fact);
return fact;
}
export async function startDesktopRecorder(params: {
chat: string;
outputDir: string;
recorderCommand: string;
sessionPath: string;
sessionRoot: string;
userDriver: string;
}): Promise<void> {
assertDesktopRecorderAvailable(params.sessionRoot);
try {
await runCommand(params.recorderCommand, [
"start",
"--provider",
"docker",
"--session",
recorderRelativePath(params.sessionPath),
"--output-dir",
recorderRelativePath(params.outputDir),
"--chat",
params.chat,
"--user-driver",
params.userDriver,
]);
} catch (startError) {
const failureFile = path.join(params.outputDir, RECORDER_AUTHORIZATION_FAILURE_FILENAME);
const failures = fs.existsSync(failureFile)
? recorderAuthorizationFailureFactSchema.parse(readJson(failureFile)).failures
: [];
const budget = recordDesktopRecorderFailures(params.sessionRoot, failures);
if (budget?.unavailable) {
throw new Error(
`${desktopUnavailableMessage(budget, desktopRecorderFailureBudgetFile(params.sessionRoot))}\n${coerceErrorMessage(startError)}`,
{ cause: startError },
);
}
throw startError;
}
}
function publicRelativePath(root: string, file: string, label: string): string {
const resolvedRoot = fs.realpathSync(root);
const relative = path.relative(resolvedRoot, file);
@@ -801,17 +907,17 @@ async function startLane(values: Map<string, string>, roots: Roots): Promise<voi
) {
throw new Error(`Finish or abort the active ${otherLane} session first.`);
}
assertDesktopRecorderAvailable(roots.sessionRoot);
const attemptsRoot = path.join(roots.sessionRoot, "attempts", lane);
fs.mkdirSync(attemptsRoot, { recursive: true });
const attempt = fs.readdirSync(attemptsRoot).filter((entry) => /^\d+$/u.test(entry)).length + 1;
const privateDir = path.join(attemptsRoot, String(attempt));
fs.mkdirSync(privateDir, { mode: 0o770 });
const recorderSession = path.join(privateDir, "recorder.json");
const recorderSession = path.join(roots.sessionRoot, "desktop-recorder.json");
const observerSocket = path.join(privateDir, "observer.sock");
const observerJournal = path.join(privateDir, "telegram-events.ndjson");
const observerLog = path.join(privateDir, "observer.log");
const observerPidFile = path.join(privateDir, "observer.pid.json");
const recorderOutputDir = recorderRelativePath(privateDir);
const startup: StartupSession = {
attempt,
lane,
@@ -859,17 +965,14 @@ async function startLane(values: Map<string, string>, roots: Roots): Promise<voi
saveStartup(roots.sessionRoot, startup);
},
}),
runCommand(requiredEnv("OPENCLAW_TELEGRAM_DESKTOP_RECORDER_CMD"), [
"start",
"--provider",
"docker",
"--output-dir",
recorderOutputDir,
"--chat",
credential.groupId,
"--user-driver",
requiredEnv("OPENCLAW_TELEGRAM_USER_DRIVER_CMD"),
]),
startDesktopRecorder({
chat: credential.groupId,
outputDir: privateDir,
recorderCommand: requiredEnv("OPENCLAW_TELEGRAM_DESKTOP_RECORDER_CMD"),
sessionPath: recorderSession,
sessionRoot: roots.sessionRoot,
userDriver: requiredEnv("OPENCLAW_TELEGRAM_USER_DRIVER_CMD"),
}),
]);
if (sutResult.status === "fulfilled") {
sut = sutResult.value;