mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 05:16:23 -06:00
perf(e2e): reuse one authorized Telegram Desktop session per Mantis run (#127835)
* refactor(mantis): reuse authorized desktop captures * fix(mantis): budget desktop authorization failures * chore(mantis): bound desktop proof retries * fix(e2e): drop unused recorder failure fact type export * fix(ci): route Mantis desktop teardown through the recorder wrapper Cleanup invoked the internal recorder executable as mantis-sut, which is deliberately kept out of the docker group and cannot read the recorder-owned session file; teardown therefore failed and blocked safe_to_release. The cleanup step already runs as the recorder user, so call the public wrapper whose exec shim cds into the session root. * fix(e2e): make recorder failure fact lane-readable; document v2 lifecycle The Mantis workflow runs the recorder as the desktop user while the lane reads the authorization-failure fact as mantis-sut; 0600 made that read fail EACCES and silently disabled the two-attempt retry budget. Write the fact 0644 — the 0770 attempt directory bounds visibility. Update the mantis doc's recorder section for the v2 session lifecycle: required --session handle with healthy-session reuse, capture-only stop, and teardown owning authorization termination and lease release.
This commit is contained in:
@@ -4,16 +4,33 @@ import { z } from "zod";
|
||||
export const TELEGRAM_DESKTOP_VERSION = "7.0.9";
|
||||
export const TELEGRAM_DESKTOP_AWS_IMAGE = "telegram-desktop=7.0.9";
|
||||
export const TELEGRAM_DESKTOP_DOCKER_IMAGE = "openclaw-telegram-desktop:7.0.9";
|
||||
export const RECORDER_AUTHORIZATION_FAILURE_FILENAME =
|
||||
"telegram-desktop-authorization-failure.json";
|
||||
|
||||
export const recorderAuthorizationFailureSchema = z.object({
|
||||
acceptedTokenCount: z.number().int().nonnegative(),
|
||||
classification: z.enum(["main-window-timeout", "qr-unreadable", "token-accepted-no-transition"]),
|
||||
failedAt: z.string(),
|
||||
loginScreenshotPath: z.string().optional(),
|
||||
qrAttemptCount: z.number().int().positive(),
|
||||
});
|
||||
|
||||
export const recorderAuthorizationFailureFactSchema = z.object({
|
||||
failures: z.array(recorderAuthorizationFailureSchema).min(1),
|
||||
schemaVersion: z.literal(1),
|
||||
});
|
||||
|
||||
export type RecorderAuthorizationFailure = z.infer<typeof recorderAuthorizationFailureSchema>;
|
||||
|
||||
const recorderSessionBaseSchema = z.object({
|
||||
artifacts: z.record(z.string(), z.string()).optional(),
|
||||
chat: z.string().regex(/^-100\d+$/u),
|
||||
cleanupErrors: z.array(z.string()).optional(),
|
||||
desktopSessionId: z.string().min(1),
|
||||
keepBox: z.boolean(),
|
||||
leaseId: z.string().min(1),
|
||||
/** False when `--lease-id` borrowed an existing box: the recorder never stops it. */
|
||||
leaseOwned: z.boolean(),
|
||||
outputDir: z.string().min(1),
|
||||
recordFps: z.number().int().positive(),
|
||||
remotePaths: z.object({
|
||||
desktopLog: z.string(),
|
||||
@@ -22,7 +39,7 @@ const recorderSessionBaseSchema = z.object({
|
||||
finalScreenshot: z.string(),
|
||||
video: z.string(),
|
||||
}),
|
||||
schemaVersion: z.literal(1),
|
||||
schemaVersion: z.literal(2),
|
||||
startedAt: z.string(),
|
||||
/** Telegram Desktop window as placed on the recorded desktop; the crop uses it. */
|
||||
window: z.object({
|
||||
@@ -61,6 +78,7 @@ export type StartOptions = {
|
||||
outputDir: string;
|
||||
provider: RecorderProvider;
|
||||
recordFps: number;
|
||||
sessionPath: string;
|
||||
ttl: string;
|
||||
userDriver: string[];
|
||||
};
|
||||
@@ -87,7 +105,11 @@ export type ScreenshotOptions = {
|
||||
export type StopOptions = {
|
||||
command: "stop";
|
||||
crop?: "telegram-window";
|
||||
keepBox: boolean;
|
||||
sessionPath: string;
|
||||
};
|
||||
|
||||
export type TeardownOptions = {
|
||||
command: "teardown";
|
||||
sessionPath: string;
|
||||
};
|
||||
|
||||
@@ -114,18 +136,20 @@ type RecorderOptions =
|
||||
| StartOptions
|
||||
| StatusOptions
|
||||
| StopOptions
|
||||
| TeardownOptions
|
||||
| ViewOptions;
|
||||
|
||||
export function recorderUsageText(): string {
|
||||
return [
|
||||
"Usage:",
|
||||
" pnpm qa:telegram-desktop-recorder artifacts --session <recorder.json>",
|
||||
' pnpm qa:telegram-desktop-recorder start --output-dir <dir> --chat <-100groupId> --user-driver "<space-separated cmd prefix>" [options]',
|
||||
' pnpm qa:telegram-desktop-recorder start --session <recorder.json> --output-dir <dir> --chat <-100groupId> --user-driver "<space-separated cmd prefix>" [options]',
|
||||
" pnpm qa:telegram-desktop-recorder view --session <recorder.json> --message-id <id>",
|
||||
" pnpm qa:telegram-desktop-recorder actions --session <recorder.json> --actions-file <json> [--timeout-seconds <seconds>]",
|
||||
" pnpm qa:telegram-desktop-recorder screenshot --session <recorder.json> [--output <png>]",
|
||||
" pnpm qa:telegram-desktop-recorder recover --session <recorder.json>",
|
||||
" pnpm qa:telegram-desktop-recorder stop --session <recorder.json> [--crop telegram-window] [--keep-box]",
|
||||
" pnpm qa:telegram-desktop-recorder stop --session <recorder.json> [--crop telegram-window]",
|
||||
" pnpm qa:telegram-desktop-recorder teardown --session <recorder.json>",
|
||||
" pnpm qa:telegram-desktop-recorder status --session <recorder.json>",
|
||||
"",
|
||||
"Start options:",
|
||||
@@ -173,7 +197,17 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
|
||||
throw new Error(recorderUsageText());
|
||||
}
|
||||
const parsedCommand = z
|
||||
.enum(["actions", "artifacts", "recover", "screenshot", "start", "status", "stop", "view"])
|
||||
.enum([
|
||||
"actions",
|
||||
"artifacts",
|
||||
"recover",
|
||||
"screenshot",
|
||||
"start",
|
||||
"status",
|
||||
"stop",
|
||||
"teardown",
|
||||
"view",
|
||||
])
|
||||
.safeParse(rawCommand);
|
||||
if (!parsedCommand.success) {
|
||||
throw new Error(`Unknown command: ${rawCommand}\n\n${recorderUsageText()}`);
|
||||
@@ -186,7 +220,7 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
|
||||
if (!flag) {
|
||||
break;
|
||||
}
|
||||
if (flag === "--json" || flag === "--keep-box") {
|
||||
if (flag === "--json") {
|
||||
switches.add(flag);
|
||||
continue;
|
||||
}
|
||||
@@ -210,6 +244,7 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
|
||||
"--output-dir",
|
||||
"--provider",
|
||||
"--record-fps",
|
||||
"--session",
|
||||
"--ttl",
|
||||
"--user-driver",
|
||||
])
|
||||
@@ -230,9 +265,6 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
|
||||
if (switches.has("--json") && command !== "start") {
|
||||
throw new Error(`--json is not available for ${command}.`);
|
||||
}
|
||||
if (switches.has("--keep-box") && command !== "stop") {
|
||||
throw new Error(`--keep-box is not available for ${command}.`);
|
||||
}
|
||||
if (command === "start") {
|
||||
const chat = requiredString(values, "--chat");
|
||||
if (!/^-100\d+$/u.test(chat)) {
|
||||
@@ -268,6 +300,7 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
|
||||
outputDir: requiredString(values, "--output-dir"),
|
||||
provider,
|
||||
recordFps: positiveInteger(values.get("--record-fps") ?? "24", "--record-fps"),
|
||||
sessionPath: requiredString(values, "--session"),
|
||||
ttl: values.get("--ttl") ?? "2h",
|
||||
userDriver,
|
||||
};
|
||||
@@ -292,12 +325,12 @@ export function parseRecorderArgs(argv: string[]): RecorderOptions {
|
||||
if (command === "stop") {
|
||||
const crop = values.get("--crop");
|
||||
if (crop === undefined) {
|
||||
return { command, keepBox: switches.has("--keep-box"), sessionPath };
|
||||
return { command, sessionPath };
|
||||
}
|
||||
if (crop !== "telegram-window") {
|
||||
throw new Error("--crop must be telegram-window.");
|
||||
}
|
||||
return { command, crop, keepBox: switches.has("--keep-box"), sessionPath };
|
||||
return { command, crop, sessionPath };
|
||||
}
|
||||
return { command, sessionPath };
|
||||
}
|
||||
|
||||
@@ -26,6 +26,9 @@ import {
|
||||
import {
|
||||
parseRecorderArgs,
|
||||
readRecorderSession,
|
||||
RECORDER_AUTHORIZATION_FAILURE_FILENAME,
|
||||
recorderAuthorizationFailureFactSchema,
|
||||
type RecorderAuthorizationFailure,
|
||||
type ActionsOptions,
|
||||
type ArtifactsOptions,
|
||||
type RecoverOptions,
|
||||
@@ -39,6 +42,7 @@ import {
|
||||
type StartOptions,
|
||||
type StatusOptions,
|
||||
type StopOptions,
|
||||
type TeardownOptions,
|
||||
type ViewOptions,
|
||||
writeRecorderSession,
|
||||
} from "./telegram-desktop-recorder-contract.ts";
|
||||
@@ -90,7 +94,14 @@ const confirmedQrSchema = z.object({
|
||||
}),
|
||||
});
|
||||
|
||||
class FreshDesktopRequiredError extends Error {}
|
||||
class DesktopAuthorizationError extends Error {
|
||||
readonly failure: RecorderAuthorizationFailure;
|
||||
|
||||
constructor(message: string, failure: RecorderAuthorizationFailure, options?: ErrorOptions) {
|
||||
super(`${failure.classification}: ${message}`, options);
|
||||
this.failure = failure;
|
||||
}
|
||||
}
|
||||
|
||||
const recorderStartupSchema = z.object({
|
||||
desktopSessionId: z.string().min(1).optional(),
|
||||
@@ -331,7 +342,9 @@ async function authorizeDesktop(params: {
|
||||
let lastFailure: unknown;
|
||||
let lastLink = "";
|
||||
let acceptedWithoutTransition = 0;
|
||||
let qrAttemptCount = 0;
|
||||
for (let attempt = 1; attempt <= 6; attempt += 1) {
|
||||
qrAttemptCount = attempt;
|
||||
let link: string;
|
||||
try {
|
||||
const qr = await params.operations.sshRun({
|
||||
@@ -390,6 +403,7 @@ async function authorizeDesktop(params: {
|
||||
// an absent screenshot.
|
||||
const evidencePath = path.join(params.outputDir, "telegram-login-screen.png");
|
||||
let evidence: string;
|
||||
let loginScreenshotPath: string | undefined;
|
||||
try {
|
||||
await params.operations.scpFromRemote({
|
||||
cwd: params.cwd,
|
||||
@@ -398,18 +412,34 @@ async function authorizeDesktop(params: {
|
||||
remote: `${REMOTE_ROOT}/telegram-login-qr.png`,
|
||||
run: params.operations.runCommand,
|
||||
});
|
||||
loginScreenshotPath = evidencePath;
|
||||
evidence = ` Login screen: ${evidencePath}`;
|
||||
} catch (error) {
|
||||
evidence = ` Login screen could not be fetched: ${coerceErrorMessage(error)}`;
|
||||
}
|
||||
const message =
|
||||
const classification =
|
||||
acceptedWithoutTransition >= 2
|
||||
? "token-accepted-no-transition"
|
||||
: acceptedWithoutTransition === 1
|
||||
? "main-window-timeout"
|
||||
: "qr-unreadable";
|
||||
const message =
|
||||
classification === "token-accepted-no-transition"
|
||||
? `Telegram server accepted ${acceptedWithoutTransition} login tokens, but Telegram Desktop stayed on the QR screen${detail}.${evidence}`
|
||||
: `Telegram Desktop did not leave the login screen after 6 attempts${detail}.${evidence}`;
|
||||
if (acceptedWithoutTransition >= 2) {
|
||||
throw new FreshDesktopRequiredError(message, { cause: lastFailure });
|
||||
}
|
||||
throw new Error(message, { cause: lastFailure });
|
||||
: classification === "main-window-timeout"
|
||||
? `Telegram Desktop did not reach the main window after an accepted login token${detail}.${evidence}`
|
||||
: `Telegram Desktop did not leave the login screen after 6 attempts${detail}.${evidence}`;
|
||||
throw new DesktopAuthorizationError(
|
||||
message,
|
||||
{
|
||||
acceptedTokenCount: acceptedWithoutTransition,
|
||||
classification,
|
||||
failedAt: new Date().toISOString(),
|
||||
loginScreenshotPath,
|
||||
qrAttemptCount,
|
||||
},
|
||||
{ cause: lastFailure },
|
||||
);
|
||||
}
|
||||
|
||||
// The recorder runs as a different user than the agent that drives it, so an output dir
|
||||
@@ -448,6 +478,27 @@ function resolveOutputDir(cwd: string, outputDir: string): string {
|
||||
return resolveRecorderPath(cwd, outputDir, "--output-dir");
|
||||
}
|
||||
|
||||
function appendAuthorizationFailure(
|
||||
outputDir: string,
|
||||
failure: RecorderAuthorizationFailure,
|
||||
): void {
|
||||
const file = path.join(outputDir, RECORDER_AUTHORIZATION_FAILURE_FILENAME);
|
||||
const current = fs.existsSync(file)
|
||||
? recorderAuthorizationFailureFactSchema.parse(JSON.parse(fs.readFileSync(file, "utf8")))
|
||||
: { failures: [], schemaVersion: 1 as const };
|
||||
const fact = recorderAuthorizationFailureFactSchema.parse({
|
||||
failures: [...current.failures, failure],
|
||||
schemaVersion: 1,
|
||||
});
|
||||
const temporary = `${file}.${process.pid}.tmp`;
|
||||
// 0644: cross-identity evidence — the Mantis workflow runs the recorder as the
|
||||
// desktop user while the lane reads this fact as mantis-sut to enforce its
|
||||
// retry budget; the 0770 attempt directory bounds visibility. 0600 would make
|
||||
// the lane's read fail EACCES and silently disable the budget.
|
||||
fs.writeFileSync(temporary, `${JSON.stringify(fact, null, 2)}\n`, { mode: 0o644 });
|
||||
fs.renameSync(temporary, file);
|
||||
}
|
||||
|
||||
async function stopBox(params: {
|
||||
crabboxBin: string;
|
||||
cwd: string;
|
||||
@@ -489,6 +540,40 @@ async function terminateDesktopSessions(params: {
|
||||
z.object({ ok: z.literal(true) }).parse(JSON.parse(result.stdout));
|
||||
}
|
||||
|
||||
async function destroyRecorderSessionResources(
|
||||
cwd: string,
|
||||
session: RecorderSession,
|
||||
operations: Pick<RecorderOperations, "runCommand">,
|
||||
): Promise<string[]> {
|
||||
const errors: string[] = [];
|
||||
try {
|
||||
await terminateDesktopSession({
|
||||
cwd,
|
||||
desktopSessionId: session.desktopSessionId,
|
||||
run: operations.runCommand,
|
||||
userDriver: session.userDriver,
|
||||
});
|
||||
} catch (error) {
|
||||
errors.push(`terminate Telegram Desktop session: ${coerceErrorMessage(error)}`);
|
||||
}
|
||||
if (session.leaseOwned) {
|
||||
try {
|
||||
await stopBox({
|
||||
crabboxBin: process.env.OPENCLAW_TELEGRAM_USER_CRABBOX_BIN?.trim() || "crabbox",
|
||||
cwd,
|
||||
leaseId: session.leaseId,
|
||||
provider: session.provider,
|
||||
run: operations.runCommand,
|
||||
});
|
||||
} catch (error) {
|
||||
if (!coerceErrorMessage(error).includes("lease not found")) {
|
||||
errors.push(`stop Crabbox: ${coerceErrorMessage(error)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
async function assertLocalTelegramImage(params: { cwd: string; run: RunCommand }): Promise<void> {
|
||||
try {
|
||||
await params.run({
|
||||
@@ -507,17 +592,109 @@ async function assertLocalTelegramImage(params: { cwd: string; run: RunCommand }
|
||||
}
|
||||
}
|
||||
|
||||
async function startRecorderAttempt(
|
||||
type RecorderResource = Pick<
|
||||
RecorderSession,
|
||||
"desktopSessionId" | "leaseId" | "leaseOwned" | "provider" | "userDriver"
|
||||
>;
|
||||
|
||||
async function beginRecorderCapture(params: {
|
||||
cwd: string;
|
||||
inspect: CrabboxInspect;
|
||||
operations: RecorderOperations;
|
||||
opts: StartOptions;
|
||||
outputDir: string;
|
||||
resource: RecorderResource;
|
||||
sessionPath: string;
|
||||
}): Promise<{ session: RecorderSession; sessionPath: string }> {
|
||||
await params.operations.sshRun({
|
||||
command: renderTelegramViewCommand({
|
||||
binary: TELEGRAM_BINARY,
|
||||
link: telegramPrivatePostLink(params.opts.chat, params.opts.messageId),
|
||||
workdir: TELEGRAM_WORKDIR,
|
||||
}),
|
||||
cwd: params.cwd,
|
||||
inspect: params.inspect,
|
||||
run: params.operations.runCommand,
|
||||
});
|
||||
const geometry = await params.operations.sshRun({
|
||||
command: renderReadWindowGeometry(),
|
||||
cwd: params.cwd,
|
||||
inspect: params.inspect,
|
||||
run: params.operations.runCommand,
|
||||
stdio: "pipe",
|
||||
});
|
||||
const windowGeometry = parseWindowGeometry(geometry.stdout);
|
||||
await params.operations.sshRun({
|
||||
command: renderHideTelegramWindow(windowGeometry.id),
|
||||
cwd: params.cwd,
|
||||
inspect: params.inspect,
|
||||
run: params.operations.runCommand,
|
||||
});
|
||||
const sessionBase = {
|
||||
chat: params.opts.chat,
|
||||
desktopSessionId: params.resource.desktopSessionId,
|
||||
leaseId: params.resource.leaseId,
|
||||
leaseOwned: params.resource.leaseOwned,
|
||||
outputDir: params.outputDir,
|
||||
recordFps: params.opts.recordFps,
|
||||
remotePaths,
|
||||
schemaVersion: 2 as const,
|
||||
startedAt: new Date().toISOString(),
|
||||
userDriver: params.resource.userDriver,
|
||||
window: windowGeometry,
|
||||
};
|
||||
const session: RecorderSession =
|
||||
params.resource.provider === "docker"
|
||||
? {
|
||||
...sessionBase,
|
||||
imageSource: TELEGRAM_DESKTOP_DOCKER_IMAGE,
|
||||
provider: "docker",
|
||||
}
|
||||
: {
|
||||
...sessionBase,
|
||||
imageSource: TELEGRAM_DESKTOP_AWS_IMAGE,
|
||||
provider: "aws",
|
||||
};
|
||||
// Publish the current capture destination before ffmpeg starts so crash recovery
|
||||
// exports or stops only this attempt, never an earlier attempt's artifacts.
|
||||
writeRecorderSession(params.sessionPath, session);
|
||||
await params.operations.sshRun({
|
||||
command: renderStartRemoteRecording({ paths: remotePaths, recordFps: params.opts.recordFps }),
|
||||
cwd: params.cwd,
|
||||
inspect: params.inspect,
|
||||
run: params.operations.runCommand,
|
||||
});
|
||||
return { session, sessionPath: params.sessionPath };
|
||||
}
|
||||
|
||||
async function inspectHealthySession(params: {
|
||||
crabboxBin: string;
|
||||
cwd: string;
|
||||
operations: RecorderOperations;
|
||||
session: RecorderSession;
|
||||
}): Promise<CrabboxInspect | undefined> {
|
||||
try {
|
||||
const inspect = await sessionInspect(params);
|
||||
const mainWindow = await desktopReachedMainWindow({
|
||||
cwd: params.cwd,
|
||||
inspect,
|
||||
operations: params.operations,
|
||||
seconds: 5,
|
||||
});
|
||||
return mainWindow.reached ? inspect : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async function provisionRecorder(
|
||||
cwd: string,
|
||||
opts: StartOptions,
|
||||
operations: RecorderOperations,
|
||||
freshContainerAttempt: number,
|
||||
outputDir: string,
|
||||
sessionPath: string,
|
||||
): Promise<{ session: RecorderSession; sessionPath: string }> {
|
||||
const crabboxBin = process.env.OPENCLAW_TELEGRAM_USER_CRABBOX_BIN?.trim() || "crabbox";
|
||||
const outputDir = resolveOutputDir(cwd, opts.outputDir);
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
assertOutputDirWritable(outputDir);
|
||||
const sessionPath = path.join(outputDir, "recorder.json");
|
||||
const startupPath = recorderStartupPath(sessionPath);
|
||||
let leaseId = opts.leaseId;
|
||||
const leaseOwned = !opts.leaseId;
|
||||
@@ -532,7 +709,7 @@ async function startRecorderAttempt(
|
||||
};
|
||||
// Provisioning crosses process and provider boundaries. Persist each acquired
|
||||
// handle so a later workflow step can reclaim it after cancellation or SIGKILL.
|
||||
writeRecorderStartup(startupPath, startup, freshContainerAttempt === 1);
|
||||
writeRecorderStartup(startupPath, startup, true);
|
||||
try {
|
||||
if (!leaseId) {
|
||||
if (opts.provider === "docker") {
|
||||
@@ -593,71 +770,27 @@ async function startRecorderAttempt(
|
||||
outputDir,
|
||||
userDriver: opts.userDriver,
|
||||
});
|
||||
// Always open the target chat before recording: at the recorder's width Telegram shows
|
||||
// either the chat list or one conversation, and the list is the QA account's own.
|
||||
await operations.sshRun({
|
||||
command: renderTelegramViewCommand({
|
||||
binary: TELEGRAM_BINARY,
|
||||
link: telegramPrivatePostLink(opts.chat, opts.messageId),
|
||||
workdir: TELEGRAM_WORKDIR,
|
||||
}),
|
||||
const result = await beginRecorderCapture({
|
||||
cwd,
|
||||
inspect,
|
||||
run: operations.runCommand,
|
||||
operations,
|
||||
opts,
|
||||
outputDir,
|
||||
resource: {
|
||||
desktopSessionId,
|
||||
leaseId,
|
||||
leaseOwned,
|
||||
provider: opts.provider,
|
||||
userDriver: opts.userDriver,
|
||||
},
|
||||
sessionPath,
|
||||
});
|
||||
// Crop from the window Telegram actually got: window managers and providers
|
||||
// place it differently, and a fixed crop silently cuts the chat pane.
|
||||
const geometry = await operations.sshRun({
|
||||
command: renderReadWindowGeometry(),
|
||||
cwd,
|
||||
inspect,
|
||||
run: operations.runCommand,
|
||||
stdio: "pipe",
|
||||
});
|
||||
const windowGeometry = parseWindowGeometry(geometry.stdout);
|
||||
// The lane clears prior history before recorder start. Keep the empty chat hidden until
|
||||
// the first session-owned send is ready, so setup frames reveal neither account UI nor chat.
|
||||
await operations.sshRun({
|
||||
command: renderHideTelegramWindow(windowGeometry.id),
|
||||
cwd,
|
||||
inspect,
|
||||
run: operations.runCommand,
|
||||
});
|
||||
await operations.sshRun({
|
||||
command: renderStartRemoteRecording({ paths: remotePaths, recordFps: opts.recordFps }),
|
||||
cwd,
|
||||
inspect,
|
||||
run: operations.runCommand,
|
||||
});
|
||||
const sessionBase: Omit<RecorderSession, "imageSource" | "provider"> = {
|
||||
chat: opts.chat,
|
||||
desktopSessionId,
|
||||
keepBox: false,
|
||||
leaseId,
|
||||
leaseOwned,
|
||||
recordFps: opts.recordFps,
|
||||
remotePaths,
|
||||
schemaVersion: 1,
|
||||
window: windowGeometry,
|
||||
startedAt: new Date().toISOString(),
|
||||
userDriver: opts.userDriver,
|
||||
};
|
||||
const session: RecorderSession =
|
||||
opts.provider === "docker"
|
||||
? {
|
||||
...sessionBase,
|
||||
imageSource: TELEGRAM_DESKTOP_DOCKER_IMAGE,
|
||||
provider: opts.provider,
|
||||
}
|
||||
: {
|
||||
...sessionBase,
|
||||
imageSource: TELEGRAM_DESKTOP_AWS_IMAGE,
|
||||
provider: opts.provider,
|
||||
};
|
||||
writeRecorderSession(sessionPath, session);
|
||||
fs.rmSync(startupPath);
|
||||
return { session, sessionPath };
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (error instanceof DesktopAuthorizationError) {
|
||||
appendAuthorizationFailure(outputDir, error.failure);
|
||||
}
|
||||
const cleanupErrors: string[] = [];
|
||||
if (desktopAuthorizationRequested) {
|
||||
try {
|
||||
@@ -688,17 +821,9 @@ async function startRecorderAttempt(
|
||||
cleanupErrors.push(coerceErrorMessage(cleanupError));
|
||||
}
|
||||
}
|
||||
if (
|
||||
error instanceof FreshDesktopRequiredError &&
|
||||
cleanupErrors.length === 0 &&
|
||||
leaseOwned &&
|
||||
opts.provider === "docker" &&
|
||||
freshContainerAttempt === 1
|
||||
) {
|
||||
return await startRecorderAttempt(cwd, opts, operations, freshContainerAttempt + 1);
|
||||
}
|
||||
if (cleanupErrors.length === 0) {
|
||||
fs.rmSync(startupPath, { force: true });
|
||||
fs.rmSync(sessionPath, { force: true });
|
||||
}
|
||||
const suffix = cleanupErrors.length ? ` Cleanup also failed: ${cleanupErrors.join("; ")}` : "";
|
||||
throw new Error(`${coerceErrorMessage(error)}${suffix}`, { cause: error });
|
||||
@@ -710,7 +835,40 @@ export async function startRecorder(
|
||||
opts: StartOptions,
|
||||
operations: RecorderOperations = defaultOperations,
|
||||
): Promise<{ session: RecorderSession; sessionPath: string }> {
|
||||
return await startRecorderAttempt(cwd, opts, operations, 1);
|
||||
const outputDir = resolveOutputDir(cwd, opts.outputDir);
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
assertOutputDirWritable(outputDir);
|
||||
const sessionPath = resolveRecorderPath(cwd, opts.sessionPath, "--session");
|
||||
if (!fs.existsSync(sessionPath)) {
|
||||
return await provisionRecorder(cwd, opts, operations, outputDir, sessionPath);
|
||||
}
|
||||
const session = readRecorderSession(sessionPath);
|
||||
if (
|
||||
session.chat !== opts.chat ||
|
||||
session.provider !== opts.provider ||
|
||||
session.userDriver.join("\0") !== opts.userDriver.join("\0")
|
||||
) {
|
||||
throw new Error("Existing recorder session does not match this start request.");
|
||||
}
|
||||
const crabboxBin = process.env.OPENCLAW_TELEGRAM_USER_CRABBOX_BIN?.trim() || "crabbox";
|
||||
const inspect = await inspectHealthySession({ crabboxBin, cwd, operations, session });
|
||||
if (inspect) {
|
||||
return await beginRecorderCapture({
|
||||
cwd,
|
||||
inspect,
|
||||
operations,
|
||||
opts,
|
||||
outputDir,
|
||||
resource: session,
|
||||
sessionPath,
|
||||
});
|
||||
}
|
||||
const cleanupErrors = await destroyRecorderSessionResources(cwd, session, operations);
|
||||
if (cleanupErrors.length) {
|
||||
throw new Error(`Unhealthy recorder session cleanup failed:\n${cleanupErrors.join("\n")}`);
|
||||
}
|
||||
fs.rmSync(sessionPath);
|
||||
return await provisionRecorder(cwd, opts, operations, outputDir, sessionPath);
|
||||
}
|
||||
|
||||
export async function recoverRecorderStartup(
|
||||
@@ -767,8 +925,8 @@ export function recorderArtifacts(
|
||||
opts: ArtifactsOptions,
|
||||
): { artifacts: Record<string, string> } {
|
||||
const sessionPath = resolveRecorderPath(cwd, opts.sessionPath, "--session");
|
||||
const outputDir = path.dirname(sessionPath);
|
||||
const session = readRecorderSession(sessionPath);
|
||||
const outputDir = session.outputDir;
|
||||
const artifacts: Record<string, string> = {};
|
||||
for (const [name, file] of Object.entries(session.artifacts ?? {})) {
|
||||
const resolved = path.resolve(file);
|
||||
@@ -942,7 +1100,7 @@ export async function screenshotRecorder(
|
||||
const output =
|
||||
opts.output ??
|
||||
path.join(
|
||||
path.dirname(opts.sessionPath),
|
||||
path.relative(cwd, session.outputDir),
|
||||
`telegram-desktop-recorder-screenshot-${new Date().toISOString().replace(/[:.]/gu, "-")}.png`,
|
||||
);
|
||||
const outputPath = resolveRecorderPath(cwd, output, "--output");
|
||||
@@ -967,7 +1125,7 @@ export async function stopRecorder(
|
||||
const sessionPath = resolveRecorderPath(cwd, opts.sessionPath, "--session");
|
||||
const session = readRecorderSession(sessionPath);
|
||||
const crabboxBin = process.env.OPENCLAW_TELEGRAM_USER_CRABBOX_BIN?.trim() || "crabbox";
|
||||
const outputDir = path.dirname(sessionPath);
|
||||
const outputDir = session.outputDir;
|
||||
const errors: string[] = [];
|
||||
const artifacts: Record<string, string> = {};
|
||||
const attempt = async (label: string, action: () => Promise<void>) => {
|
||||
@@ -978,14 +1136,12 @@ export async function stopRecorder(
|
||||
}
|
||||
};
|
||||
let inspect: CrabboxInspect | undefined;
|
||||
let leaseGone = false;
|
||||
await attempt("inspect", async () => {
|
||||
try {
|
||||
inspect = await sessionInspect({ crabboxBin, cwd, operations, session });
|
||||
} catch (error) {
|
||||
// A lease that no longer exists is the desired end state, not a failure.
|
||||
if (coerceErrorMessage(error).includes("lease not found")) {
|
||||
leaseGone = true;
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
@@ -1077,36 +1233,12 @@ export async function stopRecorder(
|
||||
});
|
||||
}
|
||||
}
|
||||
// --keep-box keeps the whole debugging surface: the Desktop authorization stays
|
||||
// valid for WebVNC until the operator finishes; a later `stop` without it revokes.
|
||||
if (!opts.keepBox) {
|
||||
await attempt("terminate Telegram Desktop session", async () => {
|
||||
await terminateDesktopSession({
|
||||
cwd,
|
||||
desktopSessionId: session.desktopSessionId,
|
||||
run: operations.runCommand,
|
||||
userDriver: session.userDriver,
|
||||
});
|
||||
});
|
||||
}
|
||||
if (!opts.keepBox && session.leaseOwned && !leaseGone) {
|
||||
await attempt("stop Crabbox", async () => {
|
||||
await stopBox({
|
||||
crabboxBin,
|
||||
cwd,
|
||||
leaseId: session.leaseId,
|
||||
provider: session.provider,
|
||||
run: operations.runCommand,
|
||||
});
|
||||
});
|
||||
}
|
||||
const stopped: RecorderSession = {
|
||||
...session,
|
||||
// Keep paths recorded by an earlier stop (--keep-box, then a later stop once
|
||||
// the lease expired); fresh copies overwrite their own entries.
|
||||
// Keep paths recorded by an earlier stop if a later cleanup sees an expired lease;
|
||||
// fresh copies overwrite their own entries.
|
||||
artifacts: { ...session.artifacts, ...artifacts },
|
||||
cleanupErrors: errors.length ? errors : undefined,
|
||||
keepBox: opts.keepBox,
|
||||
stoppedAt: new Date().toISOString(),
|
||||
};
|
||||
writeRecorderSession(sessionPath, stopped);
|
||||
@@ -1116,6 +1248,35 @@ export async function stopRecorder(
|
||||
return stopped;
|
||||
}
|
||||
|
||||
export async function teardownRecorder(
|
||||
cwd: string,
|
||||
opts: TeardownOptions,
|
||||
operations: Pick<RecorderOperations, "runCommand"> = defaultOperations,
|
||||
): Promise<{ tornDown: boolean }> {
|
||||
const sessionPath = resolveRecorderPath(cwd, opts.sessionPath, "--session");
|
||||
const startupPath = recorderStartupPath(sessionPath);
|
||||
if (fs.existsSync(startupPath)) {
|
||||
await recoverRecorderStartup(
|
||||
cwd,
|
||||
{ command: "recover", sessionPath: opts.sessionPath },
|
||||
operations,
|
||||
);
|
||||
fs.rmSync(sessionPath, { force: true });
|
||||
return { tornDown: true };
|
||||
}
|
||||
if (!fs.existsSync(sessionPath)) {
|
||||
return { tornDown: false };
|
||||
}
|
||||
const session = readRecorderSession(sessionPath);
|
||||
const errors = await destroyRecorderSessionResources(cwd, session, operations);
|
||||
if (errors.length) {
|
||||
writeRecorderSession(sessionPath, { ...session, cleanupErrors: errors });
|
||||
throw new Error(`Recorder teardown completed with errors:\n${errors.join("\n")}`);
|
||||
}
|
||||
fs.rmSync(sessionPath);
|
||||
return { tornDown: true };
|
||||
}
|
||||
|
||||
async function statusRecorder(
|
||||
cwd: string,
|
||||
opts: StatusOptions,
|
||||
@@ -1173,6 +1334,10 @@ async function main(): Promise<void> {
|
||||
console.log(JSON.stringify(await stopRecorder(cwd, opts), null, 2));
|
||||
return;
|
||||
}
|
||||
if (opts.command === "teardown") {
|
||||
console.log(JSON.stringify(await teardownRecorder(cwd, opts), null, 2));
|
||||
return;
|
||||
}
|
||||
console.log(JSON.stringify(await statusRecorder(cwd, opts, defaultOperations), null, 2));
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,12 @@ import { z } from "zod";
|
||||
import { coerceErrorMessage } from "../lib/error-format.mts";
|
||||
import { sleep } from "../lib/sleep.mjs";
|
||||
import { telegramBotApi } from "./telegram-bot-api.ts";
|
||||
import {
|
||||
RECORDER_AUTHORIZATION_FAILURE_FILENAME,
|
||||
recorderAuthorizationFailureSchema,
|
||||
recorderAuthorizationFailureFactSchema,
|
||||
type RecorderAuthorizationFailure,
|
||||
} from "./telegram-desktop-recorder-contract.ts";
|
||||
import {
|
||||
destroyMantisSut,
|
||||
type MantisSutRecovery,
|
||||
@@ -119,6 +125,13 @@ const invocationSchema = z.object({
|
||||
const recorderArtifactsSchema = z.object({
|
||||
artifacts: z.record(z.string(), z.string()),
|
||||
});
|
||||
const desktopRecorderFailureBudgetSchema = z.object({
|
||||
attemptCount: z.number().int().positive(),
|
||||
classification: recorderAuthorizationFailureSchema.shape.classification,
|
||||
loginScreenshotPath: z.string().optional(),
|
||||
schemaVersion: z.literal(1),
|
||||
unavailable: z.boolean(),
|
||||
});
|
||||
const activeSessionSchema = z.object({
|
||||
attempt: z.number().int().positive(),
|
||||
config: configSchema,
|
||||
@@ -151,6 +164,7 @@ type ObserverResponse = {
|
||||
ok: boolean;
|
||||
truncated?: boolean;
|
||||
} & Record<string, unknown>;
|
||||
type DesktopRecorderFailureBudget = z.infer<typeof desktopRecorderFailureBudgetSchema>;
|
||||
|
||||
const MAX_SENDS = 12;
|
||||
const MAX_RPC_BYTES = 4 * 1024 * 1024;
|
||||
@@ -315,6 +329,98 @@ function writeJsonAtomic(file: string, value: unknown, mode = 0o600): void {
|
||||
fs.chmodSync(file, mode);
|
||||
}
|
||||
|
||||
function desktopRecorderFailureBudgetFile(sessionRoot: string): string {
|
||||
return path.join(sessionRoot, "desktop-recorder-failures.json");
|
||||
}
|
||||
|
||||
function readDesktopRecorderFailureBudget(
|
||||
sessionRoot: string,
|
||||
): DesktopRecorderFailureBudget | undefined {
|
||||
const file = desktopRecorderFailureBudgetFile(sessionRoot);
|
||||
return fs.existsSync(file) ? desktopRecorderFailureBudgetSchema.parse(readJson(file)) : undefined;
|
||||
}
|
||||
|
||||
function desktopUnavailableMessage(fact: DesktopRecorderFailureBudget, factFile: string): string {
|
||||
const screenshotDetail = fact.loginScreenshotPath
|
||||
? `, loginScreenshotPath=${fact.loginScreenshotPath}`
|
||||
: "";
|
||||
return (
|
||||
`desktop-unavailable: stop retrying; this run's desktop is unavailable ` +
|
||||
`(attemptCount=${fact.attemptCount}, classification=${fact.classification}${screenshotDetail}, ` +
|
||||
`fact=${factFile})`
|
||||
);
|
||||
}
|
||||
|
||||
function assertDesktopRecorderAvailable(sessionRoot: string): void {
|
||||
const fact = readDesktopRecorderFailureBudget(sessionRoot);
|
||||
if (fact?.unavailable) {
|
||||
throw new Error(desktopUnavailableMessage(fact, desktopRecorderFailureBudgetFile(sessionRoot)));
|
||||
}
|
||||
}
|
||||
|
||||
function recordDesktopRecorderFailures(
|
||||
sessionRoot: string,
|
||||
failures: RecorderAuthorizationFailure[],
|
||||
): DesktopRecorderFailureBudget | undefined {
|
||||
if (failures.length === 0) {
|
||||
return readDesktopRecorderFailureBudget(sessionRoot);
|
||||
}
|
||||
const prior = readDesktopRecorderFailureBudget(sessionRoot);
|
||||
const latest = failures.at(-1);
|
||||
if (!latest) {
|
||||
return prior;
|
||||
}
|
||||
const attemptCount = (prior?.attemptCount ?? 0) + failures.length;
|
||||
const fact = desktopRecorderFailureBudgetSchema.parse({
|
||||
attemptCount,
|
||||
classification: latest.classification,
|
||||
loginScreenshotPath: latest.loginScreenshotPath,
|
||||
schemaVersion: 1,
|
||||
unavailable: attemptCount >= 2,
|
||||
});
|
||||
writeJsonAtomic(desktopRecorderFailureBudgetFile(sessionRoot), fact);
|
||||
return fact;
|
||||
}
|
||||
|
||||
export async function startDesktopRecorder(params: {
|
||||
chat: string;
|
||||
outputDir: string;
|
||||
recorderCommand: string;
|
||||
sessionPath: string;
|
||||
sessionRoot: string;
|
||||
userDriver: string;
|
||||
}): Promise<void> {
|
||||
assertDesktopRecorderAvailable(params.sessionRoot);
|
||||
try {
|
||||
await runCommand(params.recorderCommand, [
|
||||
"start",
|
||||
"--provider",
|
||||
"docker",
|
||||
"--session",
|
||||
recorderRelativePath(params.sessionPath),
|
||||
"--output-dir",
|
||||
recorderRelativePath(params.outputDir),
|
||||
"--chat",
|
||||
params.chat,
|
||||
"--user-driver",
|
||||
params.userDriver,
|
||||
]);
|
||||
} catch (startError) {
|
||||
const failureFile = path.join(params.outputDir, RECORDER_AUTHORIZATION_FAILURE_FILENAME);
|
||||
const failures = fs.existsSync(failureFile)
|
||||
? recorderAuthorizationFailureFactSchema.parse(readJson(failureFile)).failures
|
||||
: [];
|
||||
const budget = recordDesktopRecorderFailures(params.sessionRoot, failures);
|
||||
if (budget?.unavailable) {
|
||||
throw new Error(
|
||||
`${desktopUnavailableMessage(budget, desktopRecorderFailureBudgetFile(params.sessionRoot))}\n${coerceErrorMessage(startError)}`,
|
||||
{ cause: startError },
|
||||
);
|
||||
}
|
||||
throw startError;
|
||||
}
|
||||
}
|
||||
|
||||
function publicRelativePath(root: string, file: string, label: string): string {
|
||||
const resolvedRoot = fs.realpathSync(root);
|
||||
const relative = path.relative(resolvedRoot, file);
|
||||
@@ -801,17 +907,17 @@ async function startLane(values: Map<string, string>, roots: Roots): Promise<voi
|
||||
) {
|
||||
throw new Error(`Finish or abort the active ${otherLane} session first.`);
|
||||
}
|
||||
assertDesktopRecorderAvailable(roots.sessionRoot);
|
||||
const attemptsRoot = path.join(roots.sessionRoot, "attempts", lane);
|
||||
fs.mkdirSync(attemptsRoot, { recursive: true });
|
||||
const attempt = fs.readdirSync(attemptsRoot).filter((entry) => /^\d+$/u.test(entry)).length + 1;
|
||||
const privateDir = path.join(attemptsRoot, String(attempt));
|
||||
fs.mkdirSync(privateDir, { mode: 0o770 });
|
||||
const recorderSession = path.join(privateDir, "recorder.json");
|
||||
const recorderSession = path.join(roots.sessionRoot, "desktop-recorder.json");
|
||||
const observerSocket = path.join(privateDir, "observer.sock");
|
||||
const observerJournal = path.join(privateDir, "telegram-events.ndjson");
|
||||
const observerLog = path.join(privateDir, "observer.log");
|
||||
const observerPidFile = path.join(privateDir, "observer.pid.json");
|
||||
const recorderOutputDir = recorderRelativePath(privateDir);
|
||||
const startup: StartupSession = {
|
||||
attempt,
|
||||
lane,
|
||||
@@ -859,17 +965,14 @@ async function startLane(values: Map<string, string>, roots: Roots): Promise<voi
|
||||
saveStartup(roots.sessionRoot, startup);
|
||||
},
|
||||
}),
|
||||
runCommand(requiredEnv("OPENCLAW_TELEGRAM_DESKTOP_RECORDER_CMD"), [
|
||||
"start",
|
||||
"--provider",
|
||||
"docker",
|
||||
"--output-dir",
|
||||
recorderOutputDir,
|
||||
"--chat",
|
||||
credential.groupId,
|
||||
"--user-driver",
|
||||
requiredEnv("OPENCLAW_TELEGRAM_USER_DRIVER_CMD"),
|
||||
]),
|
||||
startDesktopRecorder({
|
||||
chat: credential.groupId,
|
||||
outputDir: privateDir,
|
||||
recorderCommand: requiredEnv("OPENCLAW_TELEGRAM_DESKTOP_RECORDER_CMD"),
|
||||
sessionPath: recorderSession,
|
||||
sessionRoot: roots.sessionRoot,
|
||||
userDriver: requiredEnv("OPENCLAW_TELEGRAM_USER_DRIVER_CMD"),
|
||||
}),
|
||||
]);
|
||||
if (sutResult.status === "fulfilled") {
|
||||
sut = sutResult.value;
|
||||
|
||||
Reference in New Issue
Block a user