diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.approval-checkpoint.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.approval-checkpoint.ts new file mode 100644 index 000000000000..167955068153 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.approval-checkpoint.ts @@ -0,0 +1,201 @@ +// QA Lab Slack approval checkpoint and gateway decision RPC. +import fs from "node:fs/promises"; +import path from "node:path"; +import { parseStrictPositiveInteger } from "openclaw/plugin-sdk/number-runtime"; +import { + formatApprovalResultValue, + readAcceptedApprovalRequestId, +} from "../shared/live-approval-result.js"; +import { + SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS, + SLACK_QA_APPROVAL_CHECKPOINT_DEFAULT_TIMEOUT_MS, + type SlackQaScenarioId, + type SlackQaApprovalKind, + type SlackQaApprovalDecision, + type SlackQaApprovalScenarioRun, + type SlackQaScenarioContext, + type SlackApprovalCheckpointState, + type SlackApprovalCheckpointAck, + SLACK_QA_APPROVAL_CHECKPOINT_DIR_ENV, + SLACK_QA_APPROVAL_CHECKPOINT_TIMEOUT_MS_ENV, + type SlackMessage, +} from "./slack-live.contracts.js"; +import { buildSlackApprovalCheckpointMessage } from "./slack-live.observations.js"; + +export function resolveSlackApprovalCheckpointConfig(env: NodeJS.ProcessEnv = process.env) { + const checkpointDir = env[SLACK_QA_APPROVAL_CHECKPOINT_DIR_ENV]?.trim(); + if (!checkpointDir) { + return undefined; + } + const rawTimeout = env[SLACK_QA_APPROVAL_CHECKPOINT_TIMEOUT_MS_ENV]?.trim(); + const timeoutMs = rawTimeout + ? parseStrictPositiveInteger(rawTimeout) + : SLACK_QA_APPROVAL_CHECKPOINT_DEFAULT_TIMEOUT_MS; + if (timeoutMs === undefined) { + throw new Error(`${SLACK_QA_APPROVAL_CHECKPOINT_TIMEOUT_MS_ENV} must be a positive integer.`); + } + return { + checkpointDir, + timeoutMs, + }; +} + +async function waitForSlackApprovalCheckpointAck(params: { + ackPath: string; + timeoutMs: number; +}): Promise { + const startedAt = Date.now(); + while (Date.now() - startedAt < params.timeoutMs) { + try { + const parsed = JSON.parse(await fs.readFile(params.ackPath, "utf8")) as { + capturedAt?: unknown; + error?: unknown; + screenshotPath?: unknown; + }; + if (typeof parsed.error === "string" && parsed.error.trim().length > 0) { + throw new Error(`Slack approval checkpoint watcher failed: ${parsed.error}`); + } + return { + capturedAt: typeof parsed.capturedAt === "string" ? parsed.capturedAt : undefined, + screenshotPath: + typeof parsed.screenshotPath === "string" ? parsed.screenshotPath : undefined, + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } + await new Promise((resolve) => { + setTimeout(resolve, 500); + }); + } + throw new Error(`timed out after ${params.timeoutMs}ms waiting for ${params.ackPath}`); +} + +export async function writeSlackApprovalCheckpoint(params: { + approvalId: string; + approvalKind: SlackQaApprovalKind; + channelId: string; + decision?: SlackQaApprovalDecision; + message: SlackMessage; + observedAt: string; + scenarioId: SlackQaScenarioId; + state: SlackApprovalCheckpointState; +}) { + const config = resolveSlackApprovalCheckpointConfig(); + if (!config) { + return undefined; + } + await fs.mkdir(config.checkpointDir, { recursive: true }); + const checkpointPath = path.join( + config.checkpointDir, + `${params.scenarioId}.${params.state}.json`, + ); + const ackPath = path.join(config.checkpointDir, `${params.scenarioId}.${params.state}.ack.json`); + await fs.rm(ackPath, { force: true }).catch(() => {}); + await fs.writeFile( + checkpointPath, + `${JSON.stringify( + { + version: 1, + scenarioId: params.scenarioId, + approvalKind: params.approvalKind, + state: params.state, + approvalId: params.approvalId, + channelId: params.channelId, + messageTs: params.message.ts, + threadTs: params.message.thread_ts ?? null, + decision: params.decision ?? null, + observedAt: params.observedAt, + message: buildSlackApprovalCheckpointMessage(params.message), + }, + null, + 2, + )}\n`, + ); + const ack = await waitForSlackApprovalCheckpointAck({ + ackPath, + timeoutMs: config.timeoutMs, + }); + return { + ackPath, + checkpointPath, + screenshotPath: ack.screenshotPath, + }; +} + +export async function requestSlackApproval(params: { + approvalId: string; + channelId: string; + context: Omit; + run: SlackQaApprovalScenarioRun; + sutAccountId: string; +}) { + const commonParams = { + timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS, + turnSourceAccountId: params.sutAccountId, + turnSourceChannel: "slack", + turnSourceTo: `channel:${params.channelId}`, + twoPhase: true, + }; + if (params.run.approvalKind === "exec") { + const result = await params.context.gateway.call( + "exec.approval.request", + { + ...commonParams, + ask: "always", + command: `printf '%s\\n' '${params.run.token}'`, + host: "gateway", + id: params.approvalId, + security: "full", + }, + { + expectFinal: false, + timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, + }, + ); + const acceptedId = readAcceptedApprovalRequestId(result); + if (acceptedId !== params.approvalId) { + throw new Error( + `accepted exec approval id was ${formatApprovalResultValue( + acceptedId, + )} instead of ${params.approvalId}`, + ); + } + return acceptedId; + } + const result = await params.context.gateway.call( + "plugin.approval.request", + { + ...commonParams, + agentId: "qa", + description: `Slack plugin approval QA request ${params.run.token}`, + pluginId: "qa-slack-plugin", + severity: "warning", + title: `Slack plugin approval QA ${params.run.token}`, + toolName: "slack_qa_tool", + }, + { + expectFinal: false, + timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, + }, + ); + return readAcceptedApprovalRequestId(result); +} + +export async function waitForApprovalDecision(params: { + approvalId: string; + context: Omit; + kind: SlackQaApprovalKind; +}) { + const method = + params.kind === "exec" ? "exec.approval.waitDecision" : "plugin.approval.waitDecision"; + return await params.context.gateway.call( + method, + { id: params.approvalId }, + { + expectFinal: true, + timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, + }, + ); +} diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.approvals.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.approvals.ts new file mode 100644 index 000000000000..1e82484bb112 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.approvals.ts @@ -0,0 +1,394 @@ +// QA Lab Slack native approval observation and resolution. +import { randomUUID } from "node:crypto"; +import type { WebClient } from "@slack/web-api"; +import { assertApprovalDecisionResult } from "../shared/live-approval-result.js"; +import { + writeSlackApprovalCheckpoint, + requestSlackApproval, + waitForApprovalDecision, +} from "./slack-live.approval-checkpoint.js"; +import { + SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS, + type SlackQaApprovalKind, + type SlackQaApprovalDecision, + type SlackQaApprovalScenarioRun, + type SlackQaScenarioContext, + type SlackQaScenarioDefinition, + type SlackAuthIdentity, + type SlackObservedMessage, + type SlackApprovalArtifact, + type SlackMessage, +} from "./slack-live.contracts.js"; +import { + listSlackMessages, + collectSlackBlockText, + collectSlackActionValues, + parseSlackNativeApprovalAction, + hasSlackNativeApprovalActions, + extractSlackNativeApprovalId, + isSutSlackMessage, +} from "./slack-live.observations.js"; + +function resolveApprovalDecisionLabel(decision: SlackQaApprovalDecision) { + return decision === "allow-once" + ? "Allowed once" + : decision === "allow-always" + ? "Allowed always" + : "Denied"; +} + +function resolveApprovalHeading(params: { + approvalKind: SlackQaApprovalKind; + state: "pending" | "resolved"; + decision?: SlackQaApprovalDecision; +}) { + if (params.state === "pending") { + return params.approvalKind === "exec" ? "Exec approval required" : "Plugin approval required"; + } + const label = resolveApprovalDecisionLabel(params.decision ?? "allow-once"); + return params.approvalKind === "exec" ? `Exec approval: ${label}` : `Plugin approval: ${label}`; +} + +function getSlackMessageSearchText(message: SlackMessage) { + return [message.text ?? "", ...collectSlackBlockText(message.blocks)].join("\n"); +} + +function pushObservedApprovalMessage(params: { + channelId: string; + matchedScenario: boolean; + message: SlackMessage; + observedMessages: SlackObservedMessage[]; + scenarioId: string; + scenarioTitle: string; +}) { + if (!params.message.ts) { + return; + } + params.observedMessages.push({ + actionValues: collectSlackActionValues(params.message.blocks), + blockText: collectSlackBlockText(params.message.blocks), + botId: params.message.bot_id, + channelId: params.channelId, + matchedScenario: params.matchedScenario, + scenarioId: params.scenarioId, + scenarioTitle: params.scenarioTitle, + text: params.message.text ?? "", + threadTs: params.message.thread_ts, + ts: params.message.ts, + userId: params.message.user, + }); +} + +export async function waitForSlackApprovalPrompt(params: { + approvalId?: string; + approvalKind: SlackQaApprovalKind; + channelId: string; + client: WebClient; + decision: SlackQaApprovalDecision; + observedMessages: SlackObservedMessage[]; + oldestTs: string; + scenarioId: string; + scenarioTitle: string; + sutIdentity: SlackAuthIdentity; + timeoutMs: number; + token?: string; + extraTextMatches?: string[]; +}) { + const startedAt = Date.now(); + const seenObservedMessages = new Set(); + let lastMatchedWithoutActions = ""; + while (Date.now() - startedAt < params.timeoutMs) { + const messages = await listSlackMessages({ + channelId: params.channelId, + client: params.client, + oldestTs: params.oldestTs, + }); + for (const message of messages) { + if (!message.ts || !isSutSlackMessage(message, params.sutIdentity)) { + continue; + } + const text = getSlackMessageSearchText(message); + const actionValues = collectSlackActionValues(message.blocks); + const matchedScenario = matchesSlackApprovalPromptText({ + approvalKind: params.approvalKind, + extraTextMatches: params.extraTextMatches, + text, + token: params.token, + }); + const observedKey = `${message.ts}:${message.text ?? ""}:${actionValues.join("|")}`; + if (matchedScenario || hasSlackNativeApprovalActions({ ...params, actionValues })) { + if (!seenObservedMessages.has(observedKey)) { + seenObservedMessages.add(observedKey); + pushObservedApprovalMessage({ + channelId: params.channelId, + matchedScenario, + message, + observedMessages: params.observedMessages, + scenarioId: params.scenarioId, + scenarioTitle: params.scenarioTitle, + }); + } + } + if (!matchedScenario) { + continue; + } + if ( + !hasSlackNativeApprovalActions({ + actionValues, + approvalId: params.approvalId, + decision: params.decision, + }) + ) { + lastMatchedWithoutActions = `message ${message.ts} matched approval text but did not expose native approval button values`; + continue; + } + return { + actionValues, + approvalId: + params.approvalId ?? + extractSlackNativeApprovalId({ + actionValues, + decision: params.decision, + }), + message, + observedAt: new Date().toISOString(), + }; + } + await new Promise((resolve) => { + setTimeout(resolve, 1_000); + }); + } + throw new Error( + [ + `timed out after ${params.timeoutMs}ms waiting for Slack ${params.approvalKind} approval prompt`, + lastMatchedWithoutActions, + ] + .filter(Boolean) + .join("; "), + ); +} + +export function matchesSlackApprovalPromptText(params: { + approvalKind: SlackQaApprovalKind; + extraTextMatches?: string[]; + text: string; + token?: string; +}) { + return ( + params.text.includes( + resolveApprovalHeading({ approvalKind: params.approvalKind, state: "pending" }), + ) && + (!params.token || params.text.includes(params.token)) && + (params.extraTextMatches ?? []).every((match) => params.text.includes(match)) + ); +} + +export async function waitForSlackApprovalResolvedUpdate(params: { + approvalKind: SlackQaApprovalKind; + channelId: string; + client: WebClient; + decision: SlackQaApprovalDecision; + messageTs: string; + observedMessages: SlackObservedMessage[]; + oldestTs: string; + scenarioId: string; + scenarioTitle: string; + sutIdentity: SlackAuthIdentity; + timeoutMs: number; + token?: string; + extraTextMatches?: string[]; +}) { + const startedAt = Date.now(); + const seenObservedMessages = new Set(); + while (Date.now() - startedAt < params.timeoutMs) { + const messages = await listSlackMessages({ + channelId: params.channelId, + client: params.client, + oldestTs: params.oldestTs, + }); + const message = messages.find((entry) => entry.ts === params.messageTs); + if (message && isSutSlackMessage(message, params.sutIdentity)) { + const text = getSlackMessageSearchText(message); + const actionValues = collectSlackActionValues(message.blocks); + const matchedScenario = matchesSlackApprovalResolvedUpdate({ + actionValues, + approvalKind: params.approvalKind, + decision: params.decision, + extraTextMatches: params.extraTextMatches, + text, + token: params.token, + }); + const observedKey = `${message.ts}:${message.text ?? ""}:${actionValues.join("|")}`; + if (!seenObservedMessages.has(observedKey)) { + seenObservedMessages.add(observedKey); + pushObservedApprovalMessage({ + channelId: params.channelId, + matchedScenario, + message, + observedMessages: params.observedMessages, + scenarioId: params.scenarioId, + scenarioTitle: params.scenarioTitle, + }); + } + if (matchedScenario) { + return { + actionValues, + message, + observedAt: new Date().toISOString(), + }; + } + } + await new Promise((resolve) => { + setTimeout(resolve, 1_000); + }); + } + throw new Error( + `timed out after ${params.timeoutMs}ms waiting for Slack ${params.approvalKind} approval resolution update`, + ); +} + +export function matchesSlackApprovalResolvedUpdate(params: { + actionValues: string[]; + approvalKind: SlackQaApprovalKind; + decision: SlackQaApprovalDecision; + extraTextMatches?: string[]; + text: string; + token?: string; +}) { + return ( + params.text.includes( + resolveApprovalHeading({ + approvalKind: params.approvalKind, + decision: params.decision, + state: "resolved", + }), + ) && + (!params.token || params.text.includes(params.token)) && + (params.extraTextMatches ?? []).every((match) => params.text.includes(match)) && + !params.actionValues.some((value) => parseSlackNativeApprovalAction(value)) + ); +} + +export async function resolveApprovalDecision(params: { + approvalId: string; + context: Omit; + decision: SlackQaApprovalDecision; + kind: SlackQaApprovalKind; +}) { + const method = params.kind === "exec" ? "exec.approval.resolve" : "plugin.approval.resolve"; + return await params.context.gateway.call( + method, + { decision: params.decision, id: params.approvalId }, + { + expectFinal: false, + timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, + }, + ); +} + +export async function runSlackApprovalScenario(params: { + channelId: string; + context: Omit; + observedMessages: SlackObservedMessage[]; + run: SlackQaApprovalScenarioRun; + scenario: SlackQaScenarioDefinition; + sutAccountId: string; +}) { + const requestStartedAt = new Date(); + const oldestTs = ((requestStartedAt.getTime() - 5_000) / 1_000).toFixed(6); + const requestedApprovalId = + params.run.approvalKind === "exec" + ? `slack-qa-exec-${randomUUID()}` + : `slack-qa-plugin-${randomUUID()}`; + const approvalId = await requestSlackApproval({ + approvalId: requestedApprovalId, + channelId: params.channelId, + context: params.context, + run: params.run, + sutAccountId: params.sutAccountId, + }); + const pending = await waitForSlackApprovalPrompt({ + approvalId, + approvalKind: params.run.approvalKind, + channelId: params.channelId, + client: params.context.sutReadClient, + decision: params.run.decision, + observedMessages: params.observedMessages, + oldestTs, + scenarioId: params.scenario.id, + scenarioTitle: params.scenario.title, + sutIdentity: params.context.sutIdentity, + timeoutMs: params.scenario.timeoutMs, + token: params.run.token, + }); + const pendingCheckpoint = await writeSlackApprovalCheckpoint({ + approvalId, + approvalKind: params.run.approvalKind, + channelId: params.channelId, + message: pending.message, + observedAt: pending.observedAt, + scenarioId: params.scenario.id, + state: "pending", + }); + await resolveApprovalDecision({ + approvalId, + context: params.context, + decision: params.run.decision, + kind: params.run.approvalKind, + }); + assertApprovalDecisionResult({ + decision: params.run.decision, + result: await waitForApprovalDecision({ + approvalId, + context: params.context, + kind: params.run.approvalKind, + }), + }); + const resolved = await waitForSlackApprovalResolvedUpdate({ + approvalKind: params.run.approvalKind, + channelId: params.channelId, + client: params.context.sutReadClient, + decision: params.run.decision, + messageTs: pending.message.ts, + observedMessages: params.observedMessages, + oldestTs, + scenarioId: params.scenario.id, + scenarioTitle: params.scenario.title, + sutIdentity: params.context.sutIdentity, + timeoutMs: params.scenario.timeoutMs, + token: params.run.token, + }); + const resolvedCheckpoint = await writeSlackApprovalCheckpoint({ + approvalId, + approvalKind: params.run.approvalKind, + channelId: params.channelId, + decision: params.run.decision, + message: resolved.message, + observedAt: resolved.observedAt, + scenarioId: params.scenario.id, + state: "resolved", + }); + const responseObservedAt = new Date(resolved.observedAt); + return { + artifact: { + approvalId, + approvalKind: params.run.approvalKind, + channelId: params.channelId, + decision: params.run.decision, + pendingActionValues: pending.actionValues, + pendingCheckpointPath: pendingCheckpoint?.checkpointPath, + pendingMessageTs: pending.message.ts, + pendingScreenshotPath: pendingCheckpoint?.screenshotPath, + pendingText: pending.message.text, + resolvedActionValues: resolved.actionValues, + resolvedCheckpointPath: resolvedCheckpoint?.checkpointPath, + resolvedMessageTs: resolved.message.ts, + resolvedScreenshotPath: resolvedCheckpoint?.screenshotPath, + resolvedText: resolved.message.text, + threadTs: pending.message.thread_ts, + } satisfies SlackApprovalArtifact, + requestStartedAt, + responseObservedAt, + rttMs: responseObservedAt.getTime() - requestStartedAt.getTime(), + }; +} diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.artifacts.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.artifacts.ts new file mode 100644 index 000000000000..7fec0426b90d --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.artifacts.ts @@ -0,0 +1,148 @@ +// QA Lab Slack artifact shaping and report rendering. +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { appendQaLiveLaneIssue as appendLiveLaneIssue } from "../shared/live-artifacts.js"; +import type { + SlackQaGatewayHarness, + SlackObservedMessage, + SlackObservedMessageArtifact, + SlackQaScenarioResult, +} from "./slack-live.contracts.js"; + +export function isRetryableSlackQaScenarioError(error: unknown) { + return /timed out after \d+ms waiting for Slack message/iu.test(formatErrorMessage(error)); +} + +export function toObservedSlackArtifacts(params: { + includeContent: boolean; + messages: SlackObservedMessage[]; + redactMetadata: boolean; +}): SlackObservedMessageArtifact[] { + return params.messages.map((message) => ({ + actionValues: params.includeContent ? message.actionValues : undefined, + blockText: params.includeContent ? message.blockText : undefined, + botId: params.redactMetadata ? undefined : message.botId, + channelId: params.redactMetadata ? undefined : message.channelId, + matchedScenario: message.matchedScenario, + scenarioId: message.scenarioId, + scenarioTitle: message.scenarioTitle, + text: params.includeContent ? message.text : undefined, + threadTs: params.redactMetadata ? undefined : message.threadTs, + ts: params.redactMetadata ? undefined : message.ts, + userId: params.redactMetadata ? undefined : message.userId, + })); +} + +export function toSlackQaScenarioArtifactResults(params: { + includeContent: boolean; + redactMetadata: boolean; + scenarios: SlackQaScenarioResult[]; +}): SlackQaScenarioResult[] { + return params.scenarios.map((scenario) => { + if (!scenario.approval) { + return scenario; + } + const approval = scenario.approval; + return { + ...scenario, + approval: { + approvalId: params.redactMetadata ? "" : approval.approvalId, + approvalKind: approval.approvalKind, + appServerMethod: approval.appServerMethod, + channelId: params.redactMetadata ? undefined : approval.channelId, + codexModelKey: approval.codexModelKey, + decision: approval.decision, + finalCodexTurnStatus: approval.finalCodexTurnStatus, + operationVerified: approval.operationVerified, + pendingActionValues: params.includeContent ? approval.pendingActionValues : undefined, + pendingCheckpointPath: approval.pendingCheckpointPath, + pendingMessageTs: params.redactMetadata ? undefined : approval.pendingMessageTs, + pendingScreenshotPath: approval.pendingScreenshotPath, + pendingText: params.includeContent ? approval.pendingText : undefined, + resolvedActionValues: params.includeContent ? approval.resolvedActionValues : undefined, + resolvedCheckpointPath: approval.resolvedCheckpointPath, + resolvedMessageTs: params.redactMetadata ? undefined : approval.resolvedMessageTs, + resolvedScreenshotPath: approval.resolvedScreenshotPath, + resolvedText: params.includeContent ? approval.resolvedText : undefined, + threadTs: params.redactMetadata ? undefined : approval.threadTs, + }, + }; + }); +} + +export function renderSlackQaMarkdown(params: { + channelId: string; + cleanupIssues: string[]; + credentialSource: "convex" | "env"; + finishedAt: string; + gatewayDebugDirPath?: string; + redactMetadata: boolean; + scenarios: SlackQaScenarioResult[]; + startedAt: string; +}) { + const lines = [ + "# Slack QA Report", + "", + `- Credential source: \`${params.credentialSource}\``, + `- Channel: \`${params.redactMetadata ? "" : params.channelId}\``, + `- Metadata redaction: \`${params.redactMetadata ? "enabled" : "disabled"}\``, + `- Started: ${params.startedAt}`, + `- Finished: ${params.finishedAt}`, + ]; + if (params.gatewayDebugDirPath) { + lines.push(`- Gateway debug artifacts: \`${params.gatewayDebugDirPath}\``); + } + if (params.cleanupIssues.length > 0) { + lines.push("", "## Cleanup issues", ""); + for (const issue of params.cleanupIssues) { + lines.push(`- ${issue}`); + } + } + lines.push("", "## Scenarios", ""); + for (const scenario of params.scenarios) { + lines.push(`### ${scenario.title}`, ""); + lines.push(`- Status: ${scenario.status}`); + lines.push(`- Details: ${scenario.details}`); + if (scenario.rttMs !== undefined) { + lines.push(`- RTT: ${scenario.rttMs}ms`); + } + if (scenario.approval) { + lines.push(`- Approval kind: ${scenario.approval.approvalKind}`); + if (scenario.approval.appServerMethod) { + lines.push(`- Codex app-server method: \`${scenario.approval.appServerMethod}\``); + } + if (scenario.approval.codexModelKey) { + lines.push(`- Codex model: \`${scenario.approval.codexModelKey}\``); + } + if (scenario.approval.finalCodexTurnStatus) { + lines.push(`- Codex turn status: ${scenario.approval.finalCodexTurnStatus}`); + } + if (scenario.approval.operationVerified) { + lines.push("- Codex operation marker: verified"); + } + lines.push(`- Approval ID: \`${scenario.approval.approvalId}\``); + lines.push(`- Decision: ${scenario.approval.decision}`); + if (scenario.approval.pendingScreenshotPath) { + lines.push(`- Pending screenshot: \`${scenario.approval.pendingScreenshotPath}\``); + } + if (scenario.approval.resolvedScreenshotPath) { + lines.push(`- Resolved screenshot: \`${scenario.approval.resolvedScreenshotPath}\``); + } + } + lines.push(""); + } + return lines.join("\n"); +} + +export async function preserveSlackGatewayDebugArtifacts(params: { + cleanupIssues: string[]; + gatewayDebugDirPath: string; + gatewayHarness: SlackQaGatewayHarness; +}) { + try { + await params.gatewayHarness.stop({ preserveToDir: params.gatewayDebugDirPath }); + return true; + } catch (error) { + appendLiveLaneIssue(params.cleanupIssues, "gateway debug preservation failed", error); + return false; + } +} diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval-runner.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval-runner.ts new file mode 100644 index 000000000000..678899881318 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval-runner.ts @@ -0,0 +1,227 @@ +// QA Lab Slack Codex approval scenario orchestration. +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { writeSlackApprovalCheckpoint } from "./slack-live.approval-checkpoint.js"; +import { + waitForSlackApprovalPrompt, + waitForSlackApprovalResolvedUpdate, + resolveApprovalDecision, +} from "./slack-live.approvals.js"; +import { + assertCodexApprovalOperationSucceeded, + assertPendingCodexPluginApproval, + startCodexApprovalAgentRun, + buildCodexApprovalSessionKey, + waitForCodexApprovalAgentRun, + quiesceCodexApprovalAgentRun, + resolveCodexFileApprovalTargetPath, +} from "./slack-live.codex-approval.js"; +import type { + SlackQaCodexApprovalScenarioRun, + SlackQaScenarioContext, + SlackQaScenarioDefinition, + SlackObservedMessage, + SlackApprovalArtifact, +} from "./slack-live.contracts.js"; + +export async function runSlackCodexApprovalScenario(params: { + channelId: string; + context: Omit; + observedMessages: SlackObservedMessage[]; + primaryModel: string; + run: SlackQaCodexApprovalScenarioRun; + scenario: SlackQaScenarioDefinition; + stopGateway: (preserveDebugArtifacts: boolean) => Promise; + sutAccountId: string; +}) { + const codexRun = { + runId: `slack-qa-codex-approval-${randomUUID()}`, + sessionKey: buildCodexApprovalSessionKey({ + scenario: params.scenario, + token: params.run.token, + }), + }; + const targetPath = + params.run.appServerMethod === "item/fileChange/requestApproval" + ? resolveCodexFileApprovalTargetPath(params.run.token) + : undefined; + if (targetPath) { + await fs.rm(targetPath, { force: true }); + } + const outcome = await runSlackCodexApprovalScenarioInner({ ...params, codexRun }).then( + (result) => ({ kind: "success", result }) as const, + (error: unknown) => ({ error, kind: "failure" }) as const, + ); + // Kill the gateway process tree before deleting the probe. Agent completion + // does not prove the native Codex turn has stopped writing after an interrupt. + const cleanupErrors: unknown[] = []; + try { + await quiesceCodexApprovalAgentRun({ + context: params.context, + preserveDebugArtifacts: outcome.kind === "failure", + stopGateway: params.stopGateway, + ...codexRun, + }); + } catch (error) { + cleanupErrors.push(error); + } + if (cleanupErrors.length === 0 && targetPath) { + try { + await fs.rm(targetPath, { force: true }); + } catch (error) { + cleanupErrors.push(error); + } + } + if (cleanupErrors.length > 0) { + const cleanupSummary = cleanupErrors.map(formatErrorMessage).join("; "); + if (outcome.kind === "failure") { + throw new AggregateError( + [outcome.error, ...cleanupErrors], + `Codex approval scenario failed: ${formatErrorMessage(outcome.error)}; cleanup also failed: ${cleanupSummary}`, + { cause: outcome.error }, + ); + } + throw new AggregateError(cleanupErrors, `Codex approval cleanup failed: ${cleanupSummary}`); + } + if (outcome.kind === "failure") { + throw outcome.error; + } + return outcome.result; +} + +async function runSlackCodexApprovalScenarioInner(params: { + channelId: string; + codexRun: { runId: string; sessionKey: string }; + context: Omit; + observedMessages: SlackObservedMessage[]; + primaryModel: string; + run: SlackQaCodexApprovalScenarioRun; + scenario: SlackQaScenarioDefinition; + sutAccountId: string; +}) { + const requestStartedAt = new Date(); + const oldestTs = ((requestStartedAt.getTime() - 5_000) / 1_000).toFixed(6); + await startCodexApprovalAgentRun({ + channelId: params.channelId, + context: params.context, + primaryModel: params.primaryModel, + run: params.run, + runId: params.codexRun.runId, + scenario: params.scenario, + sessionKey: params.codexRun.sessionKey, + sutAccountId: params.sutAccountId, + }); + const expectedTitle = + params.run.appServerMethod === "item/commandExecution/requestApproval" + ? "Codex app-server command approval" + : "Codex app-server file approval"; + const pending = await waitForSlackApprovalPrompt({ + approvalKind: params.run.approvalKind, + channelId: params.channelId, + client: params.context.sutReadClient, + decision: params.run.decision, + extraTextMatches: ["openclaw-codex-app-server", expectedTitle], + observedMessages: params.observedMessages, + oldestTs, + scenarioId: params.scenario.id, + scenarioTitle: params.scenario.title, + sutIdentity: params.context.sutIdentity, + timeoutMs: params.scenario.timeoutMs, + }); + const approvalId = pending.approvalId; + if (!approvalId) { + throw new Error( + "Codex Slack approval prompt exposed native actions but no plugin approval id.", + ); + } + await assertPendingCodexPluginApproval({ + approvalId, + appServerMethod: params.run.appServerMethod, + channelId: params.channelId, + context: params.context, + sessionKey: params.codexRun.sessionKey, + sutAccountId: params.sutAccountId, + }); + const pendingCheckpoint = await writeSlackApprovalCheckpoint({ + approvalId, + approvalKind: params.run.approvalKind, + channelId: params.channelId, + message: pending.message, + observedAt: pending.observedAt, + scenarioId: params.scenario.id, + state: "pending", + }); + await resolveApprovalDecision({ + approvalId, + context: params.context, + decision: params.run.decision, + kind: params.run.approvalKind, + }); + const finalCodexTurnStatus = await waitForCodexApprovalAgentRun({ + context: params.context, + runId: params.codexRun.runId, + timeoutMs: params.scenario.timeoutMs, + }); + if (finalCodexTurnStatus !== "ok") { + throw new Error( + `Codex approval run ${params.codexRun.runId} finished with status ${finalCodexTurnStatus}`, + ); + } + await assertCodexApprovalOperationSucceeded({ + context: params.context, + run: params.run, + sessionKey: params.codexRun.sessionKey, + }); + const resolved = await waitForSlackApprovalResolvedUpdate({ + approvalKind: params.run.approvalKind, + channelId: params.channelId, + client: params.context.sutReadClient, + decision: params.run.decision, + messageTs: pending.message.ts, + observedMessages: params.observedMessages, + oldestTs, + scenarioId: params.scenario.id, + scenarioTitle: params.scenario.title, + sutIdentity: params.context.sutIdentity, + timeoutMs: params.scenario.timeoutMs, + extraTextMatches: ["openclaw-codex-app-server", expectedTitle], + }); + const resolvedCheckpoint = await writeSlackApprovalCheckpoint({ + approvalId, + approvalKind: params.run.approvalKind, + channelId: params.channelId, + decision: params.run.decision, + message: resolved.message, + observedAt: resolved.observedAt, + scenarioId: params.scenario.id, + state: "resolved", + }); + const responseObservedAt = new Date(resolved.observedAt); + return { + artifact: { + approvalId, + approvalKind: params.run.approvalKind, + appServerMethod: params.run.appServerMethod, + channelId: params.channelId, + codexModelKey: params.primaryModel, + decision: params.run.decision, + finalCodexTurnStatus, + operationVerified: true, + pendingActionValues: pending.actionValues, + pendingCheckpointPath: pendingCheckpoint?.checkpointPath, + pendingMessageTs: pending.message.ts, + pendingScreenshotPath: pendingCheckpoint?.screenshotPath, + pendingText: pending.message.text, + resolvedActionValues: resolved.actionValues, + resolvedCheckpointPath: resolvedCheckpoint?.checkpointPath, + resolvedMessageTs: resolved.message.ts, + resolvedScreenshotPath: resolvedCheckpoint?.screenshotPath, + resolvedText: resolved.message.text, + threadTs: pending.message.thread_ts, + } satisfies SlackApprovalArtifact, + requestStartedAt, + responseObservedAt, + rttMs: responseObservedAt.getTime() - requestStartedAt.getTime(), + }; +} diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval.ts new file mode 100644 index 000000000000..d60cc7a35521 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.codex-approval.ts @@ -0,0 +1,311 @@ +// QA Lab Slack Codex approval helpers and transcript validation. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { listSlackReactions } from "@openclaw/slack/api.js"; +import type { WebClient } from "@slack/web-api"; +import { extractGatewayMessageText } from "../../gateway-log-sentinel.js"; +import { formatApprovalResultValue } from "../shared/live-approval-result.js"; +import { asPlainRecord } from "./slack-live.config.js"; +import { + SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS, + type SlackQaCodexApprovalMethod, + type SlackQaCodexApprovalScenarioRun, + type SlackQaScenarioContext, + type SlackQaScenarioDefinition, +} from "./slack-live.contracts.js"; + +export function resolveCodexFileApprovalTargetPath(token: string) { + return path.join(os.homedir(), `.openclaw-qa-codex-file-approval-${token.toLowerCase()}.txt`); +} + +export function buildCodexApprovalInstruction(params: { + appServerMethod: SlackQaCodexApprovalMethod; + token: string; +}) { + if (params.appServerMethod === "item/commandExecution/requestApproval") { + return [ + "Slack QA Codex command approval scenario.", + `Marker: ${params.token}`, + "Use the shell tool exactly once and request approval before running it.", + `Run exactly: printf '${params.token}\\n'`, + `After the command succeeds, reply exactly ${params.token} and nothing else.`, + ].join("\n"); + } + const targetPath = resolveCodexFileApprovalTargetPath(params.token); + return [ + "Slack QA Codex file approval scenario.", + `Marker: ${params.token}`, + "Use apply_patch exactly once to create this file outside the workspace.", + `Target file: ${targetPath}`, + `File contents: ${params.token}`, + "Invoke apply_patch now. Do not ask for approval in chat; the harness will resolve the native tool approval.", + `After the file change succeeds, reply exactly ${params.token} and nothing else.`, + ].join("\n"); +} + +export function readAcceptedAgentRunId(result: unknown) { + const started = + typeof result === "object" && result !== null + ? (result as { runId?: unknown; status?: unknown }) + : null; + if (started?.status !== "accepted") { + throw new Error( + `Codex agent run status was ${formatApprovalResultValue(started?.status)} instead of accepted`, + ); + } + if (typeof started.runId !== "string" || started.runId.trim().length === 0) { + throw new Error(`Codex agent run id was ${formatApprovalResultValue(started.runId)}`); + } + return started.runId; +} + +function readAgentWaitStatus(result: unknown) { + if (typeof result !== "object" || result === null) { + return "unknown"; + } + const status = (result as { status?: unknown }).status; + return typeof status === "string" && status.trim() ? status : "unknown"; +} + +export async function waitForSlackReaction(params: { + channelId: string; + client: WebClient; + expectedReactionName: string; + messageId: string; + sutUserId: string; + timeoutMs: number; +}) { + const deadline = Date.now() + params.timeoutMs; + while (true) { + const reactions = await listSlackReactions(params.channelId, params.messageId, { + client: params.client, + }); + const reaction = reactions?.find( + (entry) => + entry.name === params.expectedReactionName && entry.users?.includes(params.sutUserId), + ); + if (reaction) { + return reaction; + } + if (Date.now() >= deadline) { + break; + } + await new Promise((resolve) => { + setTimeout(resolve, 1_000); + }); + } + throw new Error( + `Slack message ${params.messageId} did not receive ${params.expectedReactionName} from ${params.sutUserId}`, + ); +} + +export function assertCodexApprovalTranscriptSucceeded( + messages: unknown, + run: SlackQaCodexApprovalScenarioRun, +) { + const records = Array.isArray(messages) ? messages.map(asPlainRecord) : []; + const assistantReply = records + .toReversed() + .find((message) => message.role === "assistant" && extractGatewayMessageText(message)); + if (!assistantReply || extractGatewayMessageText(assistantReply) !== run.token) { + throw new Error(`Codex approval run did not finish with assistant marker ${run.token}`); + } + if (run.appServerMethod !== "item/commandExecution/requestApproval") { + return; + } + const commandSucceeded = records.some((message) => { + if (message.role !== "toolResult" || message.isError === true) { + return false; + } + return extractGatewayMessageText(message) + .split(/\r?\n/u) + .some((line) => line.trim() === run.token); + }); + if (!commandSucceeded) { + throw new Error(`Codex command result did not contain marker ${run.token}`); + } +} + +export async function assertCodexApprovalOperationSucceeded(params: { + context: Omit; + run: SlackQaCodexApprovalScenarioRun; + sessionKey: string; +}) { + const history = asPlainRecord( + await params.context.gateway.call( + "chat.history", + { sessionKey: params.sessionKey, limit: 24 }, + { timeoutMs: 10_000 }, + ), + ); + assertCodexApprovalTranscriptSucceeded(history.messages, params.run); + if (params.run.appServerMethod !== "item/fileChange/requestApproval") { + return; + } + const targetPath = resolveCodexFileApprovalTargetPath(params.run.token); + const contents = await fs.readFile(targetPath, "utf8"); + if (contents.trim() !== params.run.token) { + throw new Error(`Codex file result at ${targetPath} did not contain the expected marker`); + } +} + +export function findPendingCodexPluginApprovalRecord(params: { + approvalId: string; + appServerMethod: SlackQaCodexApprovalMethod; + channelId: string; + records: unknown; + sessionKey: string; + sutAccountId: string; +}) { + const list = Array.isArray(params.records) ? params.records : []; + const expectedTitle = + params.appServerMethod === "item/commandExecution/requestApproval" + ? "Codex app-server command approval" + : "Codex app-server file approval"; + const expectedToolName = + params.appServerMethod === "item/commandExecution/requestApproval" + ? "codex_command_approval" + : "codex_file_approval"; + for (const entry of list) { + const record = asPlainRecord(entry); + if (record.id !== params.approvalId) { + continue; + } + const request = asPlainRecord(record.request); + if ( + request.pluginId === "openclaw-codex-app-server" && + request.title === expectedTitle && + request.toolName === expectedToolName && + request.sessionKey === params.sessionKey && + request.turnSourceChannel === "slack" && + request.turnSourceTo === `channel:${params.channelId}` && + request.turnSourceAccountId === params.sutAccountId + ) { + return record; + } + } + return undefined; +} + +export async function assertPendingCodexPluginApproval(params: { + approvalId: string; + appServerMethod: SlackQaCodexApprovalMethod; + channelId: string; + context: Omit; + sessionKey: string; + sutAccountId: string; +}) { + const records = await params.context.gateway.call( + "plugin.approval.list", + {}, + { + timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS, + }, + ); + const record = findPendingCodexPluginApprovalRecord({ + approvalId: params.approvalId, + appServerMethod: params.appServerMethod, + channelId: params.channelId, + records, + sessionKey: params.sessionKey, + sutAccountId: params.sutAccountId, + }); + if (!record) { + throw new Error( + `Pending Codex plugin approval ${params.approvalId} did not match the expected app-server route and Slack turn source.`, + ); + } +} + +export async function startCodexApprovalAgentRun(params: { + channelId: string; + context: Omit; + primaryModel: string; + run: SlackQaCodexApprovalScenarioRun; + runId: string; + scenario: SlackQaScenarioDefinition; + sessionKey: string; + sutAccountId: string; +}) { + const result = await params.context.gateway.call( + "agent", + { + accountId: params.sutAccountId, + agentId: "qa", + channel: "slack", + cleanupBundleMcpOnRunEnd: true, + deliver: false, + idempotencyKey: params.runId, + message: buildCodexApprovalInstruction({ + appServerMethod: params.run.appServerMethod, + token: params.run.token, + }), + model: params.primaryModel, + sessionKey: params.sessionKey, + thinking: "low", + timeout: Math.ceil(params.scenario.timeoutMs / 1_000), + to: `channel:${params.channelId}`, + }, + { + timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, + }, + ); + const acceptedRunId = readAcceptedAgentRunId(result); + if (acceptedRunId !== params.runId) { + throw new Error(`Codex agent run id was ${acceptedRunId} instead of ${params.runId}`); + } +} + +export function buildCodexApprovalSessionKey(params: { + scenario: SlackQaScenarioDefinition; + token: string; +}) { + return `agent:qa:${params.scenario.id}-${params.token.toLowerCase()}`; +} + +export async function waitForCodexApprovalAgentRun(params: { + context: Omit; + runId: string; + timeoutMs: number; +}) { + const result = await params.context.gateway.call( + "agent.wait", + { + runId: params.runId, + timeoutMs: params.timeoutMs, + }, + { + timeoutMs: params.timeoutMs + 5_000, + }, + ); + return readAgentWaitStatus(result); +} + +export async function quiesceCodexApprovalAgentRun(params: { + context: Omit; + preserveDebugArtifacts: boolean; + runId: string; + sessionKey: string; + stopGateway: (preserveDebugArtifacts: boolean) => Promise; +}) { + try { + await params.context.gateway.call( + "chat.abort", + { runId: params.runId, sessionKey: params.sessionKey }, + { timeoutMs: 10_000 }, + ); + } catch { + // The bounded terminal wait and gateway process-group teardown do not depend on this ack. + } + try { + await params.context.gateway.call( + "agent.wait", + { runId: params.runId, timeoutMs: 10_000 }, + { timeoutMs: 15_000 }, + ); + } catch { + // QA-owned Codex app-server processes inherit the gateway cleanup process group. + } + await params.stopGateway(params.preserveDebugArtifacts); +} diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.config.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.config.ts new file mode 100644 index 000000000000..191eb7897b1d --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.config.ts @@ -0,0 +1,315 @@ +// QA Lab Slack credentials, instrumentation, and channel config. +import type { WebClient } from "@slack/web-api"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { + type SlackQaRuntimeEnv, + type SlackQaConfigOverrides, + SLACK_QA_ENV_KEYS, + slackQaCredentialPayloadSchema, +} from "./slack-live.contracts.js"; + +function resolveEnvValue(env: NodeJS.ProcessEnv, key: (typeof SLACK_QA_ENV_KEYS)[number]) { + const value = env[key]?.trim(); + if (!value) { + throw new Error(`Missing ${key}.`); + } + return value; +} + +function normalizeSlackId(value: string, label: string) { + const normalized = value.trim(); + if (!/^[A-Z][A-Z0-9]+$/.test(normalized)) { + throw new Error(`${label} must be a Slack id like C123 or U123.`); + } + return normalized; +} + +function validateSlackQaRuntimeEnv(runtimeEnv: SlackQaRuntimeEnv, label: string) { + normalizeSlackId(runtimeEnv.channelId, `${label} channelId`); + return runtimeEnv; +} + +export function resolveSlackQaRuntimeEnv(env: NodeJS.ProcessEnv = process.env): SlackQaRuntimeEnv { + const runtimeEnv = { + channelId: resolveEnvValue(env, "OPENCLAW_QA_SLACK_CHANNEL_ID"), + driverBotToken: resolveEnvValue(env, "OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN"), + sutBotToken: resolveEnvValue(env, "OPENCLAW_QA_SLACK_SUT_BOT_TOKEN"), + sutAppToken: resolveEnvValue(env, "OPENCLAW_QA_SLACK_SUT_APP_TOKEN"), + }; + return validateSlackQaRuntimeEnv(runtimeEnv, "OPENCLAW_QA_SLACK"); +} + +export function parseSlackQaCredentialPayload(payload: unknown): SlackQaRuntimeEnv { + const parsed = slackQaCredentialPayloadSchema.parse(payload); + const runtimeEnv = { + channelId: parsed.channelId, + driverBotToken: parsed.driverBotToken, + sutBotToken: parsed.sutBotToken, + sutAppToken: parsed.sutAppToken, + }; + return validateSlackQaRuntimeEnv(runtimeEnv, "Slack credential payload"); +} + +export function asPlainRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +type SlackQaPostMessageAttempt = { + failureCode?: string; + formattingDisabled: boolean; + nativeDataBlockCount: number; + status: "failed" | "sent"; +}; + +export function countSlackNativeDataBlocks(value: unknown) { + if (!Array.isArray(value)) { + return 0; + } + return value.filter((block) => { + const type = asPlainRecord(block).type; + return type === "data_table" || type === "data_visualization"; + }).length; +} + +function readSlackApiFailureCode(error: unknown) { + const record = asPlainRecord(error); + const data = asPlainRecord(record.data); + const code = data.error ?? record.error; + return typeof code === "string" && /^[a-z0-9_]{1,64}$/u.test(code) ? code : undefined; +} + +export function instrumentSlackPostMessage(client: WebClient) { + const originalPostMessage = client.chat.postMessage; + const attempts: SlackQaPostMessageAttempt[] = []; + client.chat.postMessage = (async (payload) => { + const payloadRecord = payload as { blocks?: unknown; mrkdwn?: boolean }; + const attempt = { + formattingDisabled: payloadRecord.mrkdwn === false, + nativeDataBlockCount: countSlackNativeDataBlocks(payloadRecord.blocks), + }; + try { + const response = await originalPostMessage.call(client.chat, payload); + attempts.push({ ...attempt, status: "sent" }); + return response; + } catch (error) { + const failureCode = readSlackApiFailureCode(error); + attempts.push({ + ...attempt, + ...(failureCode ? { failureCode } : {}), + status: "failed", + }); + throw error; + } + }) as typeof client.chat.postMessage; + return { + attempts, + restore: () => { + client.chat.postMessage = originalPostMessage; + }, + }; +} + +export function buildSlackQaConfig( + baseCfg: OpenClawConfig, + params: { + channelId: string; + driverBotUserId: string; + overrides?: SlackQaConfigOverrides; + primaryModel?: string; + sutAccountId: string; + sutAppToken: string; + sutBotToken: string; + }, +): OpenClawConfig { + const codexApprovalConfig = params.overrides?.codexApproval === true; + const progressOverrides = params.overrides?.progress; + const primaryModel = params.primaryModel; + const pluginAllow = uniqueStrings([ + ...(baseCfg.plugins?.allow ?? []), + "slack", + ...(codexApprovalConfig ? ["codex"] : []), + ]); + const approvalOverrides = params.overrides?.approvals; + const codexEntry = baseCfg.plugins?.entries?.codex; + const codexEntryConfig = asPlainRecord(codexEntry?.config); + const codexAppServerConfig = asPlainRecord(codexEntryConfig.appServer); + const approvalForwardingConfig = + approvalOverrides?.exec || approvalOverrides?.plugin + ? { + approvals: { + ...baseCfg.approvals, + ...(approvalOverrides.exec + ? { + exec: { + ...baseCfg.approvals?.exec, + enabled: true, + mode: "session" as const, + }, + } + : {}), + ...(approvalOverrides.plugin + ? { + plugin: { + ...baseCfg.approvals?.plugin, + enabled: true, + mode: "session" as const, + }, + } + : {}), + }, + } + : {}; + const codexAgentDefaults = + codexApprovalConfig && primaryModel + ? { + ...baseCfg.agents?.defaults, + models: { + ...baseCfg.agents?.defaults?.models, + [primaryModel]: { + ...baseCfg.agents?.defaults?.models?.[primaryModel], + agentRuntime: { id: "codex" as const }, + }, + }, + } + : baseCfg.agents?.defaults; + const qaAgentDefaults = progressOverrides + ? { + ...codexAgentDefaults, + ...(progressOverrides.verboseDefault + ? { verboseDefault: progressOverrides.verboseDefault } + : {}), + } + : codexAgentDefaults; + const qaAgentList = progressOverrides + ? baseCfg.agents?.list?.map((agent) => { + if (agent.id !== "qa") { + return agent; + } + // Slack draft edits cannot preserve custom authorship. Remove the + // synthetic QA identity so progress scenarios reach the draft path. + const qaAgent = { ...agent }; + delete qaAgent.identity; + return qaAgent; + }) + : baseCfg.agents?.list; + const execApprovalsConfig = approvalOverrides + ? { + enabled: true, + approvers: [params.driverBotUserId], + target: approvalOverrides.target ?? ("channel" as const), + } + : undefined; + const explicitToolAllow = baseCfg.tools?.allow; + const messageToolPolicy = params.overrides?.messageTool + ? explicitToolAllow && explicitToolAllow.length > 0 + ? { allow: uniqueStrings([...explicitToolAllow, "message"]) } + : { alsoAllow: uniqueStrings([...(baseCfg.tools?.alsoAllow ?? []), "message"]) } + : {}; + const toolsConfig = + codexApprovalConfig || params.overrides?.messageTool + ? { + tools: { + ...baseCfg.tools, + ...messageToolPolicy, + ...(codexApprovalConfig + ? { + exec: { + ...baseCfg.tools?.exec, + mode: "ask" as const, + }, + } + : {}), + }, + } + : {}; + return { + ...baseCfg, + ...approvalForwardingConfig, + ...toolsConfig, + plugins: { + ...baseCfg.plugins, + allow: pluginAllow, + entries: { + ...baseCfg.plugins?.entries, + slack: { enabled: true }, + ...(codexApprovalConfig + ? { + codex: { + ...codexEntry, + enabled: true, + config: { + ...codexEntryConfig, + appServer: { + ...codexAppServerConfig, + mode: "guardian" as const, + }, + }, + }, + } + : {}), + }, + }, + ...(codexApprovalConfig || progressOverrides + ? { + agents: { + ...baseCfg.agents, + ...(qaAgentDefaults ? { defaults: qaAgentDefaults } : {}), + ...(qaAgentList ? { list: qaAgentList } : {}), + }, + } + : {}), + messages: { + ...baseCfg.messages, + groupChat: { + ...baseCfg.messages?.groupChat, + visibleReplies: "automatic", + }, + }, + channels: { + ...baseCfg.channels, + slack: { + enabled: true, + defaultAccount: params.sutAccountId, + accounts: { + [params.sutAccountId]: { + enabled: true, + mode: "socket", + botToken: params.sutBotToken, + appToken: params.sutAppToken, + allowFrom: params.overrides?.allowFrom ?? [params.driverBotUserId], + groupPolicy: "allowlist", + allowBots: true, + replyToMode: params.overrides?.replyToMode ?? "off", + ...(progressOverrides + ? { + streaming: { + mode: "progress" as const, + progress: { + label: false, + maxLines: 4, + toolProgress: progressOverrides.toolProgress, + ...(progressOverrides.commentary === undefined + ? {} + : { commentary: progressOverrides.commentary }), + }, + }, + } + : {}), + ...(execApprovalsConfig ? { execApprovals: execApprovalsConfig } : {}), + channels: { + [params.channelId]: { + enabled: params.overrides?.channelEnabled ?? true, + requireMention: true, + allowBots: true, + users: params.overrides?.users ?? [params.driverBotUserId], + }, + }, + }, + }, + }, + }, + }; +} diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.contracts.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.contracts.ts new file mode 100644 index 000000000000..c54dad416f66 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.contracts.ts @@ -0,0 +1,405 @@ +// QA Lab Slack live domain contracts and wire schemas. +import type { WebClient } from "@slack/web-api"; +import { normalizeAccountId } from "openclaw/plugin-sdk/account-id"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { z } from "zod"; +import type { startQaGatewayChild } from "../../gateway-child.js"; +import { splitQaModelRef } from "../../model-selection.js"; +import type { RuntimeId } from "../../runtime-parity.js"; +import type { + acquireQaCredentialLease, + startQaCredentialLeaseHeartbeat, +} from "../shared/credential-lease.runtime.js"; +import type { startQaLiveLaneGateway } from "../shared/live-gateway.runtime.js"; +import type { LiveTransportScenarioDefinition } from "../shared/live-transport-scenarios.js"; + +export type SlackQaRuntimeEnv = { + channelId: string; + driverBotToken: string; + sutBotToken: string; + sutAppToken: string; +}; + +export type SlackChannelStatus = { + connected?: boolean; + lastConnectedAt?: number; + lastDisconnect?: unknown; + lastError?: string | null; + restartPending?: boolean; + running?: boolean; +}; + +export type SlackChannelReadinessMode = "connected" | "started"; + +export const SLACK_QA_DEFAULT_READY_TIMEOUT_MS = 45_000; +export const SLACK_QA_READY_STABILITY_MS = 3_000; +export const SLACK_QA_GATEWAY_STOP_SETTLE_MS = 3_000; +export const SLACK_QA_RETRYABLE_SCENARIO_ATTEMPTS = 2; +export const SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS = 30_000; +export const SLACK_QA_APPROVAL_CHECKPOINT_DEFAULT_TIMEOUT_MS = 120_000; +export const SLACK_QA_REACTION_VERIFY_TIMEOUT_MS = 15_000; +export const SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS = 15_000; +export const SLACK_QA_INVALID_TABLE_DATA_ROW_COUNT = 101; +export const SLACK_QA_LOG_TAIL_TIMEOUT_MS = 20_000; +export const SLACK_QA_INVALID_TABLE_CAPTION = "QA invalid_blocks fallback"; +export const SLACK_QA_INVALID_TABLE_HEADERS = ["Row", "Value"] as const; +export const SLACK_QA_CHART_TITLE = "QA latency trend"; +export const SLACK_QA_CHART_CATEGORIES = ["P50", "P95"] as const; +export const SLACK_QA_CHART_SERIES_NAME = "Latency"; +export const SLACK_QA_CHART_VALUES = [120, 240] as const; +export const SLACK_QA_CHART_X_LABEL = "Percentile"; +export const SLACK_QA_CHART_Y_LABEL = "Milliseconds"; +export const SLACK_QA_TABLE_CAPTION = "QA pipeline report"; +export const SLACK_QA_TABLE_HEADERS = ["Account", "Stage", "ARR"] as const; +export const SLACK_QA_TABLE_ROWS = [ + ["Acme", "Won", 125_000], + ["Globex", "Review", 82_000], +] as const; +export const SLACK_QA_NATIVE_CHART = { + type: "data_visualization", + title: SLACK_QA_CHART_TITLE, + chart: { + type: "line", + series: [ + { + name: SLACK_QA_CHART_SERIES_NAME, + data: [ + { label: SLACK_QA_CHART_CATEGORIES[0], value: SLACK_QA_CHART_VALUES[0] }, + { label: SLACK_QA_CHART_CATEGORIES[1], value: SLACK_QA_CHART_VALUES[1] }, + ], + }, + ], + axis_config: { + categories: [...SLACK_QA_CHART_CATEGORIES], + x_label: SLACK_QA_CHART_X_LABEL, + y_label: SLACK_QA_CHART_Y_LABEL, + }, + }, +} as const; +export const SLACK_QA_NATIVE_TABLE = { + type: "data_table", + caption: SLACK_QA_TABLE_CAPTION, + rows: [ + SLACK_QA_TABLE_HEADERS.map((text) => ({ type: "raw_text", text })), + ...SLACK_QA_TABLE_ROWS.map((row) => + row.map((cell) => + typeof cell === "number" + ? { type: "raw_number", value: cell, text: String(cell) } + : { type: "raw_text", text: cell }, + ), + ), + ], + row_header_column_index: 0, +} as const; +// These scenarios force the Codex harness, whose default provider set is intentionally narrow. +const SLACK_QA_CODEX_PROVIDER_IDS = new Set(["codex", "openai"]); + +export type SlackQaScenarioId = + | "slack-allowlist-block" + | "slack-approval-exec-native" + | "slack-approval-plugin-native" + | "slack-canary" + | "slack-codex-approval-exec-native" + | "slack-codex-approval-plugin-native" + | "slack-chart-presentation-native" + | "slack-channel-disabled-warning" + | "slack-mention-gating" + | "slack-progress-commentary-false" + | "slack-progress-commentary-omitted" + | "slack-progress-commentary-true" + | "slack-progress-commentary-verbose-dedupe" + | "slack-reaction-glyph-native" + | "slack-table-invalid-blocks-fallback" + | "slack-table-presentation-native" + | "slack-top-level-reply-shape"; + +export type SlackQaApprovalKind = "exec" | "plugin"; +export type SlackQaApprovalDecision = "allow-always" | "allow-once" | "deny"; +export const SLACK_QA_APPROVAL_ACTION_PREFIX = "openclaw:approval:v1:"; +export const SlackQaApprovalActionValueSchema = z + .object({ + approvalId: z.string().min(1), + approvalKind: z.enum(["exec", "plugin"]), + decision: z.enum(["allow-always", "allow-once", "deny"]), + }) + .strict(); +export type SlackQaCodexApprovalMethod = + | "item/commandExecution/requestApproval" + | "item/fileChange/requestApproval"; + +export function assertSlackCodexApprovalModelSupported(modelRef: string) { + const provider = splitQaModelRef(modelRef)?.provider.trim().toLowerCase(); + if (provider && SLACK_QA_CODEX_PROVIDER_IDS.has(provider)) { + return; + } + throw new Error( + `Slack Codex approval scenarios require an openai/* or codex/* model; received "${modelRef}".`, + ); +} + +export function resolveSlackQaSutAccountId(value?: string) { + return normalizeAccountId(value?.trim() || "sut"); +} + +export type SlackQaMessageScenarioRun = { + afterNoReply?: (context: SlackQaScenarioContext) => Promise; + kind?: "message"; + expectReply: boolean; + input: string; + matchText: string; + preserveGatewayDebug?: boolean; + settleObservedMs?: number; + verify?: (message: SlackMessage, context: { requestThreadTs: string; sentTs: string }) => void; + verifyObserved?: (params: { + finalMessage: SlackMessage; + messages: readonly SlackObservedMessage[]; + }) => string | void; + beforeRun?: (context: Omit) => Promise; + afterReply?: (message: SlackMessage, context: SlackQaScenarioContext) => Promise; +}; + +type SlackQaDirectTransportScenarioRun = { + kind: "direct-transport"; + execute: ( + context: SlackQaDirectTransportScenarioContext, + ) => Promise; +}; + +export type SlackQaDirectTransportScenarioContext = { + cfg: OpenClawConfig; + channelId: string; + sutAccountId: string; + sutIdentity: SlackAuthIdentity; + sutReadClient: WebClient; + sutWriteClient: WebClient; + timeoutMs: number; +}; + +export type SlackQaDirectTransportScenarioResult = { + details: string; + message: SlackMessage; +}; + +export type SlackQaApprovalScenarioRun = { + approvalKind: SlackQaApprovalKind; + decision: SlackQaApprovalDecision; + kind: "approval"; + token: string; +}; + +export type SlackQaCodexApprovalScenarioRun = { + approvalKind: "plugin"; + appServerMethod: SlackQaCodexApprovalMethod; + decision: "allow-once"; + kind: "codex-approval"; + token: string; +}; + +type SlackQaScenarioRun = + | SlackQaApprovalScenarioRun + | SlackQaCodexApprovalScenarioRun + | SlackQaDirectTransportScenarioRun + | SlackQaMessageScenarioRun; + +type SlackQaBeforeRunResult = + | string + | void + | { + details?: string; + inputThreadTs?: string; + }; + +export type SlackQaConfigOverrides = { + allowFrom?: string[]; + channelEnabled?: boolean; + approvals?: { + exec?: boolean; + plugin?: boolean; + target?: "both" | "channel" | "dm"; + }; + codexApproval?: boolean; + messageTool?: boolean; + progress?: { + commentary?: boolean; + toolProgress: boolean; + verboseDefault?: "off" | "on" | "full"; + }; + replyToMode?: "all" | "off"; + users?: string[]; +}; + +export type SlackQaScenarioContext = { + channelId: string; + driverClient: WebClient; + gateway: Awaited>; + postSlackMessage: (params: { text: string; threadTs?: string }) => Promise<{ ts: string }>; + sentTs: string; + sutIdentity: SlackAuthIdentity; + sutReadClient: WebClient; + waitForReady: () => Promise; +}; + +export type SlackQaScenarioDefinition = LiveTransportScenarioDefinition & { + buildRun: (sutUserId: string) => SlackQaScenarioRun; + configOverrides?: SlackQaConfigOverrides; + defaultEnabled?: boolean; + forcedRuntime?: RuntimeId; +}; + +export type SlackQaGatewayHarness = Awaited>; + +export type SlackAuthIdentity = { + botId?: string; + teamId?: string; + userId: string; +}; + +export type SlackObservedMessage = { + botId?: string; + channelId: string; + matchedScenario?: boolean; + scenarioId?: string; + scenarioTitle?: string; + text: string; + actionValues?: string[]; + blockText?: string[]; + threadTs?: string; + ts: string; + userId?: string; +}; + +export type SlackObservedMessageArtifact = { + botId?: string; + channelId?: string; + matchedScenario?: boolean; + scenarioId?: string; + scenarioTitle?: string; + text?: string; + actionValues?: string[]; + blockText?: string[]; + threadTs?: string; + ts?: string; + userId?: string; +}; + +export type SlackApprovalArtifact = { + approvalId: string; + approvalKind: SlackQaApprovalKind; + appServerMethod?: SlackQaCodexApprovalMethod; + channelId?: string; + codexModelKey?: string; + decision: SlackQaApprovalDecision; + finalCodexTurnStatus?: string; + operationVerified?: boolean; + pendingActionValues?: string[]; + pendingCheckpointPath?: string; + pendingMessageTs?: string; + pendingScreenshotPath?: string; + pendingText?: string; + resolvedActionValues?: string[]; + resolvedCheckpointPath?: string; + resolvedMessageTs?: string; + resolvedScreenshotPath?: string; + resolvedText?: string; + threadTs?: string; +}; + +export type SlackApprovalCheckpointState = "pending" | "resolved"; + +export type SlackApprovalCheckpointAck = { + capturedAt?: string; + screenshotPath?: string; +}; + +export type SlackApprovalCheckpointMessage = { + actionLabels: string[]; + blockText: string[]; + hasNativeActions: boolean; + text: string; +}; + +export type SlackQaScenarioResult = { + approval?: SlackApprovalArtifact; + details: string; + id: string; + requestStartedAt?: string; + responseObservedAt?: string; + rttMs?: number; + rttMeasurement?: { + finalMatchedReplyRttMs: number; + requestStartedAt: string; + responseObservedAt: string; + source: "approval-request-to-resolution" | "request-to-observed-message"; + }; + standardId?: string; + status: "fail" | "pass"; + title: string; +}; + +export type SlackQaRunResult = { + gatewayDebugDirPath?: string; + observedMessagesPath: string; + outputDir: string; + reportPath: string; + scenarios: SlackQaScenarioResult[]; + summaryPath: string; +}; + +export type SlackCredentialLease = Awaited< + ReturnType> +>; +export type SlackCredentialHeartbeat = ReturnType; + +export const SLACK_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_SLACK_CAPTURE_CONTENT"; +export const SLACK_QA_APPROVAL_CHECKPOINT_DIR_ENV = "OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_DIR"; +export const SLACK_QA_APPROVAL_CHECKPOINT_TIMEOUT_MS_ENV = + "OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_TIMEOUT_MS"; +export const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA"; +export const SLACK_QA_WEB_API_TIMEOUT_MS = 45_000; +export const SLACK_QA_ENV_KEYS = [ + "OPENCLAW_QA_SLACK_CHANNEL_ID", + "OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN", + "OPENCLAW_QA_SLACK_SUT_BOT_TOKEN", + "OPENCLAW_QA_SLACK_SUT_APP_TOKEN", +] as const; + +export const slackQaCredentialPayloadSchema = z.object({ + channelId: z.string().trim().min(1), + driverBotToken: z.string().trim().min(1), + sutBotToken: z.string().trim().min(1), + sutAppToken: z.string().trim().min(1), +}); + +export const slackAuthTestSchema = z.object({ + ok: z.boolean().optional(), + user_id: z.string().optional(), + bot_id: z.string().optional(), + team_id: z.string().optional(), +}); + +export const slackPostMessageSchema = z.object({ + ok: z.boolean().optional(), + channel: z.string().optional(), + ts: z.string().min(1), +}); + +const slackHistoryMessageSchema = z.object({ + bot_id: z.string().optional(), + blocks: z.array(z.unknown()).optional(), + text: z.string().optional(), + thread_ts: z.string().optional(), + ts: z.string().min(1), + user: z.string().optional(), +}); + +export type SlackMessage = Omit, "ts"> & { ts?: string }; + +export const slackHistorySchema = z.object({ + ok: z.boolean().optional(), + messages: z.array(slackHistoryMessageSchema).optional(), +}); + +export const slackRepliesSchema = z.object({ + ok: z.boolean().optional(), + messages: z.array(slackHistoryMessageSchema).optional(), +}); diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.invalid-blocks.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.invalid-blocks.ts new file mode 100644 index 000000000000..0a032b980c58 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.invalid-blocks.ts @@ -0,0 +1,43 @@ +// QA Lab Slack invalid-blocks fallback fixture. +import { randomUUID } from "node:crypto"; +import { + SLACK_QA_INVALID_TABLE_DATA_ROW_COUNT, + SLACK_QA_INVALID_TABLE_CAPTION, + SLACK_QA_INVALID_TABLE_HEADERS, +} from "./slack-live.contracts.js"; + +function buildSlackInvalidBlocksTableRow(index: number) { + const rowId = String(index).padStart(3, "0"); + return [`row-${rowId}`, `value-${rowId}`] as const; +} + +export function buildSlackInvalidBlocksTableProbe() { + const summaryText = `SLACK_QA_TABLE_INVALID_BLOCKS_${randomUUID().slice(0, 8).toUpperCase()}`; + const dataRows = Array.from({ length: SLACK_QA_INVALID_TABLE_DATA_ROW_COUNT }, (_entry, index) => + buildSlackInvalidBlocksTableRow(index + 1), + ); + const block = { + type: "data_table", + caption: SLACK_QA_INVALID_TABLE_CAPTION, + rows: [ + SLACK_QA_INVALID_TABLE_HEADERS.map((text) => ({ type: "raw_text", text })), + ...dataRows.map((row) => row.map((text) => ({ type: "raw_text", text }))), + ], + row_header_column_index: 0, + } as const; + const fallbackText = [ + summaryText, + "", + `${SLACK_QA_INVALID_TABLE_CAPTION} (table)`, + SLACK_QA_INVALID_TABLE_HEADERS.join("\t"), + ...dataRows.map((row) => row.join("\t")), + ].join("\n"); + return { + block, + dataRowCount: dataRows.length, + fallbackText, + firstRowText: buildSlackInvalidBlocksTableRow(1).join("\t"), + finalRowText: buildSlackInvalidBlocksTableRow(SLACK_QA_INVALID_TABLE_DATA_ROW_COUNT).join("\t"), + summaryText, + }; +} diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.message-observations.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.message-observations.ts new file mode 100644 index 000000000000..51fe27d09855 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.message-observations.ts @@ -0,0 +1,347 @@ +// QA Lab Slack scenario reply observation and channel readiness. +import type { WebClient } from "@slack/web-api"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { parseStrictPositiveInteger } from "openclaw/plugin-sdk/number-runtime"; +import type { startQaGatewayChild } from "../../gateway-child.js"; +import { + type SlackChannelStatus, + type SlackChannelReadinessMode, + SLACK_QA_DEFAULT_READY_TIMEOUT_MS, + SLACK_QA_READY_STABILITY_MS, + type SlackAuthIdentity, + type SlackObservedMessage, + type SlackMessage, +} from "./slack-live.contracts.js"; +import { + listSlackMessages, + listSlackThreadMessages, + collectSlackBlockText, + collectSlackActionValues, + isSutSlackMessage, +} from "./slack-live.observations.js"; + +type SlackScenarioObservationContext = { + channelId: string; + matchText: string; + observedMessages: SlackObservedMessage[]; + observationScenarioId: string; + observationScenarioTitle: string; + sentTs: string; + sutIdentity: SlackAuthIdentity; +}; + +function recordSlackScenarioMessages( + params: SlackScenarioObservationContext & { messages: SlackMessage[] }, +) { + let matchedMessage: SlackMessage | undefined; + for (const message of params.messages) { + const text = message.text ?? ""; + if ( + !message.ts || + message.ts === params.sentTs || + !isSutSlackMessage(message, params.sutIdentity) + ) { + continue; + } + const matchedScenario = text.includes(params.matchText); + params.observedMessages.push({ + actionValues: collectSlackActionValues(message.blocks), + blockText: collectSlackBlockText(message.blocks), + botId: message.bot_id, + channelId: params.channelId, + matchedScenario, + scenarioId: params.observationScenarioId, + scenarioTitle: params.observationScenarioTitle, + text, + threadTs: message.thread_ts, + ts: message.ts, + userId: message.user, + }); + if (matchedScenario && !matchedMessage) { + matchedMessage = message; + } + } + return matchedMessage; +} + +export async function waitForSlackScenarioReply( + params: SlackScenarioObservationContext & { + client: WebClient; + threadTs?: string; + timeoutMs: number; + }, +) { + const observationContext: SlackScenarioObservationContext = params; + const startedAt = Date.now(); + const inspectMessages = (messages: SlackMessage[]) => { + const matchedMessage = recordSlackScenarioMessages({ ...observationContext, messages }); + return matchedMessage + ? { message: matchedMessage, observedAt: new Date().toISOString() } + : undefined; + }; + + while (Date.now() - startedAt < params.timeoutMs) { + const channelMessages = await listSlackMessages({ + channelId: params.channelId, + client: params.client, + oldestTs: params.sentTs, + }); + const channelReply = inspectMessages(channelMessages); + if (channelReply) { + return channelReply; + } + + try { + const threadMessages = await listSlackThreadMessages({ + channelId: params.channelId, + client: params.client, + threadTs: params.threadTs ?? params.sentTs, + }); + const threadReply = inspectMessages(threadMessages); + if (threadReply) { + return threadReply; + } + } catch (error) { + throw new Error( + `Slack conversations.replies failed while waiting for ${params.observationScenarioId}: ${formatErrorMessage(error)}`, + { cause: error }, + ); + } + await new Promise((resolve) => { + setTimeout(resolve, 1_000); + }); + } + throw new Error(`timed out after ${params.timeoutMs}ms waiting for Slack message`); +} + +export async function observeSlackScenarioMessages( + params: SlackScenarioObservationContext & { + client: WebClient; + settleMs: number; + threadTs?: string; + }, +) { + const observationContext: SlackScenarioObservationContext = params; + const startedAt = Date.now(); + + while (true) { + recordSlackScenarioMessages({ + ...observationContext, + messages: await listSlackMessages({ + channelId: params.channelId, + client: params.client, + oldestTs: params.sentTs, + }), + }); + try { + recordSlackScenarioMessages({ + ...observationContext, + messages: await listSlackThreadMessages({ + channelId: params.channelId, + client: params.client, + threadTs: params.threadTs ?? params.sentTs, + }), + }); + } catch (error) { + throw new Error( + `Slack conversations.replies failed while settling ${params.observationScenarioId}: ${formatErrorMessage(error)}`, + { cause: error }, + ); + } + const remainingMs = params.settleMs - (Date.now() - startedAt); + if (remainingMs <= 0) { + return; + } + await new Promise((resolve) => { + setTimeout(resolve, Math.min(1_000, remainingMs)); + }); + } +} + +export async function waitForSlackNoReply(params: { + channelId: string; + client: WebClient; + matchText: string; + observedMessages: SlackObservedMessage[]; + observationScenarioId: string; + observationScenarioTitle: string; + sentTs: string; + sutIdentity: SlackAuthIdentity; + timeoutMs: number; +}) { + const startedAt = Date.now(); + const observedKeys = new Set( + params.observedMessages + .map((message) => `${message.channelId ?? params.channelId}:${message.ts ?? ""}`) + .filter((key) => !key.endsWith(":")), + ); + let elapsedMs = Date.now() - startedAt; + while (elapsedMs < params.timeoutMs) { + const messages = await listSlackMessages({ + channelId: params.channelId, + client: params.client, + oldestTs: params.sentTs, + }); + for (const message of messages) { + const text = message.text ?? ""; + if ( + !message.ts || + message.ts === params.sentTs || + !isSutSlackMessage(message, params.sutIdentity) + ) { + continue; + } + const matchedScenario = text.includes(params.matchText); + const observedKey = `${params.channelId}:${message.ts}`; + if (!observedKeys.has(observedKey)) { + observedKeys.add(observedKey); + params.observedMessages.push({ + actionValues: collectSlackActionValues(message.blocks), + blockText: collectSlackBlockText(message.blocks), + botId: message.bot_id, + channelId: params.channelId, + matchedScenario, + scenarioId: params.observationScenarioId, + scenarioTitle: params.observationScenarioTitle, + text, + threadTs: message.thread_ts, + ts: message.ts, + userId: message.user, + }); + } + if (matchedScenario) { + throw new Error("unexpected Slack SUT reply observed"); + } + } + elapsedMs = Date.now() - startedAt; + const remainingMs = params.timeoutMs - elapsedMs; + if (remainingMs > 0) { + await new Promise((resolve) => { + setTimeout(resolve, Math.min(1_000, remainingMs)); + }); + } + elapsedMs = Date.now() - startedAt; + } +} + +async function waitForSlackChannelRunning( + gateway: Awaited>, + accountId: string, + mode: SlackChannelReadinessMode, +): Promise { + const startedAt = Date.now(); + const timeoutMs = resolveSlackQaReadyTimeoutMs(); + let lastStatus: SlackChannelStatus | undefined; + while (Date.now() - startedAt < timeoutMs) { + try { + const payload = (await gateway.call( + "channels.status", + { probe: false, timeoutMs: 2_000 }, + { timeoutMs: 5_000 }, + )) as { + channelAccounts?: Record< + string, + Array<{ + accountId?: string; + connected?: boolean; + lastConnectedAt?: number; + lastDisconnect?: unknown; + lastError?: string | null; + restartPending?: boolean; + running?: boolean; + }> + >; + }; + const accounts = payload.channelAccounts?.slack ?? []; + const match = accounts.find((entry) => entry.accountId === accountId); + lastStatus = match + ? { + connected: match.connected, + lastConnectedAt: match.lastConnectedAt, + lastDisconnect: match.lastDisconnect, + lastError: match.lastError, + restartPending: match.restartPending, + running: match.running, + } + : undefined; + if (isSlackChannelReadyForQa(lastStatus, mode)) { + if (!lastStatus) { + throw new Error(`slack account "${accountId}" status disappeared after readiness check`); + } + return lastStatus; + } + } catch { + // retry + } + await new Promise((resolve) => { + setTimeout(resolve, 500); + }); + } + throw new Error( + `slack account "${accountId}" did not become ready` + + (lastStatus ? `; last status: ${JSON.stringify(lastStatus)}` : ""), + ); +} + +export async function waitForSlackChannelStable( + gateway: Awaited>, + accountId: string, + mode: SlackChannelReadinessMode, +) { + const startedAt = Date.now(); + const timeoutMs = resolveSlackQaReadyTimeoutMs(); + let readySince: number | undefined; + while (Date.now() - startedAt < timeoutMs) { + const status = await waitForSlackChannelRunning(gateway, accountId, mode); + const observedAt = Date.now(); + readySince = resolveSlackChannelReadySince({ + observedAt, + previousReadySince: readySince, + status, + }); + const readyForMs = observedAt - readySince; + if (readyForMs >= SLACK_QA_READY_STABILITY_MS) { + return; + } + await new Promise((resolve) => { + setTimeout(resolve, Math.max(500, SLACK_QA_READY_STABILITY_MS - readyForMs)); + }); + } + throw new Error( + `slack account "${accountId}" did not remain ready for ${SLACK_QA_READY_STABILITY_MS}ms`, + ); +} + +export function isSlackChannelReadyForQa( + status: SlackChannelStatus | undefined, + mode: SlackChannelReadinessMode, +): boolean { + if ( + !status?.running || + status.restartPending === true || + status.lastError != null || + status.connected === false + ) { + return false; + } + return mode === "started" || status.connected === true; +} + +export function resolveSlackChannelReadySince(params: { + observedAt: number; + previousReadySince: number | undefined; + status: SlackChannelStatus; +}): number { + if (typeof params.status.lastConnectedAt === "number" && params.status.lastConnectedAt > 0) { + return params.status.lastConnectedAt; + } + return params.previousReadySince ?? params.observedAt; +} + +export function resolveSlackQaReadyTimeoutMs(env: NodeJS.ProcessEnv = process.env) { + const raw = env.OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS; + if (!raw) { + return SLACK_QA_DEFAULT_READY_TIMEOUT_MS; + } + return parseStrictPositiveInteger(raw) ?? SLACK_QA_DEFAULT_READY_TIMEOUT_MS; +} diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.observations.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.observations.ts new file mode 100644 index 000000000000..71510283fb71 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.observations.ts @@ -0,0 +1,395 @@ +// QA Lab Slack Web API and stored-message observations. +import { isDeepStrictEqual } from "node:util"; +import { createSlackWebClient, sendSlackMessage } from "@openclaw/slack/api.js"; +import type { WebClient } from "@slack/web-api"; +import { + asPlainRecord, + countSlackNativeDataBlocks, + instrumentSlackPostMessage, +} from "./slack-live.config.js"; +import { + SLACK_QA_NATIVE_CHART, + SLACK_QA_NATIVE_TABLE, + type SlackQaApprovalDecision, + SLACK_QA_APPROVAL_ACTION_PREFIX, + SlackQaApprovalActionValueSchema, + type SlackQaDirectTransportScenarioContext, + type SlackQaDirectTransportScenarioResult, + type SlackAuthIdentity, + type SlackApprovalCheckpointMessage, + SLACK_QA_WEB_API_TIMEOUT_MS, + slackAuthTestSchema, + slackPostMessageSchema, + type SlackMessage, + slackHistorySchema, + slackRepliesSchema, +} from "./slack-live.contracts.js"; +import { buildSlackInvalidBlocksTableProbe } from "./slack-live.invalid-blocks.js"; + +export async function getSlackIdentity(token: string): Promise { + const client = createSlackWebClient(token, { timeout: SLACK_QA_WEB_API_TIMEOUT_MS }); + const auth = slackAuthTestSchema.parse(await client.auth.test()); + if (!auth.user_id) { + throw new Error("Slack auth.test did not return user_id."); + } + return { + userId: auth.user_id, + botId: auth.bot_id, + teamId: auth.team_id, + }; +} + +export async function sendSlackChannelMessage(params: { + channelId: string; + client: WebClient; + text: string; + threadTs?: string; +}) { + const postSlackMessage = params.client.chat.postMessage.bind(params.client.chat); + const sent = slackPostMessageSchema.parse( + await postSlackMessage({ + channel: params.channelId, + text: params.text, + thread_ts: params.threadTs, + unfurl_links: false, + unfurl_media: false, + }), + ); + return { + channelId: sent.channel ?? params.channelId, + ts: sent.ts, + }; +} + +export async function listSlackMessages(params: { + channelId: string; + client: WebClient; + oldestTs: string; +}) { + const history = slackHistorySchema.parse( + await params.client.conversations.history({ + channel: params.channelId, + inclusive: true, + limit: 50, + oldest: params.oldestTs, + }), + ); + return history.messages ?? []; +} + +export async function listSlackThreadMessages(params: { + channelId: string; + client: WebClient; + threadTs: string; +}) { + const replies = slackRepliesSchema.parse( + await params.client.conversations.replies({ + channel: params.channelId, + inclusive: true, + limit: 50, + ts: params.threadTs, + }), + ); + return replies.messages ?? []; +} + +function collectSlackBlockStringFields( + value: unknown, + fieldName: string, + values: string[] = [], +): string[] { + if (Array.isArray(value)) { + for (const entry of value) { + collectSlackBlockStringFields(entry, fieldName, values); + } + return values; + } + if (!value || typeof value !== "object") { + return values; + } + for (const [key, entry] of Object.entries(value)) { + if (key === fieldName && typeof entry === "string" && entry.trim().length > 0) { + values.push(entry); + continue; + } + collectSlackBlockStringFields(entry, fieldName, values); + } + return values; +} + +export function collectSlackBlockText(blocks?: unknown[]) { + return collectSlackBlockStringFields(blocks ?? [], "text"); +} + +export function collectSlackActionValues(blocks?: unknown[]) { + return collectSlackBlockStringFields(blocks ?? [], "value"); +} + +export function parseSlackNativeApprovalAction(value: string) { + if (!value.startsWith(SLACK_QA_APPROVAL_ACTION_PREFIX)) { + return undefined; + } + try { + const decoded: unknown = JSON.parse(value.slice(SLACK_QA_APPROVAL_ACTION_PREFIX.length)); + const parsed = SlackQaApprovalActionValueSchema.safeParse(decoded); + return parsed.success ? parsed.data : undefined; + } catch { + return undefined; + } +} + +export function collectSlackButtonLabels(blocks?: unknown[]) { + const labels: string[] = []; + function visit(value: unknown) { + if (Array.isArray(value)) { + for (const entry of value) { + visit(entry); + } + return; + } + if (!value || typeof value !== "object") { + return; + } + const candidate = value as Record; + if (candidate.type === "button") { + const text = candidate.text; + if (text && typeof text === "object") { + const label = (text as { text?: unknown }).text; + if (typeof label === "string" && label.trim().length > 0) { + labels.push(label); + } + } + } + for (const entry of Object.values(candidate)) { + visit(entry); + } + } + visit(blocks ?? []); + return labels; +} + +export function buildSlackApprovalCheckpointMessage( + message: SlackMessage, +): SlackApprovalCheckpointMessage { + const actionValues = collectSlackActionValues(message.blocks); + return { + actionLabels: collectSlackButtonLabels(message.blocks), + blockText: collectSlackBlockText(message.blocks), + hasNativeActions: actionValues.some((value) => parseSlackNativeApprovalAction(value)), + text: message.text ?? "", + }; +} + +export function hasSlackNativeApprovalActions(params: { + actionValues: string[]; + approvalId?: string; + decision: SlackQaApprovalDecision; +}) { + return params.actionValues.some((value) => { + const action = parseSlackNativeApprovalAction(value); + return ( + action?.decision === params.decision && + (!params.approvalId || action.approvalId === params.approvalId) + ); + }); +} + +export function extractSlackNativeApprovalId(params: { + actionValues: string[]; + decision: SlackQaApprovalDecision; +}) { + for (const value of params.actionValues) { + const action = parseSlackNativeApprovalAction(value); + if (action?.decision === params.decision) { + return action.approvalId; + } + } + return undefined; +} + +export function isSutSlackMessage(message: SlackMessage, sutIdentity: SlackAuthIdentity) { + return ( + (message.user !== undefined && message.user === sutIdentity.userId) || + (message.bot_id !== undefined && message.bot_id === sutIdentity.botId) + ); +} + +// Slack history can flatten top-level accessibility newlines on readback. +// Normalize only whitespace; the native chart structure stays byte-for-byte strict below. +function normalizeSlackAccessibleText(value: string) { + return value.trim().replace(/\s+/gu, " "); +} + +export function isExpectedSlackNativeChartMessage( + message: SlackMessage, + expectedAccessibleText: string, +) { + if ( + normalizeSlackAccessibleText(message.text ?? "") !== + normalizeSlackAccessibleText(expectedAccessibleText) + ) { + return false; + } + return (message.blocks ?? []).some((value) => { + const block = asPlainRecord(value); + return isDeepStrictEqual( + { type: block.type, title: block.title, chart: block.chart }, + SLACK_QA_NATIVE_CHART, + ); + }); +} + +export async function waitForSlackStoredMessage(params: { + channelId: string; + client: WebClient; + description: string; + matchesMessage: (message: SlackMessage) => boolean; + oldestTs: string; + sutIdentity: SlackAuthIdentity; + timeoutMs: number; +}) { + const startedAt = Date.now(); + while (true) { + const messages = await listSlackMessages({ + channelId: params.channelId, + client: params.client, + oldestTs: params.oldestTs, + }); + const message = messages.find( + (entry) => + entry.ts !== params.oldestTs && + isSutSlackMessage(entry, params.sutIdentity) && + params.matchesMessage(entry), + ); + if (message) { + return message; + } + const remainingMs = params.timeoutMs - (Date.now() - startedAt); + if (remainingMs <= 0) { + break; + } + await new Promise((resolve) => { + setTimeout(resolve, Math.min(1_000, remainingMs)); + }); + } + throw new Error(`timed out after ${params.timeoutMs}ms waiting for Slack ${params.description}`); +} + +export function isExpectedSlackNativeTableMessage( + message: SlackMessage, + expectedAccessibleText: string, +) { + if ( + normalizeSlackAccessibleText(message.text ?? "") !== + normalizeSlackAccessibleText(expectedAccessibleText) + ) { + return false; + } + return (message.blocks ?? []).some((value) => { + const block = asPlainRecord(value); + return isDeepStrictEqual( + { + type: block.type, + caption: block.caption, + rows: block.rows, + row_header_column_index: block.row_header_column_index, + }, + SLACK_QA_NATIVE_TABLE, + ); + }); +} + +export async function runSlackTableInvalidBlocksFallbackScenario( + context: SlackQaDirectTransportScenarioContext, +): Promise { + const probe = buildSlackInvalidBlocksTableProbe(); + const oldestTs = ((Date.now() - 5_000) / 1_000).toFixed(6); + const instrumentation = instrumentSlackPostMessage(context.sutWriteClient); + let sent: Awaited>; + try { + try { + sent = await sendSlackMessage(`channel:${context.channelId}`, probe.summaryText, { + accountId: context.sutAccountId, + blocks: [probe.block] as never, + cfg: context.cfg, + client: context.sutWriteClient, + nativeDataFallbackBaseText: probe.summaryText, + }); + } catch { + const [nativeAttempt, fallbackAttempt] = instrumentation.attempts; + if (nativeAttempt?.failureCode !== "invalid_blocks") { + throw new Error( + `expected first Slack API failure code invalid_blocks; observed ${nativeAttempt?.failureCode ?? "none"}`, + ); + } + throw new Error( + `Slack fallback failed after invalid_blocks; observed ${fallbackAttempt?.failureCode ?? "no fallback API failure code"}`, + ); + } + } finally { + instrumentation.restore(); + } + + const [nativeAttempt, fallbackAttempt] = instrumentation.attempts; + if (instrumentation.attempts.length !== 2) { + throw new Error( + `expected exactly two Slack API attempts; observed ${instrumentation.attempts.length}`, + ); + } + if ( + nativeAttempt?.status !== "failed" || + nativeAttempt.failureCode !== "invalid_blocks" || + nativeAttempt.nativeDataBlockCount !== 1 + ) { + throw new Error( + `expected first Slack API attempt to fail with invalid_blocks for one native data block; observed ${nativeAttempt?.failureCode ?? "none"}`, + ); + } + if ( + fallbackAttempt?.status !== "sent" || + fallbackAttempt.nativeDataBlockCount !== 0 || + !fallbackAttempt.formattingDisabled + ) { + throw new Error("Slack fallback did not use one formatting-disabled blockless API request"); + } + + const message = await waitForSlackStoredMessage({ + channelId: context.channelId, + client: context.sutReadClient, + description: "stored invalid_blocks fallback message", + matchesMessage: (candidate) => candidate.ts === sent.messageId, + oldestTs, + sutIdentity: context.sutIdentity, + timeoutMs: context.timeoutMs, + }); + const storedText = message.text ?? ""; + if (countSlackNativeDataBlocks(message.blocks) !== 0) { + throw new Error("stored Slack fallback retained a native data block"); + } + const storedLines = storedText.split("\n"); + if (!storedLines.includes(probe.firstRowText)) { + throw new Error("stored Slack fallback omitted the exact first data row"); + } + if (!storedLines.includes(probe.finalRowText)) { + throw new Error("stored Slack fallback omitted the exact final data row"); + } + if (storedText !== probe.fallbackText) { + throw new Error( + `stored Slack fallback was incomplete: expected ${probe.fallbackText.length} characters, observed ${storedText.length}`, + ); + } + return { + details: [ + "direct transport", + "first API failure=invalid_blocks", + "API attempts=2", + `data rows=${probe.dataRowCount}`, + "fallback formatting disabled=true", + "stored native data blocks=0", + "first row=present", + "final row=present", + "complete delivery=true", + ].join("; "), + message, + }; +} diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts index 3a15e68569fd..227fadf47e03 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts @@ -1,3179 +1,100 @@ -// Qa Lab plugin module implements slack live behavior. -import { randomUUID } from "node:crypto"; +// QA Lab Slack live orchestration and stable public test surface. import fs from "node:fs/promises"; -import os from "node:os"; import path from "node:path"; -import { isDeepStrictEqual } from "node:util"; -import { - createSlackWebClient, - createSlackWriteClient, - listSlackReactions, - sendSlackMessage, -} from "@openclaw/slack/api.js"; -import type { WebClient } from "@slack/web-api"; -import { normalizeAccountId } from "openclaw/plugin-sdk/account-id"; -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { createSlackWebClient, createSlackWriteClient } from "@openclaw/slack/api.js"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; -import { parseStrictPositiveInteger } from "openclaw/plugin-sdk/number-runtime"; -import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { z } from "zod"; import { createQaArtifactRunId } from "../../artifact-run-id.js"; import { QA_EVIDENCE_FILENAME, buildLiveTransportEvidenceSummary } from "../../evidence-summary.js"; -import { startQaGatewayChild } from "../../gateway-child.js"; -import { extractGatewayMessageText } from "../../gateway-log-sentinel.js"; import { isTruthyOptIn } from "../../mantis-options.runtime.js"; -import { splitQaModelRef } from "../../model-selection.js"; import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js"; import { defaultQaModelForMode, normalizeQaProviderMode, type QaProviderModeInput, } from "../../run-config.js"; -import type { RuntimeId } from "../../runtime-parity.js"; import { acquireQaCredentialLease, startQaCredentialLeaseHeartbeat, } from "../shared/credential-lease.runtime.js"; -import { - assertApprovalDecisionResult, - formatApprovalResultValue, - readAcceptedApprovalRequestId, -} from "../shared/live-approval-result.js"; import { appendQaLiveLaneIssue as appendLiveLaneIssue, buildQaLiveLaneArtifactsError as buildLiveLaneArtifactsError, } from "../shared/live-artifacts.js"; import { inferQaCredentialSource as inferSlackCredentialSource } from "../shared/live-credential-source.js"; import { startQaLiveLaneGateway } from "../shared/live-gateway.runtime.js"; +import { resolveSlackApprovalCheckpointConfig } from "./slack-live.approval-checkpoint.js"; import { - collectLiveTransportStandardScenarioCoverage, - selectLiveTransportScenarios, - type LiveTransportScenarioDefinition, -} from "../shared/live-transport-scenarios.js"; - -type SlackQaRuntimeEnv = { - channelId: string; - driverBotToken: string; - sutBotToken: string; - sutAppToken: string; -}; - -type SlackChannelStatus = { - connected?: boolean; - lastConnectedAt?: number; - lastDisconnect?: unknown; - lastError?: string | null; - restartPending?: boolean; - running?: boolean; -}; - -type SlackChannelReadinessMode = "connected" | "started"; - -const SLACK_QA_DEFAULT_READY_TIMEOUT_MS = 45_000; -const SLACK_QA_READY_STABILITY_MS = 3_000; -const SLACK_QA_GATEWAY_STOP_SETTLE_MS = 3_000; -const SLACK_QA_RETRYABLE_SCENARIO_ATTEMPTS = 2; -const SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS = 30_000; -const SLACK_QA_APPROVAL_CHECKPOINT_DEFAULT_TIMEOUT_MS = 120_000; -const SLACK_QA_REACTION_VERIFY_TIMEOUT_MS = 15_000; -const SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS = 15_000; -const SLACK_QA_INVALID_TABLE_DATA_ROW_COUNT = 101; -const SLACK_QA_LOG_TAIL_TIMEOUT_MS = 20_000; -const SLACK_QA_INVALID_TABLE_CAPTION = "QA invalid_blocks fallback"; -const SLACK_QA_INVALID_TABLE_HEADERS = ["Row", "Value"] as const; -const SLACK_QA_CHART_TITLE = "QA latency trend"; -const SLACK_QA_CHART_CATEGORIES = ["P50", "P95"] as const; -const SLACK_QA_CHART_SERIES_NAME = "Latency"; -const SLACK_QA_CHART_VALUES = [120, 240] as const; -const SLACK_QA_CHART_X_LABEL = "Percentile"; -const SLACK_QA_CHART_Y_LABEL = "Milliseconds"; -const SLACK_QA_TABLE_CAPTION = "QA pipeline report"; -const SLACK_QA_TABLE_HEADERS = ["Account", "Stage", "ARR"] as const; -const SLACK_QA_TABLE_ROWS = [ - ["Acme", "Won", 125_000], - ["Globex", "Review", 82_000], -] as const; -const SLACK_QA_NATIVE_CHART = { - type: "data_visualization", - title: SLACK_QA_CHART_TITLE, - chart: { - type: "line", - series: [ - { - name: SLACK_QA_CHART_SERIES_NAME, - data: [ - { label: SLACK_QA_CHART_CATEGORIES[0], value: SLACK_QA_CHART_VALUES[0] }, - { label: SLACK_QA_CHART_CATEGORIES[1], value: SLACK_QA_CHART_VALUES[1] }, - ], - }, - ], - axis_config: { - categories: [...SLACK_QA_CHART_CATEGORIES], - x_label: SLACK_QA_CHART_X_LABEL, - y_label: SLACK_QA_CHART_Y_LABEL, - }, - }, -} as const; -const SLACK_QA_NATIVE_TABLE = { - type: "data_table", - caption: SLACK_QA_TABLE_CAPTION, - rows: [ - SLACK_QA_TABLE_HEADERS.map((text) => ({ type: "raw_text", text })), - ...SLACK_QA_TABLE_ROWS.map((row) => - row.map((cell) => - typeof cell === "number" - ? { type: "raw_number", value: cell, text: String(cell) } - : { type: "raw_text", text: cell }, - ), - ), - ], - row_header_column_index: 0, -} as const; -// These scenarios force the Codex harness, whose default provider set is intentionally narrow. -const SLACK_QA_CODEX_PROVIDER_IDS = new Set(["codex", "openai"]); - -type SlackQaScenarioId = - | "slack-allowlist-block" - | "slack-approval-exec-native" - | "slack-approval-plugin-native" - | "slack-canary" - | "slack-codex-approval-exec-native" - | "slack-codex-approval-plugin-native" - | "slack-chart-presentation-native" - | "slack-channel-disabled-warning" - | "slack-mention-gating" - | "slack-progress-commentary-false" - | "slack-progress-commentary-omitted" - | "slack-progress-commentary-true" - | "slack-progress-commentary-verbose-dedupe" - | "slack-reaction-glyph-native" - | "slack-table-invalid-blocks-fallback" - | "slack-table-presentation-native" - | "slack-top-level-reply-shape"; - -type SlackQaApprovalKind = "exec" | "plugin"; -type SlackQaApprovalDecision = "allow-always" | "allow-once" | "deny"; -const SLACK_QA_APPROVAL_ACTION_PREFIX = "openclaw:approval:v1:"; -const SlackQaApprovalActionValueSchema = z - .object({ - approvalId: z.string().min(1), - approvalKind: z.enum(["exec", "plugin"]), - decision: z.enum(["allow-always", "allow-once", "deny"]), - }) - .strict(); -type SlackQaCodexApprovalMethod = - | "item/commandExecution/requestApproval" - | "item/fileChange/requestApproval"; - -function assertSlackCodexApprovalModelSupported(modelRef: string) { - const provider = splitQaModelRef(modelRef)?.provider.trim().toLowerCase(); - if (provider && SLACK_QA_CODEX_PROVIDER_IDS.has(provider)) { - return; - } - throw new Error( - `Slack Codex approval scenarios require an openai/* or codex/* model; received "${modelRef}".`, - ); -} - -function resolveSlackQaSutAccountId(value?: string) { - return normalizeAccountId(value?.trim() || "sut"); -} - -type SlackQaMessageScenarioRun = { - afterNoReply?: (context: SlackQaScenarioContext) => Promise; - kind?: "message"; - expectReply: boolean; - input: string; - matchText: string; - preserveGatewayDebug?: boolean; - settleObservedMs?: number; - verify?: (message: SlackMessage, context: { requestThreadTs: string; sentTs: string }) => void; - verifyObserved?: (params: { - finalMessage: SlackMessage; - messages: readonly SlackObservedMessage[]; - }) => string | void; - beforeRun?: (context: Omit) => Promise; - afterReply?: (message: SlackMessage, context: SlackQaScenarioContext) => Promise; -}; - -type SlackQaDirectTransportScenarioRun = { - kind: "direct-transport"; - execute: ( - context: SlackQaDirectTransportScenarioContext, - ) => Promise; -}; - -type SlackQaDirectTransportScenarioContext = { - cfg: OpenClawConfig; - channelId: string; - sutAccountId: string; - sutIdentity: SlackAuthIdentity; - sutReadClient: WebClient; - sutWriteClient: WebClient; - timeoutMs: number; -}; - -type SlackQaDirectTransportScenarioResult = { - details: string; - message: SlackMessage; -}; - -type SlackQaApprovalScenarioRun = { - approvalKind: SlackQaApprovalKind; - decision: SlackQaApprovalDecision; - kind: "approval"; - token: string; -}; - -type SlackQaCodexApprovalScenarioRun = { - approvalKind: "plugin"; - appServerMethod: SlackQaCodexApprovalMethod; - decision: "allow-once"; - kind: "codex-approval"; - token: string; -}; - -type SlackQaScenarioRun = - | SlackQaApprovalScenarioRun - | SlackQaCodexApprovalScenarioRun - | SlackQaDirectTransportScenarioRun - | SlackQaMessageScenarioRun; - -type SlackQaBeforeRunResult = - | string - | void - | { - details?: string; - inputThreadTs?: string; - }; - -type SlackQaConfigOverrides = { - allowFrom?: string[]; - channelEnabled?: boolean; - approvals?: { - exec?: boolean; - plugin?: boolean; - target?: "both" | "channel" | "dm"; - }; - codexApproval?: boolean; - messageTool?: boolean; - progress?: { - commentary?: boolean; - toolProgress: boolean; - verboseDefault?: "off" | "on" | "full"; - }; - replyToMode?: "all" | "off"; - users?: string[]; -}; - -type SlackQaScenarioContext = { - channelId: string; - driverClient: WebClient; - gateway: Awaited>; - postSlackMessage: (params: { text: string; threadTs?: string }) => Promise<{ ts: string }>; - sentTs: string; - sutIdentity: SlackAuthIdentity; - sutReadClient: WebClient; - waitForReady: () => Promise; -}; - -type SlackQaScenarioDefinition = LiveTransportScenarioDefinition & { - buildRun: (sutUserId: string) => SlackQaScenarioRun; - configOverrides?: SlackQaConfigOverrides; - defaultEnabled?: boolean; - forcedRuntime?: RuntimeId; -}; - -type SlackQaGatewayHarness = Awaited>; - -type SlackAuthIdentity = { - botId?: string; - teamId?: string; - userId: string; -}; - -type SlackObservedMessage = { - botId?: string; - channelId: string; - matchedScenario?: boolean; - scenarioId?: string; - scenarioTitle?: string; - text: string; - actionValues?: string[]; - blockText?: string[]; - threadTs?: string; - ts: string; - userId?: string; -}; - -type SlackObservedMessageArtifact = { - botId?: string; - channelId?: string; - matchedScenario?: boolean; - scenarioId?: string; - scenarioTitle?: string; - text?: string; - actionValues?: string[]; - blockText?: string[]; - threadTs?: string; - ts?: string; - userId?: string; -}; - -type SlackApprovalArtifact = { - approvalId: string; - approvalKind: SlackQaApprovalKind; - appServerMethod?: SlackQaCodexApprovalMethod; - channelId?: string; - codexModelKey?: string; - decision: SlackQaApprovalDecision; - finalCodexTurnStatus?: string; - operationVerified?: boolean; - pendingActionValues?: string[]; - pendingCheckpointPath?: string; - pendingMessageTs?: string; - pendingScreenshotPath?: string; - pendingText?: string; - resolvedActionValues?: string[]; - resolvedCheckpointPath?: string; - resolvedMessageTs?: string; - resolvedScreenshotPath?: string; - resolvedText?: string; - threadTs?: string; -}; - -type SlackApprovalCheckpointState = "pending" | "resolved"; - -type SlackApprovalCheckpointAck = { - capturedAt?: string; - screenshotPath?: string; -}; - -type SlackApprovalCheckpointMessage = { - actionLabels: string[]; - blockText: string[]; - hasNativeActions: boolean; - text: string; -}; - -type SlackQaScenarioResult = { - approval?: SlackApprovalArtifact; - details: string; - id: string; - requestStartedAt?: string; - responseObservedAt?: string; - rttMs?: number; - rttMeasurement?: { - finalMatchedReplyRttMs: number; - requestStartedAt: string; - responseObservedAt: string; - source: "approval-request-to-resolution" | "request-to-observed-message"; - }; - standardId?: string; - status: "fail" | "pass"; - title: string; -}; - -type SlackQaRunResult = { - gatewayDebugDirPath?: string; - observedMessagesPath: string; - outputDir: string; - reportPath: string; - scenarios: SlackQaScenarioResult[]; - summaryPath: string; -}; - -type SlackCredentialLease = Awaited>>; -type SlackCredentialHeartbeat = ReturnType; - -const SLACK_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_SLACK_CAPTURE_CONTENT"; -const SLACK_QA_APPROVAL_CHECKPOINT_DIR_ENV = "OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_DIR"; -const SLACK_QA_APPROVAL_CHECKPOINT_TIMEOUT_MS_ENV = - "OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_TIMEOUT_MS"; -const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA"; -const SLACK_QA_WEB_API_TIMEOUT_MS = 45_000; -const SLACK_QA_ENV_KEYS = [ - "OPENCLAW_QA_SLACK_CHANNEL_ID", - "OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN", - "OPENCLAW_QA_SLACK_SUT_BOT_TOKEN", - "OPENCLAW_QA_SLACK_SUT_APP_TOKEN", -] as const; - -const slackQaCredentialPayloadSchema = z.object({ - channelId: z.string().trim().min(1), - driverBotToken: z.string().trim().min(1), - sutBotToken: z.string().trim().min(1), - sutAppToken: z.string().trim().min(1), -}); - -const slackAuthTestSchema = z.object({ - ok: z.boolean().optional(), - user_id: z.string().optional(), - bot_id: z.string().optional(), - team_id: z.string().optional(), -}); - -const slackPostMessageSchema = z.object({ - ok: z.boolean().optional(), - channel: z.string().optional(), - ts: z.string().min(1), -}); - -const slackHistoryMessageSchema = z.object({ - bot_id: z.string().optional(), - blocks: z.array(z.unknown()).optional(), - text: z.string().optional(), - thread_ts: z.string().optional(), - ts: z.string().min(1), - user: z.string().optional(), -}); - -type SlackMessage = Omit, "ts"> & { ts?: string }; - -const slackHistorySchema = z.object({ - ok: z.boolean().optional(), - messages: z.array(slackHistoryMessageSchema).optional(), -}); - -const slackRepliesSchema = z.object({ - ok: z.boolean().optional(), - messages: z.array(slackHistoryMessageSchema).optional(), -}); - -function buildSlackChartMessageToolArgs(summaryText: string) { - return { - action: "send", - message: summaryText, - presentation: { - blocks: [ - { - type: "chart", - chartType: "line", - title: SLACK_QA_CHART_TITLE, - categories: [...SLACK_QA_CHART_CATEGORIES], - series: [{ name: SLACK_QA_CHART_SERIES_NAME, values: [...SLACK_QA_CHART_VALUES] }], - xLabel: SLACK_QA_CHART_X_LABEL, - yLabel: SLACK_QA_CHART_Y_LABEL, - }, - ], - }, - }; -} - -function renderSlackChartAccessibleText(summaryText: string) { - return [ - summaryText, - "", - `${SLACK_QA_CHART_TITLE} (line chart)`, - `X axis: ${SLACK_QA_CHART_X_LABEL}`, - `Y axis: ${SLACK_QA_CHART_Y_LABEL}`, - `- ${SLACK_QA_CHART_SERIES_NAME}: ${SLACK_QA_CHART_CATEGORIES[0]}: ${SLACK_QA_CHART_VALUES[0]}; ${SLACK_QA_CHART_CATEGORIES[1]}: ${SLACK_QA_CHART_VALUES[1]}`, - ].join("\n"); -} - -function buildSlackTableMessageToolArgs(summaryText: string) { - return { - action: "send", - message: summaryText, - presentation: { - blocks: [ - { - type: "table", - caption: SLACK_QA_TABLE_CAPTION, - headers: [...SLACK_QA_TABLE_HEADERS], - rows: SLACK_QA_TABLE_ROWS.map((row) => [...row]), - rowHeaderColumnIndex: 0, - }, - ], - }, - }; -} - -function renderSlackTableAccessibleText(summaryText: string) { - return [ - summaryText, - "", - `${SLACK_QA_TABLE_CAPTION} (table)`, - SLACK_QA_TABLE_HEADERS.join("\t"), - ...SLACK_QA_TABLE_ROWS.map((row) => row.join("\t")), - ].join("\n"); -} - -function buildSlackInvalidBlocksTableRow(index: number) { - const rowId = String(index).padStart(3, "0"); - return [`row-${rowId}`, `value-${rowId}`] as const; -} - -function buildSlackInvalidBlocksTableProbe() { - const summaryText = `SLACK_QA_TABLE_INVALID_BLOCKS_${randomUUID().slice(0, 8).toUpperCase()}`; - const dataRows = Array.from({ length: SLACK_QA_INVALID_TABLE_DATA_ROW_COUNT }, (_entry, index) => - buildSlackInvalidBlocksTableRow(index + 1), - ); - const block = { - type: "data_table", - caption: SLACK_QA_INVALID_TABLE_CAPTION, - rows: [ - SLACK_QA_INVALID_TABLE_HEADERS.map((text) => ({ type: "raw_text", text })), - ...dataRows.map((row) => row.map((text) => ({ type: "raw_text", text }))), - ], - row_header_column_index: 0, - } as const; - const fallbackText = [ - summaryText, - "", - `${SLACK_QA_INVALID_TABLE_CAPTION} (table)`, - SLACK_QA_INVALID_TABLE_HEADERS.join("\t"), - ...dataRows.map((row) => row.join("\t")), - ].join("\n"); - return { - block, - dataRowCount: dataRows.length, - fallbackText, - firstRowText: buildSlackInvalidBlocksTableRow(1).join("\t"), - finalRowText: buildSlackInvalidBlocksTableRow(SLACK_QA_INVALID_TABLE_DATA_ROW_COUNT).join("\t"), - summaryText, - }; -} - -type SlackProgressCommentaryExpectation = { - commentary: "absent" | "draft" | "standalone"; - toolProgress: "absent" | "draft" | "standalone"; -}; - -function buildSlackProgressCommentaryRun( - sutUserId: string, - expectation: SlackProgressCommentaryExpectation, -): SlackQaMessageScenarioRun { - const suffix = randomUUID().slice(0, 8).toUpperCase(); - // Slack mrkdwn escapes underscores in progress drafts. Hyphenated markers - // stay byte-identical across draft edits and final-message reads. - const commentaryMarker = `SLACK-QA-COMMENTARY-${suffix}`; - const toolMarker = `SLACK-QA-TOOL-${suffix}`; - const finalMarker = `SLACK-QA-COMMENTARY-DONE-${suffix}`; - return { - expectReply: true, - input: [ - `<@${sutUserId}> This is a Slack progress protocol test. First, emit an assistant commentary message whose entire text is exactly ${commentaryMarker}.`, - "Do not call any tool until that commentary message is complete.", - `Then use the exec tool exactly once to run: grep '${toolMarker}' /dev/null || sleep 5.`, - `After the command finishes, reply with only this exact marker: ${finalMarker}`, - ].join(" "), - matchText: finalMarker, - settleObservedMs: 3_000, - verifyObserved: ({ finalMessage, messages }) => { - if (!finalMessage.ts) { - throw new Error("Slack progress commentary final message had no ts"); - } - if ((finalMessage.text ?? "").trim() !== finalMarker) { - throw new Error("expected the Slack final answer to contain only the final marker"); - } - const progressMessages = messages.filter((message) => !message.text.includes(finalMarker)); - const commentaryMessages = progressMessages.filter((message) => - message.text.includes(commentaryMarker), - ); - const commentaryTimestamps = new Set(commentaryMessages.map((message) => message.ts)); - if (expectation.commentary === "absent" && commentaryTimestamps.size !== 0) { - throw new Error("expected commentary to stay out of Slack progress messages"); - } - if (expectation.commentary !== "absent" && commentaryTimestamps.size !== 1) { - throw new Error( - `expected exactly one Slack message identity containing commentary; got ${commentaryTimestamps.size}`, - ); - } - const commentaryTs = [...commentaryTimestamps][0]; - if (expectation.commentary === "draft" && commentaryTs !== finalMessage.ts) { - throw new Error("expected commentary on the progress draft finalized as the answer"); - } - if (expectation.commentary === "standalone" && commentaryTs === finalMessage.ts) { - throw new Error("expected commentary only in the standalone verbose message"); - } - const toolTimestamps = new Set( - progressMessages - .filter((message) => message.text.includes(toolMarker)) - .map((message) => message.ts), - ); - if (expectation.toolProgress === "draft") { - if (toolTimestamps.size !== 1 || !toolTimestamps.has(finalMessage.ts)) { - throw new Error("expected tool progress on the progress draft finalized as the answer"); - } - } else if (expectation.toolProgress === "standalone") { - if (toolTimestamps.size === 0 || toolTimestamps.has(finalMessage.ts)) { - throw new Error("expected tool progress only in standalone verbose messages"); - } - } else if (toolTimestamps.size !== 0) { - throw new Error("expected tool progress to stay out of Slack progress messages"); - } - const finalTimestamps = new Set( - messages - .filter((message) => message.text.includes(finalMarker)) - .map((message) => message.ts), - ); - if (finalTimestamps.size !== 1 || !finalTimestamps.has(finalMessage.ts)) { - throw new Error( - "expected one final-marker Slack message identity matching the final answer", - ); - } - const commentaryDetails = - expectation.commentary === "draft" - ? "commentary on progress/final identity" - : expectation.commentary === "standalone" - ? "one standalone commentary identity" - : "commentary absent from Slack progress"; - return `verified ${commentaryDetails}; tool progress ${expectation.toolProgress}; final identity unique`; - }, - }; -} - -const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ - { - id: "slack-canary", - standardId: "canary", - title: "Slack canary echo", - timeoutMs: 45_000, - buildRun: (sutUserId) => { - const token = `SLACK_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - expectReply: true, - input: `<@${sutUserId}> reply with only this exact marker: ${token}`, - matchText: token, - }; - }, - }, - { - id: "slack-mention-gating", - standardId: "mention-gating", - title: "Slack unmentioned bot message does not trigger", - timeoutMs: 8_000, - buildRun: () => { - const token = `SLACK_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - expectReply: false, - input: `reply with only this exact marker: ${token}`, - matchText: token, - }; - }, - }, - { - id: "slack-allowlist-block", - standardId: "allowlist-block", - title: "Slack non-allowlisted sender does not trigger", - timeoutMs: 8_000, - configOverrides: { - allowFrom: ["U_OPENCLAW_QA_NEVER_ALLOWED"], - users: ["U_OPENCLAW_QA_NEVER_ALLOWED"], - }, - buildRun: (sutUserId) => { - const token = `SLACK_QA_BLOCK_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - expectReply: false, - input: `<@${sutUserId}> reply with only this exact marker: ${token}`, - matchText: token, - }; - }, - }, - { - id: "slack-channel-disabled-warning", - title: "Slack disabled channel warns and does not trigger", - timeoutMs: 8_000, - defaultEnabled: false, - configOverrides: { channelEnabled: false }, - buildRun: (sutUserId) => { - const marker = `SLACK_QA_DISABLED_${randomUUID().slice(0, 8).toUpperCase()}`; - let logCursor = 0; - return { - expectReply: false, - input: `<@${sutUserId}> reply with only this exact marker: ${marker}`, - matchText: marker, - preserveGatewayDebug: true, - beforeRun: async ({ gateway }) => { - const gatewayLogTail = (await gateway.call( - "logs.tail", - { limit: 1, maxBytes: 32_000 }, - { timeoutMs: SLACK_QA_LOG_TAIL_TIMEOUT_MS }, - )) as { cursor?: unknown }; - logCursor = typeof gatewayLogTail.cursor === "number" ? gatewayLogTail.cursor : 0; - }, - afterNoReply: async ({ gateway }) => { - const gatewayLogTail = (await gateway.call( - "logs.tail", - { cursor: logCursor, limit: 200, maxBytes: 256_000 }, - { timeoutMs: SLACK_QA_LOG_TAIL_TIMEOUT_MS }, - )) as { lines?: unknown }; - const gatewayLogLines = Array.isArray(gatewayLogTail.lines) - ? gatewayLogTail.lines.filter((line): line is string => typeof line === "string") - : []; - const expectedFields = [ - "Slack channel denied by configuration", - "channel_not_allowed", - "channel_disabled", - ]; - if ( - !gatewayLogLines.some((line) => expectedFields.every((field) => line.includes(field))) - ) { - throw new Error("disabled Slack channel did not emit the structured warning"); - } - return "structured disabled-channel warning observed"; - }, - }; - }, - }, - { - id: "slack-top-level-reply-shape", - standardId: "top-level-reply-shape", - title: "Slack top-level reply stays top-level", - timeoutMs: 45_000, - configOverrides: { replyToMode: "off" }, - buildRun: (sutUserId) => { - const token = `SLACK_QA_TOPLEVEL_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - expectReply: true, - input: `<@${sutUserId}> reply with only this exact marker: ${token}`, - matchText: token, - verify: (message) => { - if (message.thread_ts) { - throw new Error( - `expected top-level Slack reply without thread_ts; got ${message.thread_ts}`, - ); - } - }, - }; - }, - }, - { - id: "slack-progress-commentary-true", - title: "Slack progress commentary true is independent from tool progress", - defaultEnabled: false, - timeoutMs: 90_000, - configOverrides: { - progress: { commentary: true, toolProgress: false }, - }, - buildRun: (sutUserId) => - buildSlackProgressCommentaryRun(sutUserId, { - commentary: "draft", - toolProgress: "absent", - }), - }, - { - id: "slack-progress-commentary-false", - title: "Slack progress commentary false stays out of the progress draft", - defaultEnabled: false, - timeoutMs: 90_000, - configOverrides: { - progress: { commentary: false, toolProgress: false }, - }, - buildRun: (sutUserId) => - buildSlackProgressCommentaryRun(sutUserId, { - commentary: "absent", - toolProgress: "absent", - }), - }, - { - id: "slack-progress-commentary-omitted", - title: "Slack omitted progress commentary preserves the tool-progress default", - defaultEnabled: false, - timeoutMs: 90_000, - configOverrides: { - progress: { toolProgress: true }, - }, - buildRun: (sutUserId) => - buildSlackProgressCommentaryRun(sutUserId, { - commentary: "draft", - toolProgress: "draft", - }), - }, - { - id: "slack-progress-commentary-verbose-dedupe", - title: "Slack explicit commentary yields to durable verbose progress", - defaultEnabled: false, - timeoutMs: 90_000, - configOverrides: { - progress: { commentary: true, toolProgress: false, verboseDefault: "on" }, - }, - buildRun: (sutUserId) => - buildSlackProgressCommentaryRun(sutUserId, { - commentary: "standalone", - toolProgress: "standalone", - }), - }, - { - id: "slack-chart-presentation-native", - title: "Slack portable chart renders as a native data visualization", - timeoutMs: 90_000, - configOverrides: { messageTool: true }, - buildRun: (sutUserId) => { - const suffix = randomUUID().slice(0, 8).toUpperCase(); - const summaryText = `SLACK_QA_CHART_SUMMARY_${suffix}`; - const finalMarker = `SLACK_QA_CHART_DONE_${suffix}`; - const messageToolArgs = buildSlackChartMessageToolArgs(summaryText); - return { - expectReply: true, - input: [ - `<@${sutUserId}> Slack native chart QA check ${summaryText}.`, - `Call the message tool exactly once with these exact arguments: ${JSON.stringify(messageToolArgs)}.`, - `After the chart send succeeds, reply with only this exact marker: ${finalMarker}`, - ].join(" "), - matchText: finalMarker, - afterReply: async (_message, context) => { - await waitForSlackStoredMessage({ - channelId: context.channelId, - client: context.sutReadClient, - description: "message with native chart", - matchesMessage: (message) => - isExpectedSlackNativeChartMessage( - message, - renderSlackChartAccessibleText(summaryText), - ), - oldestTs: context.sentTs, - sutIdentity: context.sutIdentity, - timeoutMs: SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS, - }); - return "verified native data_visualization block and deterministic accessible text"; - }, - }; - }, - }, - { - id: "slack-table-presentation-native", - title: "Slack portable table renders as a native data table", - timeoutMs: 90_000, - configOverrides: { messageTool: true }, - buildRun: (sutUserId) => { - const suffix = randomUUID().slice(0, 8).toUpperCase(); - const summaryText = `SLACK_QA_TABLE_SUMMARY_${suffix}`; - const finalMarker = `SLACK_QA_TABLE_DONE_${suffix}`; - const messageToolArgs = buildSlackTableMessageToolArgs(summaryText); - return { - expectReply: true, - input: [ - `<@${sutUserId}> Slack native table QA check ${summaryText}.`, - `Call the message tool exactly once with these exact arguments: ${JSON.stringify(messageToolArgs)}.`, - `After the table send succeeds, reply with only this exact marker: ${finalMarker}`, - ].join(" "), - matchText: finalMarker, - afterReply: async (_message, context) => { - await waitForSlackStoredMessage({ - channelId: context.channelId, - client: context.sutReadClient, - description: "message with native table", - matchesMessage: (message) => - isExpectedSlackNativeTableMessage( - message, - renderSlackTableAccessibleText(summaryText), - ), - oldestTs: context.sentTs, - sutIdentity: context.sutIdentity, - timeoutMs: SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS, - }); - return "verified native data_table block and deterministic accessible text"; - }, - }; - }, - }, - { - id: "slack-table-invalid-blocks-fallback", - title: "Slack rejects an over-limit native table and stores its complete fallback", - defaultEnabled: false, - timeoutMs: 45_000, - buildRun: () => ({ - kind: "direct-transport", - execute: runSlackTableInvalidBlocksFallbackScenario, - }), - }, - { - id: "slack-reaction-glyph-native", - title: "Slack message tool normalizes an emoji glyph reaction", - timeoutMs: 90_000, - configOverrides: { messageTool: true }, - buildRun: (sutUserId) => { - const token = `SLACK_QA_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - expectReply: true, - input: [ - `<@${sutUserId}> use the message tool exactly once to react to this message.`, - 'Set action to "react", channel to "slack", and emoji to exactly "āœ…".', - "Do not substitute a shortcode.", - `After the reaction succeeds, reply with only this exact marker: ${token}`, - ].join(" "), - matchText: token, - afterReply: async (_message, context) => { - await waitForSlackReaction({ - channelId: context.channelId, - client: context.sutReadClient, - expectedReactionName: "white_check_mark", - messageId: context.sentTs, - sutUserId: context.sutIdentity.userId, - timeoutMs: SLACK_QA_REACTION_VERIFY_TIMEOUT_MS, - }); - return "verified SUT white_check_mark reaction from exact glyph instruction"; - }, - }; - }, - }, - { - id: "slack-approval-exec-native", - title: "Slack native exec approval prompt resolves", - timeoutMs: 60_000, - configOverrides: { - approvals: { - exec: true, - target: "channel", - }, - }, - buildRun: () => ({ - approvalKind: "exec", - decision: "allow-once", - kind: "approval", - token: `SLACK_QA_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, - }), - }, - { - id: "slack-approval-plugin-native", - title: "Slack native plugin approval prompt resolves with exec approvals enabled", - timeoutMs: 60_000, - configOverrides: { - approvals: { - exec: true, - plugin: true, - target: "channel", - }, - }, - buildRun: () => ({ - approvalKind: "plugin", - decision: "allow-once", - kind: "approval", - token: `SLACK_QA_PLUGIN_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, - }), - }, - { - id: "slack-codex-approval-exec-native", - title: "Slack native Codex command approval prompt resolves", - timeoutMs: 180_000, - configOverrides: { - approvals: { - exec: true, - plugin: true, - target: "channel", - }, - codexApproval: true, - }, - forcedRuntime: "codex", - buildRun: () => ({ - approvalKind: "plugin", - appServerMethod: "item/commandExecution/requestApproval", - decision: "allow-once", - kind: "codex-approval", - token: `SLACK_QA_CODEX_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, - }), - }, - { - id: "slack-codex-approval-plugin-native", - title: "Slack native Codex file approval prompt resolves", - timeoutMs: 180_000, - configOverrides: { - approvals: { - exec: true, - plugin: true, - target: "channel", - }, - codexApproval: true, - }, - forcedRuntime: "codex", - buildRun: () => ({ - approvalKind: "plugin", - appServerMethod: "item/fileChange/requestApproval", - decision: "allow-once", - kind: "codex-approval", - token: `SLACK_QA_CODEX_FILE_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, - }), - }, -]; - -const SLACK_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({ - scenarios: SLACK_QA_SCENARIOS, -}); - -export function listSlackQaScenarioCatalog() { - return SLACK_QA_SCENARIOS.map((scenario) => ({ id: scenario.id })); -} - -function resolveEnvValue(env: NodeJS.ProcessEnv, key: (typeof SLACK_QA_ENV_KEYS)[number]) { - const value = env[key]?.trim(); - if (!value) { - throw new Error(`Missing ${key}.`); - } - return value; -} - -function normalizeSlackId(value: string, label: string) { - const normalized = value.trim(); - if (!/^[A-Z][A-Z0-9]+$/.test(normalized)) { - throw new Error(`${label} must be a Slack id like C123 or U123.`); - } - return normalized; -} - -function validateSlackQaRuntimeEnv(runtimeEnv: SlackQaRuntimeEnv, label: string) { - normalizeSlackId(runtimeEnv.channelId, `${label} channelId`); - return runtimeEnv; -} - -function resolveSlackQaRuntimeEnv(env: NodeJS.ProcessEnv = process.env): SlackQaRuntimeEnv { - const runtimeEnv = { - channelId: resolveEnvValue(env, "OPENCLAW_QA_SLACK_CHANNEL_ID"), - driverBotToken: resolveEnvValue(env, "OPENCLAW_QA_SLACK_DRIVER_BOT_TOKEN"), - sutBotToken: resolveEnvValue(env, "OPENCLAW_QA_SLACK_SUT_BOT_TOKEN"), - sutAppToken: resolveEnvValue(env, "OPENCLAW_QA_SLACK_SUT_APP_TOKEN"), - }; - return validateSlackQaRuntimeEnv(runtimeEnv, "OPENCLAW_QA_SLACK"); -} - -function parseSlackQaCredentialPayload(payload: unknown): SlackQaRuntimeEnv { - const parsed = slackQaCredentialPayloadSchema.parse(payload); - const runtimeEnv = { - channelId: parsed.channelId, - driverBotToken: parsed.driverBotToken, - sutBotToken: parsed.sutBotToken, - sutAppToken: parsed.sutAppToken, - }; - return validateSlackQaRuntimeEnv(runtimeEnv, "Slack credential payload"); -} - -function findScenario(ids?: string[]) { - const selected = selectLiveTransportScenarios({ - ids, - laneLabel: "Slack", - scenarios: SLACK_QA_SCENARIOS, - }); - return ids?.length ? selected : selected.filter((scenario) => scenario.defaultEnabled !== false); -} - -function asPlainRecord(value: unknown): Record { - return value && typeof value === "object" && !Array.isArray(value) - ? (value as Record) - : {}; -} - -type SlackQaPostMessageAttempt = { - failureCode?: string; - formattingDisabled: boolean; - nativeDataBlockCount: number; - status: "failed" | "sent"; -}; - -function countSlackNativeDataBlocks(value: unknown) { - if (!Array.isArray(value)) { - return 0; - } - return value.filter((block) => { - const type = asPlainRecord(block).type; - return type === "data_table" || type === "data_visualization"; - }).length; -} - -function readSlackApiFailureCode(error: unknown) { - const record = asPlainRecord(error); - const data = asPlainRecord(record.data); - const code = data.error ?? record.error; - return typeof code === "string" && /^[a-z0-9_]{1,64}$/u.test(code) ? code : undefined; -} - -function instrumentSlackPostMessage(client: WebClient) { - const originalPostMessage = client.chat.postMessage; - const attempts: SlackQaPostMessageAttempt[] = []; - client.chat.postMessage = (async (payload) => { - const payloadRecord = payload as { blocks?: unknown; mrkdwn?: boolean }; - const attempt = { - formattingDisabled: payloadRecord.mrkdwn === false, - nativeDataBlockCount: countSlackNativeDataBlocks(payloadRecord.blocks), - }; - try { - const response = await originalPostMessage.call(client.chat, payload); - attempts.push({ ...attempt, status: "sent" }); - return response; - } catch (error) { - const failureCode = readSlackApiFailureCode(error); - attempts.push({ - ...attempt, - ...(failureCode ? { failureCode } : {}), - status: "failed", - }); - throw error; - } - }) as typeof client.chat.postMessage; - return { - attempts, - restore: () => { - client.chat.postMessage = originalPostMessage; - }, - }; -} - -function buildSlackQaConfig( - baseCfg: OpenClawConfig, - params: { - channelId: string; - driverBotUserId: string; - overrides?: SlackQaConfigOverrides; - primaryModel?: string; - sutAccountId: string; - sutAppToken: string; - sutBotToken: string; - }, -): OpenClawConfig { - const codexApprovalConfig = params.overrides?.codexApproval === true; - const progressOverrides = params.overrides?.progress; - const primaryModel = params.primaryModel; - const pluginAllow = uniqueStrings([ - ...(baseCfg.plugins?.allow ?? []), - "slack", - ...(codexApprovalConfig ? ["codex"] : []), - ]); - const approvalOverrides = params.overrides?.approvals; - const codexEntry = baseCfg.plugins?.entries?.codex; - const codexEntryConfig = asPlainRecord(codexEntry?.config); - const codexAppServerConfig = asPlainRecord(codexEntryConfig.appServer); - const approvalForwardingConfig = - approvalOverrides?.exec || approvalOverrides?.plugin - ? { - approvals: { - ...baseCfg.approvals, - ...(approvalOverrides.exec - ? { - exec: { - ...baseCfg.approvals?.exec, - enabled: true, - mode: "session" as const, - }, - } - : {}), - ...(approvalOverrides.plugin - ? { - plugin: { - ...baseCfg.approvals?.plugin, - enabled: true, - mode: "session" as const, - }, - } - : {}), - }, - } - : {}; - const codexAgentDefaults = - codexApprovalConfig && primaryModel - ? { - ...baseCfg.agents?.defaults, - models: { - ...baseCfg.agents?.defaults?.models, - [primaryModel]: { - ...baseCfg.agents?.defaults?.models?.[primaryModel], - agentRuntime: { id: "codex" as const }, - }, - }, - } - : baseCfg.agents?.defaults; - const qaAgentDefaults = progressOverrides - ? { - ...codexAgentDefaults, - ...(progressOverrides.verboseDefault - ? { verboseDefault: progressOverrides.verboseDefault } - : {}), - } - : codexAgentDefaults; - const qaAgentList = progressOverrides - ? baseCfg.agents?.list?.map((agent) => { - if (agent.id !== "qa") { - return agent; - } - // Slack draft edits cannot preserve custom authorship. Remove the - // synthetic QA identity so progress scenarios reach the draft path. - const qaAgent = { ...agent }; - delete qaAgent.identity; - return qaAgent; - }) - : baseCfg.agents?.list; - const execApprovalsConfig = approvalOverrides - ? { - enabled: true, - approvers: [params.driverBotUserId], - target: approvalOverrides.target ?? ("channel" as const), - } - : undefined; - const explicitToolAllow = baseCfg.tools?.allow; - const messageToolPolicy = params.overrides?.messageTool - ? explicitToolAllow && explicitToolAllow.length > 0 - ? { allow: uniqueStrings([...explicitToolAllow, "message"]) } - : { alsoAllow: uniqueStrings([...(baseCfg.tools?.alsoAllow ?? []), "message"]) } - : {}; - const toolsConfig = - codexApprovalConfig || params.overrides?.messageTool - ? { - tools: { - ...baseCfg.tools, - ...messageToolPolicy, - ...(codexApprovalConfig - ? { - exec: { - ...baseCfg.tools?.exec, - mode: "ask" as const, - }, - } - : {}), - }, - } - : {}; - return { - ...baseCfg, - ...approvalForwardingConfig, - ...toolsConfig, - plugins: { - ...baseCfg.plugins, - allow: pluginAllow, - entries: { - ...baseCfg.plugins?.entries, - slack: { enabled: true }, - ...(codexApprovalConfig - ? { - codex: { - ...codexEntry, - enabled: true, - config: { - ...codexEntryConfig, - appServer: { - ...codexAppServerConfig, - mode: "guardian" as const, - }, - }, - }, - } - : {}), - }, - }, - ...(codexApprovalConfig || progressOverrides - ? { - agents: { - ...baseCfg.agents, - ...(qaAgentDefaults ? { defaults: qaAgentDefaults } : {}), - ...(qaAgentList ? { list: qaAgentList } : {}), - }, - } - : {}), - messages: { - ...baseCfg.messages, - groupChat: { - ...baseCfg.messages?.groupChat, - visibleReplies: "automatic", - }, - }, - channels: { - ...baseCfg.channels, - slack: { - enabled: true, - defaultAccount: params.sutAccountId, - accounts: { - [params.sutAccountId]: { - enabled: true, - mode: "socket", - botToken: params.sutBotToken, - appToken: params.sutAppToken, - allowFrom: params.overrides?.allowFrom ?? [params.driverBotUserId], - groupPolicy: "allowlist", - allowBots: true, - replyToMode: params.overrides?.replyToMode ?? "off", - ...(progressOverrides - ? { - streaming: { - mode: "progress" as const, - progress: { - label: false, - maxLines: 4, - toolProgress: progressOverrides.toolProgress, - ...(progressOverrides.commentary === undefined - ? {} - : { commentary: progressOverrides.commentary }), - }, - }, - } - : {}), - ...(execApprovalsConfig ? { execApprovals: execApprovalsConfig } : {}), - channels: { - [params.channelId]: { - enabled: params.overrides?.channelEnabled ?? true, - requireMention: true, - allowBots: true, - users: params.overrides?.users ?? [params.driverBotUserId], - }, - }, - }, - }, - }, - }, - }; -} - -async function getSlackIdentity(token: string): Promise { - const client = createSlackWebClient(token, { timeout: SLACK_QA_WEB_API_TIMEOUT_MS }); - const auth = slackAuthTestSchema.parse(await client.auth.test()); - if (!auth.user_id) { - throw new Error("Slack auth.test did not return user_id."); - } - return { - userId: auth.user_id, - botId: auth.bot_id, - teamId: auth.team_id, - }; -} - -async function sendSlackChannelMessage(params: { - channelId: string; - client: WebClient; - text: string; - threadTs?: string; -}) { - const postSlackMessage = params.client.chat.postMessage.bind(params.client.chat); - const sent = slackPostMessageSchema.parse( - await postSlackMessage({ - channel: params.channelId, - text: params.text, - thread_ts: params.threadTs, - unfurl_links: false, - unfurl_media: false, - }), - ); - return { - channelId: sent.channel ?? params.channelId, - ts: sent.ts, - }; -} - -async function listSlackMessages(params: { - channelId: string; - client: WebClient; - oldestTs: string; -}) { - const history = slackHistorySchema.parse( - await params.client.conversations.history({ - channel: params.channelId, - inclusive: true, - limit: 50, - oldest: params.oldestTs, - }), - ); - return history.messages ?? []; -} - -async function listSlackThreadMessages(params: { - channelId: string; - client: WebClient; - threadTs: string; -}) { - const replies = slackRepliesSchema.parse( - await params.client.conversations.replies({ - channel: params.channelId, - inclusive: true, - limit: 50, - ts: params.threadTs, - }), - ); - return replies.messages ?? []; -} - -function collectSlackBlockStringFields( - value: unknown, - fieldName: string, - values: string[] = [], -): string[] { - if (Array.isArray(value)) { - for (const entry of value) { - collectSlackBlockStringFields(entry, fieldName, values); - } - return values; - } - if (!value || typeof value !== "object") { - return values; - } - for (const [key, entry] of Object.entries(value)) { - if (key === fieldName && typeof entry === "string" && entry.trim().length > 0) { - values.push(entry); - continue; - } - collectSlackBlockStringFields(entry, fieldName, values); - } - return values; -} - -function collectSlackBlockText(blocks?: unknown[]) { - return collectSlackBlockStringFields(blocks ?? [], "text"); -} - -function collectSlackActionValues(blocks?: unknown[]) { - return collectSlackBlockStringFields(blocks ?? [], "value"); -} - -function parseSlackNativeApprovalAction(value: string) { - if (!value.startsWith(SLACK_QA_APPROVAL_ACTION_PREFIX)) { - return undefined; - } - try { - const decoded: unknown = JSON.parse(value.slice(SLACK_QA_APPROVAL_ACTION_PREFIX.length)); - const parsed = SlackQaApprovalActionValueSchema.safeParse(decoded); - return parsed.success ? parsed.data : undefined; - } catch { - return undefined; - } -} - -function collectSlackButtonLabels(blocks?: unknown[]) { - const labels: string[] = []; - function visit(value: unknown) { - if (Array.isArray(value)) { - for (const entry of value) { - visit(entry); - } - return; - } - if (!value || typeof value !== "object") { - return; - } - const candidate = value as Record; - if (candidate.type === "button") { - const text = candidate.text; - if (text && typeof text === "object") { - const label = (text as { text?: unknown }).text; - if (typeof label === "string" && label.trim().length > 0) { - labels.push(label); - } - } - } - for (const entry of Object.values(candidate)) { - visit(entry); - } - } - visit(blocks ?? []); - return labels; -} - -function buildSlackApprovalCheckpointMessage( - message: SlackMessage, -): SlackApprovalCheckpointMessage { - const actionValues = collectSlackActionValues(message.blocks); - return { - actionLabels: collectSlackButtonLabels(message.blocks), - blockText: collectSlackBlockText(message.blocks), - hasNativeActions: actionValues.some((value) => parseSlackNativeApprovalAction(value)), - text: message.text ?? "", - }; -} - -function hasSlackNativeApprovalActions(params: { - actionValues: string[]; - approvalId?: string; - decision: SlackQaApprovalDecision; -}) { - return params.actionValues.some((value) => { - const action = parseSlackNativeApprovalAction(value); - return ( - action?.decision === params.decision && - (!params.approvalId || action.approvalId === params.approvalId) - ); - }); -} - -function extractSlackNativeApprovalId(params: { - actionValues: string[]; - decision: SlackQaApprovalDecision; -}) { - for (const value of params.actionValues) { - const action = parseSlackNativeApprovalAction(value); - if (action?.decision === params.decision) { - return action.approvalId; - } - } - return undefined; -} - -function isSutSlackMessage(message: SlackMessage, sutIdentity: SlackAuthIdentity) { - return ( - (message.user !== undefined && message.user === sutIdentity.userId) || - (message.bot_id !== undefined && message.bot_id === sutIdentity.botId) - ); -} - -// Slack history can flatten top-level accessibility newlines on readback. -// Normalize only whitespace; the native chart structure stays byte-for-byte strict below. -function normalizeSlackAccessibleText(value: string) { - return value.trim().replace(/\s+/gu, " "); -} - -function isExpectedSlackNativeChartMessage(message: SlackMessage, expectedAccessibleText: string) { - if ( - normalizeSlackAccessibleText(message.text ?? "") !== - normalizeSlackAccessibleText(expectedAccessibleText) - ) { - return false; - } - return (message.blocks ?? []).some((value) => { - const block = asPlainRecord(value); - return isDeepStrictEqual( - { type: block.type, title: block.title, chart: block.chart }, - SLACK_QA_NATIVE_CHART, - ); - }); -} - -async function waitForSlackStoredMessage(params: { - channelId: string; - client: WebClient; - description: string; - matchesMessage: (message: SlackMessage) => boolean; - oldestTs: string; - sutIdentity: SlackAuthIdentity; - timeoutMs: number; -}) { - const startedAt = Date.now(); - while (true) { - const messages = await listSlackMessages({ - channelId: params.channelId, - client: params.client, - oldestTs: params.oldestTs, - }); - const message = messages.find( - (entry) => - entry.ts !== params.oldestTs && - isSutSlackMessage(entry, params.sutIdentity) && - params.matchesMessage(entry), - ); - if (message) { - return message; - } - const remainingMs = params.timeoutMs - (Date.now() - startedAt); - if (remainingMs <= 0) { - break; - } - await new Promise((resolve) => { - setTimeout(resolve, Math.min(1_000, remainingMs)); - }); - } - throw new Error(`timed out after ${params.timeoutMs}ms waiting for Slack ${params.description}`); -} - -function isExpectedSlackNativeTableMessage(message: SlackMessage, expectedAccessibleText: string) { - if ( - normalizeSlackAccessibleText(message.text ?? "") !== - normalizeSlackAccessibleText(expectedAccessibleText) - ) { - return false; - } - return (message.blocks ?? []).some((value) => { - const block = asPlainRecord(value); - return isDeepStrictEqual( - { - type: block.type, - caption: block.caption, - rows: block.rows, - row_header_column_index: block.row_header_column_index, - }, - SLACK_QA_NATIVE_TABLE, - ); - }); -} - -async function runSlackTableInvalidBlocksFallbackScenario( - context: SlackQaDirectTransportScenarioContext, -): Promise { - const probe = buildSlackInvalidBlocksTableProbe(); - const oldestTs = ((Date.now() - 5_000) / 1_000).toFixed(6); - const instrumentation = instrumentSlackPostMessage(context.sutWriteClient); - let sent: Awaited>; - try { - try { - sent = await sendSlackMessage(`channel:${context.channelId}`, probe.summaryText, { - accountId: context.sutAccountId, - blocks: [probe.block] as never, - cfg: context.cfg, - client: context.sutWriteClient, - nativeDataFallbackBaseText: probe.summaryText, - }); - } catch { - const [nativeAttempt, fallbackAttempt] = instrumentation.attempts; - if (nativeAttempt?.failureCode !== "invalid_blocks") { - throw new Error( - `expected first Slack API failure code invalid_blocks; observed ${nativeAttempt?.failureCode ?? "none"}`, - ); - } - throw new Error( - `Slack fallback failed after invalid_blocks; observed ${fallbackAttempt?.failureCode ?? "no fallback API failure code"}`, - ); - } - } finally { - instrumentation.restore(); - } - - const [nativeAttempt, fallbackAttempt] = instrumentation.attempts; - if (instrumentation.attempts.length !== 2) { - throw new Error( - `expected exactly two Slack API attempts; observed ${instrumentation.attempts.length}`, - ); - } - if ( - nativeAttempt?.status !== "failed" || - nativeAttempt.failureCode !== "invalid_blocks" || - nativeAttempt.nativeDataBlockCount !== 1 - ) { - throw new Error( - `expected first Slack API attempt to fail with invalid_blocks for one native data block; observed ${nativeAttempt?.failureCode ?? "none"}`, - ); - } - if ( - fallbackAttempt?.status !== "sent" || - fallbackAttempt.nativeDataBlockCount !== 0 || - !fallbackAttempt.formattingDisabled - ) { - throw new Error("Slack fallback did not use one formatting-disabled blockless API request"); - } - - const message = await waitForSlackStoredMessage({ - channelId: context.channelId, - client: context.sutReadClient, - description: "stored invalid_blocks fallback message", - matchesMessage: (candidate) => candidate.ts === sent.messageId, - oldestTs, - sutIdentity: context.sutIdentity, - timeoutMs: context.timeoutMs, - }); - const storedText = message.text ?? ""; - if (countSlackNativeDataBlocks(message.blocks) !== 0) { - throw new Error("stored Slack fallback retained a native data block"); - } - const storedLines = storedText.split("\n"); - if (!storedLines.includes(probe.firstRowText)) { - throw new Error("stored Slack fallback omitted the exact first data row"); - } - if (!storedLines.includes(probe.finalRowText)) { - throw new Error("stored Slack fallback omitted the exact final data row"); - } - if (storedText !== probe.fallbackText) { - throw new Error( - `stored Slack fallback was incomplete: expected ${probe.fallbackText.length} characters, observed ${storedText.length}`, - ); - } - return { - details: [ - "direct transport", - "first API failure=invalid_blocks", - "API attempts=2", - `data rows=${probe.dataRowCount}`, - "fallback formatting disabled=true", - "stored native data blocks=0", - "first row=present", - "final row=present", - "complete delivery=true", - ].join("; "), - message, - }; -} - -type SlackScenarioObservationContext = { - channelId: string; - matchText: string; - observedMessages: SlackObservedMessage[]; - observationScenarioId: string; - observationScenarioTitle: string; - sentTs: string; - sutIdentity: SlackAuthIdentity; -}; - -function recordSlackScenarioMessages( - params: SlackScenarioObservationContext & { messages: SlackMessage[] }, -) { - let matchedMessage: SlackMessage | undefined; - for (const message of params.messages) { - const text = message.text ?? ""; - if ( - !message.ts || - message.ts === params.sentTs || - !isSutSlackMessage(message, params.sutIdentity) - ) { - continue; - } - const matchedScenario = text.includes(params.matchText); - params.observedMessages.push({ - actionValues: collectSlackActionValues(message.blocks), - blockText: collectSlackBlockText(message.blocks), - botId: message.bot_id, - channelId: params.channelId, - matchedScenario, - scenarioId: params.observationScenarioId, - scenarioTitle: params.observationScenarioTitle, - text, - threadTs: message.thread_ts, - ts: message.ts, - userId: message.user, - }); - if (matchedScenario && !matchedMessage) { - matchedMessage = message; - } - } - return matchedMessage; -} - -async function waitForSlackScenarioReply( - params: SlackScenarioObservationContext & { - client: WebClient; - threadTs?: string; - timeoutMs: number; - }, -) { - const observationContext: SlackScenarioObservationContext = params; - const startedAt = Date.now(); - const inspectMessages = (messages: SlackMessage[]) => { - const matchedMessage = recordSlackScenarioMessages({ ...observationContext, messages }); - return matchedMessage - ? { message: matchedMessage, observedAt: new Date().toISOString() } - : undefined; - }; - - while (Date.now() - startedAt < params.timeoutMs) { - const channelMessages = await listSlackMessages({ - channelId: params.channelId, - client: params.client, - oldestTs: params.sentTs, - }); - const channelReply = inspectMessages(channelMessages); - if (channelReply) { - return channelReply; - } - - try { - const threadMessages = await listSlackThreadMessages({ - channelId: params.channelId, - client: params.client, - threadTs: params.threadTs ?? params.sentTs, - }); - const threadReply = inspectMessages(threadMessages); - if (threadReply) { - return threadReply; - } - } catch (error) { - throw new Error( - `Slack conversations.replies failed while waiting for ${params.observationScenarioId}: ${formatErrorMessage(error)}`, - { cause: error }, - ); - } - await new Promise((resolve) => { - setTimeout(resolve, 1_000); - }); - } - throw new Error(`timed out after ${params.timeoutMs}ms waiting for Slack message`); -} - -async function observeSlackScenarioMessages( - params: SlackScenarioObservationContext & { - client: WebClient; - settleMs: number; - threadTs?: string; - }, -) { - const observationContext: SlackScenarioObservationContext = params; - const startedAt = Date.now(); - - while (true) { - recordSlackScenarioMessages({ - ...observationContext, - messages: await listSlackMessages({ - channelId: params.channelId, - client: params.client, - oldestTs: params.sentTs, - }), - }); - try { - recordSlackScenarioMessages({ - ...observationContext, - messages: await listSlackThreadMessages({ - channelId: params.channelId, - client: params.client, - threadTs: params.threadTs ?? params.sentTs, - }), - }); - } catch (error) { - throw new Error( - `Slack conversations.replies failed while settling ${params.observationScenarioId}: ${formatErrorMessage(error)}`, - { cause: error }, - ); - } - const remainingMs = params.settleMs - (Date.now() - startedAt); - if (remainingMs <= 0) { - return; - } - await new Promise((resolve) => { - setTimeout(resolve, Math.min(1_000, remainingMs)); - }); - } -} - -async function waitForSlackNoReply(params: { - channelId: string; - client: WebClient; - matchText: string; - observedMessages: SlackObservedMessage[]; - observationScenarioId: string; - observationScenarioTitle: string; - sentTs: string; - sutIdentity: SlackAuthIdentity; - timeoutMs: number; -}) { - const startedAt = Date.now(); - const observedKeys = new Set( - params.observedMessages - .map((message) => `${message.channelId ?? params.channelId}:${message.ts ?? ""}`) - .filter((key) => !key.endsWith(":")), - ); - let elapsedMs = Date.now() - startedAt; - while (elapsedMs < params.timeoutMs) { - const messages = await listSlackMessages({ - channelId: params.channelId, - client: params.client, - oldestTs: params.sentTs, - }); - for (const message of messages) { - const text = message.text ?? ""; - if ( - !message.ts || - message.ts === params.sentTs || - !isSutSlackMessage(message, params.sutIdentity) - ) { - continue; - } - const matchedScenario = text.includes(params.matchText); - const observedKey = `${params.channelId}:${message.ts}`; - if (!observedKeys.has(observedKey)) { - observedKeys.add(observedKey); - params.observedMessages.push({ - actionValues: collectSlackActionValues(message.blocks), - blockText: collectSlackBlockText(message.blocks), - botId: message.bot_id, - channelId: params.channelId, - matchedScenario, - scenarioId: params.observationScenarioId, - scenarioTitle: params.observationScenarioTitle, - text, - threadTs: message.thread_ts, - ts: message.ts, - userId: message.user, - }); - } - if (matchedScenario) { - throw new Error("unexpected Slack SUT reply observed"); - } - } - elapsedMs = Date.now() - startedAt; - const remainingMs = params.timeoutMs - elapsedMs; - if (remainingMs > 0) { - await new Promise((resolve) => { - setTimeout(resolve, Math.min(1_000, remainingMs)); - }); - } - elapsedMs = Date.now() - startedAt; - } -} - -function resolveApprovalDecisionLabel(decision: SlackQaApprovalDecision) { - return decision === "allow-once" - ? "Allowed once" - : decision === "allow-always" - ? "Allowed always" - : "Denied"; -} - -function resolveApprovalHeading(params: { - approvalKind: SlackQaApprovalKind; - state: "pending" | "resolved"; - decision?: SlackQaApprovalDecision; -}) { - if (params.state === "pending") { - return params.approvalKind === "exec" ? "Exec approval required" : "Plugin approval required"; - } - const label = resolveApprovalDecisionLabel(params.decision ?? "allow-once"); - return params.approvalKind === "exec" ? `Exec approval: ${label}` : `Plugin approval: ${label}`; -} - -function getSlackMessageSearchText(message: SlackMessage) { - return [message.text ?? "", ...collectSlackBlockText(message.blocks)].join("\n"); -} - -function pushObservedApprovalMessage(params: { - channelId: string; - matchedScenario: boolean; - message: SlackMessage; - observedMessages: SlackObservedMessage[]; - scenarioId: string; - scenarioTitle: string; -}) { - if (!params.message.ts) { - return; - } - params.observedMessages.push({ - actionValues: collectSlackActionValues(params.message.blocks), - blockText: collectSlackBlockText(params.message.blocks), - botId: params.message.bot_id, - channelId: params.channelId, - matchedScenario: params.matchedScenario, - scenarioId: params.scenarioId, - scenarioTitle: params.scenarioTitle, - text: params.message.text ?? "", - threadTs: params.message.thread_ts, - ts: params.message.ts, - userId: params.message.user, - }); -} - -async function waitForSlackApprovalPrompt(params: { - approvalId?: string; - approvalKind: SlackQaApprovalKind; - channelId: string; - client: WebClient; - decision: SlackQaApprovalDecision; - observedMessages: SlackObservedMessage[]; - oldestTs: string; - scenarioId: string; - scenarioTitle: string; - sutIdentity: SlackAuthIdentity; - timeoutMs: number; - token?: string; - extraTextMatches?: string[]; -}) { - const startedAt = Date.now(); - const seenObservedMessages = new Set(); - let lastMatchedWithoutActions = ""; - while (Date.now() - startedAt < params.timeoutMs) { - const messages = await listSlackMessages({ - channelId: params.channelId, - client: params.client, - oldestTs: params.oldestTs, - }); - for (const message of messages) { - if (!message.ts || !isSutSlackMessage(message, params.sutIdentity)) { - continue; - } - const text = getSlackMessageSearchText(message); - const actionValues = collectSlackActionValues(message.blocks); - const matchedScenario = matchesSlackApprovalPromptText({ - approvalKind: params.approvalKind, - extraTextMatches: params.extraTextMatches, - text, - token: params.token, - }); - const observedKey = `${message.ts}:${message.text ?? ""}:${actionValues.join("|")}`; - if (matchedScenario || hasSlackNativeApprovalActions({ ...params, actionValues })) { - if (!seenObservedMessages.has(observedKey)) { - seenObservedMessages.add(observedKey); - pushObservedApprovalMessage({ - channelId: params.channelId, - matchedScenario, - message, - observedMessages: params.observedMessages, - scenarioId: params.scenarioId, - scenarioTitle: params.scenarioTitle, - }); - } - } - if (!matchedScenario) { - continue; - } - if ( - !hasSlackNativeApprovalActions({ - actionValues, - approvalId: params.approvalId, - decision: params.decision, - }) - ) { - lastMatchedWithoutActions = `message ${message.ts} matched approval text but did not expose native approval button values`; - continue; - } - return { - actionValues, - approvalId: - params.approvalId ?? - extractSlackNativeApprovalId({ - actionValues, - decision: params.decision, - }), - message, - observedAt: new Date().toISOString(), - }; - } - await new Promise((resolve) => { - setTimeout(resolve, 1_000); - }); - } - throw new Error( - [ - `timed out after ${params.timeoutMs}ms waiting for Slack ${params.approvalKind} approval prompt`, - lastMatchedWithoutActions, - ] - .filter(Boolean) - .join("; "), - ); -} - -function matchesSlackApprovalPromptText(params: { - approvalKind: SlackQaApprovalKind; - extraTextMatches?: string[]; - text: string; - token?: string; -}) { - return ( - params.text.includes( - resolveApprovalHeading({ approvalKind: params.approvalKind, state: "pending" }), - ) && - (!params.token || params.text.includes(params.token)) && - (params.extraTextMatches ?? []).every((match) => params.text.includes(match)) - ); -} - -async function waitForSlackApprovalResolvedUpdate(params: { - approvalKind: SlackQaApprovalKind; - channelId: string; - client: WebClient; - decision: SlackQaApprovalDecision; - messageTs: string; - observedMessages: SlackObservedMessage[]; - oldestTs: string; - scenarioId: string; - scenarioTitle: string; - sutIdentity: SlackAuthIdentity; - timeoutMs: number; - token?: string; - extraTextMatches?: string[]; -}) { - const startedAt = Date.now(); - const seenObservedMessages = new Set(); - while (Date.now() - startedAt < params.timeoutMs) { - const messages = await listSlackMessages({ - channelId: params.channelId, - client: params.client, - oldestTs: params.oldestTs, - }); - const message = messages.find((entry) => entry.ts === params.messageTs); - if (message && isSutSlackMessage(message, params.sutIdentity)) { - const text = getSlackMessageSearchText(message); - const actionValues = collectSlackActionValues(message.blocks); - const matchedScenario = matchesSlackApprovalResolvedUpdate({ - actionValues, - approvalKind: params.approvalKind, - decision: params.decision, - extraTextMatches: params.extraTextMatches, - text, - token: params.token, - }); - const observedKey = `${message.ts}:${message.text ?? ""}:${actionValues.join("|")}`; - if (!seenObservedMessages.has(observedKey)) { - seenObservedMessages.add(observedKey); - pushObservedApprovalMessage({ - channelId: params.channelId, - matchedScenario, - message, - observedMessages: params.observedMessages, - scenarioId: params.scenarioId, - scenarioTitle: params.scenarioTitle, - }); - } - if (matchedScenario) { - return { - actionValues, - message, - observedAt: new Date().toISOString(), - }; - } - } - await new Promise((resolve) => { - setTimeout(resolve, 1_000); - }); - } - throw new Error( - `timed out after ${params.timeoutMs}ms waiting for Slack ${params.approvalKind} approval resolution update`, - ); -} - -function matchesSlackApprovalResolvedUpdate(params: { - actionValues: string[]; - approvalKind: SlackQaApprovalKind; - decision: SlackQaApprovalDecision; - extraTextMatches?: string[]; - text: string; - token?: string; -}) { - return ( - params.text.includes( - resolveApprovalHeading({ - approvalKind: params.approvalKind, - decision: params.decision, - state: "resolved", - }), - ) && - (!params.token || params.text.includes(params.token)) && - (params.extraTextMatches ?? []).every((match) => params.text.includes(match)) && - !params.actionValues.some((value) => parseSlackNativeApprovalAction(value)) - ); -} - -function resolveSlackApprovalCheckpointConfig(env: NodeJS.ProcessEnv = process.env) { - const checkpointDir = env[SLACK_QA_APPROVAL_CHECKPOINT_DIR_ENV]?.trim(); - if (!checkpointDir) { - return undefined; - } - const rawTimeout = env[SLACK_QA_APPROVAL_CHECKPOINT_TIMEOUT_MS_ENV]?.trim(); - const timeoutMs = rawTimeout - ? parseStrictPositiveInteger(rawTimeout) - : SLACK_QA_APPROVAL_CHECKPOINT_DEFAULT_TIMEOUT_MS; - if (timeoutMs === undefined) { - throw new Error(`${SLACK_QA_APPROVAL_CHECKPOINT_TIMEOUT_MS_ENV} must be a positive integer.`); - } - return { - checkpointDir, - timeoutMs, - }; -} - -async function waitForSlackApprovalCheckpointAck(params: { - ackPath: string; - timeoutMs: number; -}): Promise { - const startedAt = Date.now(); - while (Date.now() - startedAt < params.timeoutMs) { - try { - const parsed = JSON.parse(await fs.readFile(params.ackPath, "utf8")) as { - capturedAt?: unknown; - error?: unknown; - screenshotPath?: unknown; - }; - if (typeof parsed.error === "string" && parsed.error.trim().length > 0) { - throw new Error(`Slack approval checkpoint watcher failed: ${parsed.error}`); - } - return { - capturedAt: typeof parsed.capturedAt === "string" ? parsed.capturedAt : undefined, - screenshotPath: - typeof parsed.screenshotPath === "string" ? parsed.screenshotPath : undefined, - }; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - throw error; - } - } - await new Promise((resolve) => { - setTimeout(resolve, 500); - }); - } - throw new Error(`timed out after ${params.timeoutMs}ms waiting for ${params.ackPath}`); -} - -async function writeSlackApprovalCheckpoint(params: { - approvalId: string; - approvalKind: SlackQaApprovalKind; - channelId: string; - decision?: SlackQaApprovalDecision; - message: SlackMessage; - observedAt: string; - scenarioId: SlackQaScenarioId; - state: SlackApprovalCheckpointState; -}) { - const config = resolveSlackApprovalCheckpointConfig(); - if (!config) { - return undefined; - } - await fs.mkdir(config.checkpointDir, { recursive: true }); - const checkpointPath = path.join( - config.checkpointDir, - `${params.scenarioId}.${params.state}.json`, - ); - const ackPath = path.join(config.checkpointDir, `${params.scenarioId}.${params.state}.ack.json`); - await fs.rm(ackPath, { force: true }).catch(() => {}); - await fs.writeFile( - checkpointPath, - `${JSON.stringify( - { - version: 1, - scenarioId: params.scenarioId, - approvalKind: params.approvalKind, - state: params.state, - approvalId: params.approvalId, - channelId: params.channelId, - messageTs: params.message.ts, - threadTs: params.message.thread_ts ?? null, - decision: params.decision ?? null, - observedAt: params.observedAt, - message: buildSlackApprovalCheckpointMessage(params.message), - }, - null, - 2, - )}\n`, - ); - const ack = await waitForSlackApprovalCheckpointAck({ - ackPath, - timeoutMs: config.timeoutMs, - }); - return { - ackPath, - checkpointPath, - screenshotPath: ack.screenshotPath, - }; -} - -async function requestSlackApproval(params: { - approvalId: string; - channelId: string; - context: Omit; - run: SlackQaApprovalScenarioRun; - sutAccountId: string; -}) { - const commonParams = { - timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS, - turnSourceAccountId: params.sutAccountId, - turnSourceChannel: "slack", - turnSourceTo: `channel:${params.channelId}`, - twoPhase: true, - }; - if (params.run.approvalKind === "exec") { - const result = await params.context.gateway.call( - "exec.approval.request", - { - ...commonParams, - ask: "always", - command: `printf '%s\\n' '${params.run.token}'`, - host: "gateway", - id: params.approvalId, - security: "full", - }, - { - expectFinal: false, - timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, - }, - ); - const acceptedId = readAcceptedApprovalRequestId(result); - if (acceptedId !== params.approvalId) { - throw new Error( - `accepted exec approval id was ${formatApprovalResultValue( - acceptedId, - )} instead of ${params.approvalId}`, - ); - } - return acceptedId; - } - const result = await params.context.gateway.call( - "plugin.approval.request", - { - ...commonParams, - agentId: "qa", - description: `Slack plugin approval QA request ${params.run.token}`, - pluginId: "qa-slack-plugin", - severity: "warning", - title: `Slack plugin approval QA ${params.run.token}`, - toolName: "slack_qa_tool", - }, - { - expectFinal: false, - timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, - }, - ); - return readAcceptedApprovalRequestId(result); -} - -async function waitForApprovalDecision(params: { - approvalId: string; - context: Omit; - kind: SlackQaApprovalKind; -}) { - const method = - params.kind === "exec" ? "exec.approval.waitDecision" : "plugin.approval.waitDecision"; - return await params.context.gateway.call( - method, - { id: params.approvalId }, - { - expectFinal: true, - timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, - }, - ); -} - -async function resolveApprovalDecision(params: { - approvalId: string; - context: Omit; - decision: SlackQaApprovalDecision; - kind: SlackQaApprovalKind; -}) { - const method = params.kind === "exec" ? "exec.approval.resolve" : "plugin.approval.resolve"; - return await params.context.gateway.call( - method, - { decision: params.decision, id: params.approvalId }, - { - expectFinal: false, - timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, - }, - ); -} - -async function runSlackApprovalScenario(params: { - channelId: string; - context: Omit; - observedMessages: SlackObservedMessage[]; - run: SlackQaApprovalScenarioRun; - scenario: SlackQaScenarioDefinition; - sutAccountId: string; -}) { - const requestStartedAt = new Date(); - const oldestTs = ((requestStartedAt.getTime() - 5_000) / 1_000).toFixed(6); - const requestedApprovalId = - params.run.approvalKind === "exec" - ? `slack-qa-exec-${randomUUID()}` - : `slack-qa-plugin-${randomUUID()}`; - const approvalId = await requestSlackApproval({ - approvalId: requestedApprovalId, - channelId: params.channelId, - context: params.context, - run: params.run, - sutAccountId: params.sutAccountId, - }); - const pending = await waitForSlackApprovalPrompt({ - approvalId, - approvalKind: params.run.approvalKind, - channelId: params.channelId, - client: params.context.sutReadClient, - decision: params.run.decision, - observedMessages: params.observedMessages, - oldestTs, - scenarioId: params.scenario.id, - scenarioTitle: params.scenario.title, - sutIdentity: params.context.sutIdentity, - timeoutMs: params.scenario.timeoutMs, - token: params.run.token, - }); - const pendingCheckpoint = await writeSlackApprovalCheckpoint({ - approvalId, - approvalKind: params.run.approvalKind, - channelId: params.channelId, - message: pending.message, - observedAt: pending.observedAt, - scenarioId: params.scenario.id, - state: "pending", - }); - await resolveApprovalDecision({ - approvalId, - context: params.context, - decision: params.run.decision, - kind: params.run.approvalKind, - }); - assertApprovalDecisionResult({ - decision: params.run.decision, - result: await waitForApprovalDecision({ - approvalId, - context: params.context, - kind: params.run.approvalKind, - }), - }); - const resolved = await waitForSlackApprovalResolvedUpdate({ - approvalKind: params.run.approvalKind, - channelId: params.channelId, - client: params.context.sutReadClient, - decision: params.run.decision, - messageTs: pending.message.ts, - observedMessages: params.observedMessages, - oldestTs, - scenarioId: params.scenario.id, - scenarioTitle: params.scenario.title, - sutIdentity: params.context.sutIdentity, - timeoutMs: params.scenario.timeoutMs, - token: params.run.token, - }); - const resolvedCheckpoint = await writeSlackApprovalCheckpoint({ - approvalId, - approvalKind: params.run.approvalKind, - channelId: params.channelId, - decision: params.run.decision, - message: resolved.message, - observedAt: resolved.observedAt, - scenarioId: params.scenario.id, - state: "resolved", - }); - const responseObservedAt = new Date(resolved.observedAt); - return { - artifact: { - approvalId, - approvalKind: params.run.approvalKind, - channelId: params.channelId, - decision: params.run.decision, - pendingActionValues: pending.actionValues, - pendingCheckpointPath: pendingCheckpoint?.checkpointPath, - pendingMessageTs: pending.message.ts, - pendingScreenshotPath: pendingCheckpoint?.screenshotPath, - pendingText: pending.message.text, - resolvedActionValues: resolved.actionValues, - resolvedCheckpointPath: resolvedCheckpoint?.checkpointPath, - resolvedMessageTs: resolved.message.ts, - resolvedScreenshotPath: resolvedCheckpoint?.screenshotPath, - resolvedText: resolved.message.text, - threadTs: pending.message.thread_ts, - } satisfies SlackApprovalArtifact, - requestStartedAt, - responseObservedAt, - rttMs: responseObservedAt.getTime() - requestStartedAt.getTime(), - }; -} - -function buildCodexApprovalInstruction(params: { - appServerMethod: SlackQaCodexApprovalMethod; - token: string; -}) { - if (params.appServerMethod === "item/commandExecution/requestApproval") { - return [ - "Slack QA Codex command approval scenario.", - `Marker: ${params.token}`, - "Use the shell tool exactly once and request approval before running it.", - `Run exactly: printf '${params.token}\\n'`, - `After the command succeeds, reply exactly ${params.token} and nothing else.`, - ].join("\n"); - } - const targetPath = resolveCodexFileApprovalTargetPath(params.token); - return [ - "Slack QA Codex file approval scenario.", - `Marker: ${params.token}`, - "Use apply_patch exactly once to create this file outside the workspace.", - `Target file: ${targetPath}`, - `File contents: ${params.token}`, - "Invoke apply_patch now. Do not ask for approval in chat; the harness will resolve the native tool approval.", - `After the file change succeeds, reply exactly ${params.token} and nothing else.`, - ].join("\n"); -} - -function readAcceptedAgentRunId(result: unknown) { - const started = - typeof result === "object" && result !== null - ? (result as { runId?: unknown; status?: unknown }) - : null; - if (started?.status !== "accepted") { - throw new Error( - `Codex agent run status was ${formatApprovalResultValue(started?.status)} instead of accepted`, - ); - } - if (typeof started.runId !== "string" || started.runId.trim().length === 0) { - throw new Error(`Codex agent run id was ${formatApprovalResultValue(started.runId)}`); - } - return started.runId; -} - -function readAgentWaitStatus(result: unknown) { - if (typeof result !== "object" || result === null) { - return "unknown"; - } - const status = (result as { status?: unknown }).status; - return typeof status === "string" && status.trim() ? status : "unknown"; -} - -async function waitForSlackReaction(params: { - channelId: string; - client: WebClient; - expectedReactionName: string; - messageId: string; - sutUserId: string; - timeoutMs: number; -}) { - const deadline = Date.now() + params.timeoutMs; - while (true) { - const reactions = await listSlackReactions(params.channelId, params.messageId, { - client: params.client, - }); - const reaction = reactions?.find( - (entry) => - entry.name === params.expectedReactionName && entry.users?.includes(params.sutUserId), - ); - if (reaction) { - return reaction; - } - if (Date.now() >= deadline) { - break; - } - await new Promise((resolve) => { - setTimeout(resolve, 1_000); - }); - } - throw new Error( - `Slack message ${params.messageId} did not receive ${params.expectedReactionName} from ${params.sutUserId}`, - ); -} - -function assertCodexApprovalTranscriptSucceeded( - messages: unknown, - run: SlackQaCodexApprovalScenarioRun, -) { - const records = Array.isArray(messages) ? messages.map(asPlainRecord) : []; - const assistantReply = records - .toReversed() - .find((message) => message.role === "assistant" && extractGatewayMessageText(message)); - if (!assistantReply || extractGatewayMessageText(assistantReply) !== run.token) { - throw new Error(`Codex approval run did not finish with assistant marker ${run.token}`); - } - if (run.appServerMethod !== "item/commandExecution/requestApproval") { - return; - } - const commandSucceeded = records.some((message) => { - if (message.role !== "toolResult" || message.isError === true) { - return false; - } - return extractGatewayMessageText(message) - .split(/\r?\n/u) - .some((line) => line.trim() === run.token); - }); - if (!commandSucceeded) { - throw new Error(`Codex command result did not contain marker ${run.token}`); - } -} - -async function assertCodexApprovalOperationSucceeded(params: { - context: Omit; - run: SlackQaCodexApprovalScenarioRun; - sessionKey: string; -}) { - const history = asPlainRecord( - await params.context.gateway.call( - "chat.history", - { sessionKey: params.sessionKey, limit: 24 }, - { timeoutMs: 10_000 }, - ), - ); - assertCodexApprovalTranscriptSucceeded(history.messages, params.run); - if (params.run.appServerMethod !== "item/fileChange/requestApproval") { - return; - } - const targetPath = resolveCodexFileApprovalTargetPath(params.run.token); - const contents = await fs.readFile(targetPath, "utf8"); - if (contents.trim() !== params.run.token) { - throw new Error(`Codex file result at ${targetPath} did not contain the expected marker`); - } -} - -function findPendingCodexPluginApprovalRecord(params: { - approvalId: string; - appServerMethod: SlackQaCodexApprovalMethod; - channelId: string; - records: unknown; - sessionKey: string; - sutAccountId: string; -}) { - const list = Array.isArray(params.records) ? params.records : []; - const expectedTitle = - params.appServerMethod === "item/commandExecution/requestApproval" - ? "Codex app-server command approval" - : "Codex app-server file approval"; - const expectedToolName = - params.appServerMethod === "item/commandExecution/requestApproval" - ? "codex_command_approval" - : "codex_file_approval"; - for (const entry of list) { - const record = asPlainRecord(entry); - if (record.id !== params.approvalId) { - continue; - } - const request = asPlainRecord(record.request); - if ( - request.pluginId === "openclaw-codex-app-server" && - request.title === expectedTitle && - request.toolName === expectedToolName && - request.sessionKey === params.sessionKey && - request.turnSourceChannel === "slack" && - request.turnSourceTo === `channel:${params.channelId}` && - request.turnSourceAccountId === params.sutAccountId - ) { - return record; - } - } - return undefined; -} - -async function assertPendingCodexPluginApproval(params: { - approvalId: string; - appServerMethod: SlackQaCodexApprovalMethod; - channelId: string; - context: Omit; - sessionKey: string; - sutAccountId: string; -}) { - const records = await params.context.gateway.call( - "plugin.approval.list", - {}, - { - timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS, - }, - ); - const record = findPendingCodexPluginApprovalRecord({ - approvalId: params.approvalId, - appServerMethod: params.appServerMethod, - channelId: params.channelId, - records, - sessionKey: params.sessionKey, - sutAccountId: params.sutAccountId, - }); - if (!record) { - throw new Error( - `Pending Codex plugin approval ${params.approvalId} did not match the expected app-server route and Slack turn source.`, - ); - } -} - -async function startCodexApprovalAgentRun(params: { - channelId: string; - context: Omit; - primaryModel: string; - run: SlackQaCodexApprovalScenarioRun; - runId: string; - scenario: SlackQaScenarioDefinition; - sessionKey: string; - sutAccountId: string; -}) { - const result = await params.context.gateway.call( - "agent", - { - accountId: params.sutAccountId, - agentId: "qa", - channel: "slack", - cleanupBundleMcpOnRunEnd: true, - deliver: false, - idempotencyKey: params.runId, - message: buildCodexApprovalInstruction({ - appServerMethod: params.run.appServerMethod, - token: params.run.token, - }), - model: params.primaryModel, - sessionKey: params.sessionKey, - thinking: "low", - timeout: Math.ceil(params.scenario.timeoutMs / 1_000), - to: `channel:${params.channelId}`, - }, - { - timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, - }, - ); - const acceptedRunId = readAcceptedAgentRunId(result); - if (acceptedRunId !== params.runId) { - throw new Error(`Codex agent run id was ${acceptedRunId} instead of ${params.runId}`); - } -} - -function buildCodexApprovalSessionKey(params: { - scenario: SlackQaScenarioDefinition; - token: string; -}) { - return `agent:qa:${params.scenario.id}-${params.token.toLowerCase()}`; -} - -async function waitForCodexApprovalAgentRun(params: { - context: Omit; - runId: string; - timeoutMs: number; -}) { - const result = await params.context.gateway.call( - "agent.wait", - { - runId: params.runId, - timeoutMs: params.timeoutMs, - }, - { - timeoutMs: params.timeoutMs + 5_000, - }, - ); - return readAgentWaitStatus(result); -} - -async function quiesceCodexApprovalAgentRun(params: { - context: Omit; - preserveDebugArtifacts: boolean; - runId: string; - sessionKey: string; - stopGateway: (preserveDebugArtifacts: boolean) => Promise; -}) { - try { - await params.context.gateway.call( - "chat.abort", - { runId: params.runId, sessionKey: params.sessionKey }, - { timeoutMs: 10_000 }, - ); - } catch { - // The bounded terminal wait and gateway process-group teardown do not depend on this ack. - } - try { - await params.context.gateway.call( - "agent.wait", - { runId: params.runId, timeoutMs: 10_000 }, - { timeoutMs: 15_000 }, - ); - } catch { - // QA-owned Codex app-server processes inherit the gateway cleanup process group. - } - await params.stopGateway(params.preserveDebugArtifacts); -} - -async function runSlackCodexApprovalScenario(params: { - channelId: string; - context: Omit; - observedMessages: SlackObservedMessage[]; - primaryModel: string; - run: SlackQaCodexApprovalScenarioRun; - scenario: SlackQaScenarioDefinition; - stopGateway: (preserveDebugArtifacts: boolean) => Promise; - sutAccountId: string; -}) { - const codexRun = { - runId: `slack-qa-codex-approval-${randomUUID()}`, - sessionKey: buildCodexApprovalSessionKey({ - scenario: params.scenario, - token: params.run.token, - }), - }; - const targetPath = - params.run.appServerMethod === "item/fileChange/requestApproval" - ? resolveCodexFileApprovalTargetPath(params.run.token) - : undefined; - if (targetPath) { - await fs.rm(targetPath, { force: true }); - } - const outcome = await runSlackCodexApprovalScenarioInner({ ...params, codexRun }).then( - (result) => ({ kind: "success", result }) as const, - (error: unknown) => ({ error, kind: "failure" }) as const, - ); - // Kill the gateway process tree before deleting the probe. Agent completion - // does not prove the native Codex turn has stopped writing after an interrupt. - const cleanupErrors: unknown[] = []; - try { - await quiesceCodexApprovalAgentRun({ - context: params.context, - preserveDebugArtifacts: outcome.kind === "failure", - stopGateway: params.stopGateway, - ...codexRun, - }); - } catch (error) { - cleanupErrors.push(error); - } - if (cleanupErrors.length === 0 && targetPath) { - try { - await fs.rm(targetPath, { force: true }); - } catch (error) { - cleanupErrors.push(error); - } - } - if (cleanupErrors.length > 0) { - const cleanupSummary = cleanupErrors.map(formatErrorMessage).join("; "); - if (outcome.kind === "failure") { - throw new AggregateError( - [outcome.error, ...cleanupErrors], - `Codex approval scenario failed: ${formatErrorMessage(outcome.error)}; cleanup also failed: ${cleanupSummary}`, - { cause: outcome.error }, - ); - } - throw new AggregateError(cleanupErrors, `Codex approval cleanup failed: ${cleanupSummary}`); - } - if (outcome.kind === "failure") { - throw outcome.error; - } - return outcome.result; -} - -function resolveCodexFileApprovalTargetPath(token: string) { - return path.join(os.homedir(), `.openclaw-qa-codex-file-approval-${token.toLowerCase()}.txt`); -} - -async function runSlackCodexApprovalScenarioInner(params: { - channelId: string; - codexRun: { runId: string; sessionKey: string }; - context: Omit; - observedMessages: SlackObservedMessage[]; - primaryModel: string; - run: SlackQaCodexApprovalScenarioRun; - scenario: SlackQaScenarioDefinition; - sutAccountId: string; -}) { - const requestStartedAt = new Date(); - const oldestTs = ((requestStartedAt.getTime() - 5_000) / 1_000).toFixed(6); - await startCodexApprovalAgentRun({ - channelId: params.channelId, - context: params.context, - primaryModel: params.primaryModel, - run: params.run, - runId: params.codexRun.runId, - scenario: params.scenario, - sessionKey: params.codexRun.sessionKey, - sutAccountId: params.sutAccountId, - }); - const expectedTitle = - params.run.appServerMethod === "item/commandExecution/requestApproval" - ? "Codex app-server command approval" - : "Codex app-server file approval"; - const pending = await waitForSlackApprovalPrompt({ - approvalKind: params.run.approvalKind, - channelId: params.channelId, - client: params.context.sutReadClient, - decision: params.run.decision, - extraTextMatches: ["openclaw-codex-app-server", expectedTitle], - observedMessages: params.observedMessages, - oldestTs, - scenarioId: params.scenario.id, - scenarioTitle: params.scenario.title, - sutIdentity: params.context.sutIdentity, - timeoutMs: params.scenario.timeoutMs, - }); - const approvalId = pending.approvalId; - if (!approvalId) { - throw new Error( - "Codex Slack approval prompt exposed native actions but no plugin approval id.", - ); - } - await assertPendingCodexPluginApproval({ - approvalId, - appServerMethod: params.run.appServerMethod, - channelId: params.channelId, - context: params.context, - sessionKey: params.codexRun.sessionKey, - sutAccountId: params.sutAccountId, - }); - const pendingCheckpoint = await writeSlackApprovalCheckpoint({ - approvalId, - approvalKind: params.run.approvalKind, - channelId: params.channelId, - message: pending.message, - observedAt: pending.observedAt, - scenarioId: params.scenario.id, - state: "pending", - }); - await resolveApprovalDecision({ - approvalId, - context: params.context, - decision: params.run.decision, - kind: params.run.approvalKind, - }); - const finalCodexTurnStatus = await waitForCodexApprovalAgentRun({ - context: params.context, - runId: params.codexRun.runId, - timeoutMs: params.scenario.timeoutMs, - }); - if (finalCodexTurnStatus !== "ok") { - throw new Error( - `Codex approval run ${params.codexRun.runId} finished with status ${finalCodexTurnStatus}`, - ); - } - await assertCodexApprovalOperationSucceeded({ - context: params.context, - run: params.run, - sessionKey: params.codexRun.sessionKey, - }); - const resolved = await waitForSlackApprovalResolvedUpdate({ - approvalKind: params.run.approvalKind, - channelId: params.channelId, - client: params.context.sutReadClient, - decision: params.run.decision, - messageTs: pending.message.ts, - observedMessages: params.observedMessages, - oldestTs, - scenarioId: params.scenario.id, - scenarioTitle: params.scenario.title, - sutIdentity: params.context.sutIdentity, - timeoutMs: params.scenario.timeoutMs, - extraTextMatches: ["openclaw-codex-app-server", expectedTitle], - }); - const resolvedCheckpoint = await writeSlackApprovalCheckpoint({ - approvalId, - approvalKind: params.run.approvalKind, - channelId: params.channelId, - decision: params.run.decision, - message: resolved.message, - observedAt: resolved.observedAt, - scenarioId: params.scenario.id, - state: "resolved", - }); - const responseObservedAt = new Date(resolved.observedAt); - return { - artifact: { - approvalId, - approvalKind: params.run.approvalKind, - appServerMethod: params.run.appServerMethod, - channelId: params.channelId, - codexModelKey: params.primaryModel, - decision: params.run.decision, - finalCodexTurnStatus, - operationVerified: true, - pendingActionValues: pending.actionValues, - pendingCheckpointPath: pendingCheckpoint?.checkpointPath, - pendingMessageTs: pending.message.ts, - pendingScreenshotPath: pendingCheckpoint?.screenshotPath, - pendingText: pending.message.text, - resolvedActionValues: resolved.actionValues, - resolvedCheckpointPath: resolvedCheckpoint?.checkpointPath, - resolvedMessageTs: resolved.message.ts, - resolvedScreenshotPath: resolvedCheckpoint?.screenshotPath, - resolvedText: resolved.message.text, - threadTs: pending.message.thread_ts, - } satisfies SlackApprovalArtifact, - requestStartedAt, - responseObservedAt, - rttMs: responseObservedAt.getTime() - requestStartedAt.getTime(), - }; -} - -async function waitForSlackChannelRunning( - gateway: Awaited>, - accountId: string, - mode: SlackChannelReadinessMode, -): Promise { - const startedAt = Date.now(); - const timeoutMs = resolveSlackQaReadyTimeoutMs(); - let lastStatus: SlackChannelStatus | undefined; - while (Date.now() - startedAt < timeoutMs) { - try { - const payload = (await gateway.call( - "channels.status", - { probe: false, timeoutMs: 2_000 }, - { timeoutMs: 5_000 }, - )) as { - channelAccounts?: Record< - string, - Array<{ - accountId?: string; - connected?: boolean; - lastConnectedAt?: number; - lastDisconnect?: unknown; - lastError?: string | null; - restartPending?: boolean; - running?: boolean; - }> - >; - }; - const accounts = payload.channelAccounts?.slack ?? []; - const match = accounts.find((entry) => entry.accountId === accountId); - lastStatus = match - ? { - connected: match.connected, - lastConnectedAt: match.lastConnectedAt, - lastDisconnect: match.lastDisconnect, - lastError: match.lastError, - restartPending: match.restartPending, - running: match.running, - } - : undefined; - if (isSlackChannelReadyForQa(lastStatus, mode)) { - if (!lastStatus) { - throw new Error(`slack account "${accountId}" status disappeared after readiness check`); - } - return lastStatus; - } - } catch { - // retry - } - await new Promise((resolve) => { - setTimeout(resolve, 500); - }); - } - throw new Error( - `slack account "${accountId}" did not become ready` + - (lastStatus ? `; last status: ${JSON.stringify(lastStatus)}` : ""), - ); -} - -async function waitForSlackChannelStable( - gateway: Awaited>, - accountId: string, - mode: SlackChannelReadinessMode, -) { - const startedAt = Date.now(); - const timeoutMs = resolveSlackQaReadyTimeoutMs(); - let readySince: number | undefined; - while (Date.now() - startedAt < timeoutMs) { - const status = await waitForSlackChannelRunning(gateway, accountId, mode); - const observedAt = Date.now(); - readySince = resolveSlackChannelReadySince({ - observedAt, - previousReadySince: readySince, - status, - }); - const readyForMs = observedAt - readySince; - if (readyForMs >= SLACK_QA_READY_STABILITY_MS) { - return; - } - await new Promise((resolve) => { - setTimeout(resolve, Math.max(500, SLACK_QA_READY_STABILITY_MS - readyForMs)); - }); - } - throw new Error( - `slack account "${accountId}" did not remain ready for ${SLACK_QA_READY_STABILITY_MS}ms`, - ); -} - -function isSlackChannelReadyForQa( - status: SlackChannelStatus | undefined, - mode: SlackChannelReadinessMode, -): boolean { - if ( - !status?.running || - status.restartPending === true || - status.lastError != null || - status.connected === false - ) { - return false; - } - return mode === "started" || status.connected === true; -} - -function resolveSlackChannelReadySince(params: { - observedAt: number; - previousReadySince: number | undefined; - status: SlackChannelStatus; -}): number { - if (typeof params.status.lastConnectedAt === "number" && params.status.lastConnectedAt > 0) { - return params.status.lastConnectedAt; - } - return params.previousReadySince ?? params.observedAt; -} - -function resolveSlackQaReadyTimeoutMs(env: NodeJS.ProcessEnv = process.env) { - const raw = env.OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS; - if (!raw) { - return SLACK_QA_DEFAULT_READY_TIMEOUT_MS; - } - return parseStrictPositiveInteger(raw) ?? SLACK_QA_DEFAULT_READY_TIMEOUT_MS; -} - -function isRetryableSlackQaScenarioError(error: unknown) { - return /timed out after \d+ms waiting for Slack message/iu.test(formatErrorMessage(error)); -} - -function toObservedSlackArtifacts(params: { - includeContent: boolean; - messages: SlackObservedMessage[]; - redactMetadata: boolean; -}): SlackObservedMessageArtifact[] { - return params.messages.map((message) => ({ - actionValues: params.includeContent ? message.actionValues : undefined, - blockText: params.includeContent ? message.blockText : undefined, - botId: params.redactMetadata ? undefined : message.botId, - channelId: params.redactMetadata ? undefined : message.channelId, - matchedScenario: message.matchedScenario, - scenarioId: message.scenarioId, - scenarioTitle: message.scenarioTitle, - text: params.includeContent ? message.text : undefined, - threadTs: params.redactMetadata ? undefined : message.threadTs, - ts: params.redactMetadata ? undefined : message.ts, - userId: params.redactMetadata ? undefined : message.userId, - })); -} - -function toSlackQaScenarioArtifactResults(params: { - includeContent: boolean; - redactMetadata: boolean; - scenarios: SlackQaScenarioResult[]; -}): SlackQaScenarioResult[] { - return params.scenarios.map((scenario) => { - if (!scenario.approval) { - return scenario; - } - const approval = scenario.approval; - return { - ...scenario, - approval: { - approvalId: params.redactMetadata ? "" : approval.approvalId, - approvalKind: approval.approvalKind, - appServerMethod: approval.appServerMethod, - channelId: params.redactMetadata ? undefined : approval.channelId, - codexModelKey: approval.codexModelKey, - decision: approval.decision, - finalCodexTurnStatus: approval.finalCodexTurnStatus, - operationVerified: approval.operationVerified, - pendingActionValues: params.includeContent ? approval.pendingActionValues : undefined, - pendingCheckpointPath: approval.pendingCheckpointPath, - pendingMessageTs: params.redactMetadata ? undefined : approval.pendingMessageTs, - pendingScreenshotPath: approval.pendingScreenshotPath, - pendingText: params.includeContent ? approval.pendingText : undefined, - resolvedActionValues: params.includeContent ? approval.resolvedActionValues : undefined, - resolvedCheckpointPath: approval.resolvedCheckpointPath, - resolvedMessageTs: params.redactMetadata ? undefined : approval.resolvedMessageTs, - resolvedScreenshotPath: approval.resolvedScreenshotPath, - resolvedText: params.includeContent ? approval.resolvedText : undefined, - threadTs: params.redactMetadata ? undefined : approval.threadTs, - }, - }; - }); -} - -function renderSlackQaMarkdown(params: { - channelId: string; - cleanupIssues: string[]; - credentialSource: "convex" | "env"; - finishedAt: string; - gatewayDebugDirPath?: string; - redactMetadata: boolean; - scenarios: SlackQaScenarioResult[]; - startedAt: string; -}) { - const lines = [ - "# Slack QA Report", - "", - `- Credential source: \`${params.credentialSource}\``, - `- Channel: \`${params.redactMetadata ? "" : params.channelId}\``, - `- Metadata redaction: \`${params.redactMetadata ? "enabled" : "disabled"}\``, - `- Started: ${params.startedAt}`, - `- Finished: ${params.finishedAt}`, - ]; - if (params.gatewayDebugDirPath) { - lines.push(`- Gateway debug artifacts: \`${params.gatewayDebugDirPath}\``); - } - if (params.cleanupIssues.length > 0) { - lines.push("", "## Cleanup issues", ""); - for (const issue of params.cleanupIssues) { - lines.push(`- ${issue}`); - } - } - lines.push("", "## Scenarios", ""); - for (const scenario of params.scenarios) { - lines.push(`### ${scenario.title}`, ""); - lines.push(`- Status: ${scenario.status}`); - lines.push(`- Details: ${scenario.details}`); - if (scenario.rttMs !== undefined) { - lines.push(`- RTT: ${scenario.rttMs}ms`); - } - if (scenario.approval) { - lines.push(`- Approval kind: ${scenario.approval.approvalKind}`); - if (scenario.approval.appServerMethod) { - lines.push(`- Codex app-server method: \`${scenario.approval.appServerMethod}\``); - } - if (scenario.approval.codexModelKey) { - lines.push(`- Codex model: \`${scenario.approval.codexModelKey}\``); - } - if (scenario.approval.finalCodexTurnStatus) { - lines.push(`- Codex turn status: ${scenario.approval.finalCodexTurnStatus}`); - } - if (scenario.approval.operationVerified) { - lines.push("- Codex operation marker: verified"); - } - lines.push(`- Approval ID: \`${scenario.approval.approvalId}\``); - lines.push(`- Decision: ${scenario.approval.decision}`); - if (scenario.approval.pendingScreenshotPath) { - lines.push(`- Pending screenshot: \`${scenario.approval.pendingScreenshotPath}\``); - } - if (scenario.approval.resolvedScreenshotPath) { - lines.push(`- Resolved screenshot: \`${scenario.approval.resolvedScreenshotPath}\``); - } - } - lines.push(""); - } - return lines.join("\n"); -} - -async function preserveSlackGatewayDebugArtifacts(params: { - cleanupIssues: string[]; - gatewayDebugDirPath: string; - gatewayHarness: SlackQaGatewayHarness; -}) { - try { - await params.gatewayHarness.stop({ preserveToDir: params.gatewayDebugDirPath }); - return true; - } catch (error) { - appendLiveLaneIssue(params.cleanupIssues, "gateway debug preservation failed", error); - return false; - } -} + matchesSlackApprovalPromptText, + matchesSlackApprovalResolvedUpdate, + resolveApprovalDecision, + runSlackApprovalScenario, +} from "./slack-live.approvals.js"; +import { + isRetryableSlackQaScenarioError, + toObservedSlackArtifacts, + toSlackQaScenarioArtifactResults, + renderSlackQaMarkdown, + preserveSlackGatewayDebugArtifacts, +} from "./slack-live.artifacts.js"; +import { runSlackCodexApprovalScenario } from "./slack-live.codex-approval-runner.js"; +import { + buildCodexApprovalInstruction, + readAcceptedAgentRunId, + waitForSlackReaction, + assertCodexApprovalTranscriptSucceeded, + findPendingCodexPluginApprovalRecord, + quiesceCodexApprovalAgentRun, + resolveCodexFileApprovalTargetPath, +} from "./slack-live.codex-approval.js"; +import { + resolveSlackQaRuntimeEnv, + parseSlackQaCredentialPayload, + buildSlackQaConfig, +} from "./slack-live.config.js"; +import { + type SlackQaRuntimeEnv, + type SlackChannelReadinessMode, + SLACK_QA_GATEWAY_STOP_SETTLE_MS, + SLACK_QA_RETRYABLE_SCENARIO_ATTEMPTS, + assertSlackCodexApprovalModelSupported, + resolveSlackQaSutAccountId, + type SlackQaGatewayHarness, + type SlackObservedMessage, + type SlackQaScenarioResult, + type SlackQaRunResult, + type SlackCredentialLease, + type SlackCredentialHeartbeat, + SLACK_QA_CAPTURE_CONTENT_ENV, + QA_REDACT_PUBLIC_METADATA_ENV, + SLACK_QA_WEB_API_TIMEOUT_MS, +} from "./slack-live.contracts.js"; +import { buildSlackInvalidBlocksTableProbe } from "./slack-live.invalid-blocks.js"; +import { + waitForSlackScenarioReply, + observeSlackScenarioMessages, + waitForSlackNoReply, + waitForSlackChannelStable, + isSlackChannelReadyForQa, + resolveSlackChannelReadySince, + resolveSlackQaReadyTimeoutMs, +} from "./slack-live.message-observations.js"; +import { + getSlackIdentity, + sendSlackChannelMessage, + listSlackMessages, + listSlackThreadMessages, + collectSlackBlockText, + collectSlackActionValues, + parseSlackNativeApprovalAction, + collectSlackButtonLabels, + buildSlackApprovalCheckpointMessage, + extractSlackNativeApprovalId, + waitForSlackStoredMessage, + runSlackTableInvalidBlocksFallbackScenario, +} from "./slack-live.observations.js"; +import { SLACK_QA_STANDARD_SCENARIO_IDS, findScenario } from "./slack-live.scenarios.js"; + +export { listSlackQaScenarioCatalog } from "./slack-live.scenarios.js"; export async function runSlackQaLive(params: { alternateModel?: string; diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-fixtures.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-fixtures.ts new file mode 100644 index 000000000000..ab272fc853bc --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.scenario-fixtures.ts @@ -0,0 +1,162 @@ +// QA Lab Slack presentation and progress scenario fixtures. +import { randomUUID } from "node:crypto"; +import { + SLACK_QA_CHART_TITLE, + SLACK_QA_CHART_CATEGORIES, + SLACK_QA_CHART_SERIES_NAME, + SLACK_QA_CHART_VALUES, + SLACK_QA_CHART_X_LABEL, + SLACK_QA_CHART_Y_LABEL, + SLACK_QA_TABLE_CAPTION, + SLACK_QA_TABLE_HEADERS, + SLACK_QA_TABLE_ROWS, + type SlackQaMessageScenarioRun, +} from "./slack-live.contracts.js"; + +export function buildSlackChartMessageToolArgs(summaryText: string) { + return { + action: "send", + message: summaryText, + presentation: { + blocks: [ + { + type: "chart", + chartType: "line", + title: SLACK_QA_CHART_TITLE, + categories: [...SLACK_QA_CHART_CATEGORIES], + series: [{ name: SLACK_QA_CHART_SERIES_NAME, values: [...SLACK_QA_CHART_VALUES] }], + xLabel: SLACK_QA_CHART_X_LABEL, + yLabel: SLACK_QA_CHART_Y_LABEL, + }, + ], + }, + }; +} + +export function renderSlackChartAccessibleText(summaryText: string) { + return [ + summaryText, + "", + `${SLACK_QA_CHART_TITLE} (line chart)`, + `X axis: ${SLACK_QA_CHART_X_LABEL}`, + `Y axis: ${SLACK_QA_CHART_Y_LABEL}`, + `- ${SLACK_QA_CHART_SERIES_NAME}: ${SLACK_QA_CHART_CATEGORIES[0]}: ${SLACK_QA_CHART_VALUES[0]}; ${SLACK_QA_CHART_CATEGORIES[1]}: ${SLACK_QA_CHART_VALUES[1]}`, + ].join("\n"); +} + +export function buildSlackTableMessageToolArgs(summaryText: string) { + return { + action: "send", + message: summaryText, + presentation: { + blocks: [ + { + type: "table", + caption: SLACK_QA_TABLE_CAPTION, + headers: [...SLACK_QA_TABLE_HEADERS], + rows: SLACK_QA_TABLE_ROWS.map((row) => [...row]), + rowHeaderColumnIndex: 0, + }, + ], + }, + }; +} + +export function renderSlackTableAccessibleText(summaryText: string) { + return [ + summaryText, + "", + `${SLACK_QA_TABLE_CAPTION} (table)`, + SLACK_QA_TABLE_HEADERS.join("\t"), + ...SLACK_QA_TABLE_ROWS.map((row) => row.join("\t")), + ].join("\n"); +} + +type SlackProgressCommentaryExpectation = { + commentary: "absent" | "draft" | "standalone"; + toolProgress: "absent" | "draft" | "standalone"; +}; + +export function buildSlackProgressCommentaryRun( + sutUserId: string, + expectation: SlackProgressCommentaryExpectation, +): SlackQaMessageScenarioRun { + const suffix = randomUUID().slice(0, 8).toUpperCase(); + // Slack mrkdwn escapes underscores in progress drafts. Hyphenated markers + // stay byte-identical across draft edits and final-message reads. + const commentaryMarker = `SLACK-QA-COMMENTARY-${suffix}`; + const toolMarker = `SLACK-QA-TOOL-${suffix}`; + const finalMarker = `SLACK-QA-COMMENTARY-DONE-${suffix}`; + return { + expectReply: true, + input: [ + `<@${sutUserId}> This is a Slack progress protocol test. First, emit an assistant commentary message whose entire text is exactly ${commentaryMarker}.`, + "Do not call any tool until that commentary message is complete.", + `Then use the exec tool exactly once to run: grep '${toolMarker}' /dev/null || sleep 5.`, + `After the command finishes, reply with only this exact marker: ${finalMarker}`, + ].join(" "), + matchText: finalMarker, + settleObservedMs: 3_000, + verifyObserved: ({ finalMessage, messages }) => { + if (!finalMessage.ts) { + throw new Error("Slack progress commentary final message had no ts"); + } + if ((finalMessage.text ?? "").trim() !== finalMarker) { + throw new Error("expected the Slack final answer to contain only the final marker"); + } + const progressMessages = messages.filter((message) => !message.text.includes(finalMarker)); + const commentaryMessages = progressMessages.filter((message) => + message.text.includes(commentaryMarker), + ); + const commentaryTimestamps = new Set(commentaryMessages.map((message) => message.ts)); + if (expectation.commentary === "absent" && commentaryTimestamps.size !== 0) { + throw new Error("expected commentary to stay out of Slack progress messages"); + } + if (expectation.commentary !== "absent" && commentaryTimestamps.size !== 1) { + throw new Error( + `expected exactly one Slack message identity containing commentary; got ${commentaryTimestamps.size}`, + ); + } + const commentaryTs = [...commentaryTimestamps][0]; + if (expectation.commentary === "draft" && commentaryTs !== finalMessage.ts) { + throw new Error("expected commentary on the progress draft finalized as the answer"); + } + if (expectation.commentary === "standalone" && commentaryTs === finalMessage.ts) { + throw new Error("expected commentary only in the standalone verbose message"); + } + const toolTimestamps = new Set( + progressMessages + .filter((message) => message.text.includes(toolMarker)) + .map((message) => message.ts), + ); + if (expectation.toolProgress === "draft") { + if (toolTimestamps.size !== 1 || !toolTimestamps.has(finalMessage.ts)) { + throw new Error("expected tool progress on the progress draft finalized as the answer"); + } + } else if (expectation.toolProgress === "standalone") { + if (toolTimestamps.size === 0 || toolTimestamps.has(finalMessage.ts)) { + throw new Error("expected tool progress only in standalone verbose messages"); + } + } else if (toolTimestamps.size !== 0) { + throw new Error("expected tool progress to stay out of Slack progress messages"); + } + const finalTimestamps = new Set( + messages + .filter((message) => message.text.includes(finalMarker)) + .map((message) => message.ts), + ); + if (finalTimestamps.size !== 1 || !finalTimestamps.has(finalMessage.ts)) { + throw new Error( + "expected one final-marker Slack message identity matching the final answer", + ); + } + const commentaryDetails = + expectation.commentary === "draft" + ? "commentary on progress/final identity" + : expectation.commentary === "standalone" + ? "one standalone commentary identity" + : "commentary absent from Slack progress"; + return `verified ${commentaryDetails}; tool progress ${expectation.toolProgress}; final identity unique`; + }, + }; +} diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.scenarios.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.scenarios.ts new file mode 100644 index 000000000000..7b98fb7c7f6c --- /dev/null +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.scenarios.ts @@ -0,0 +1,407 @@ +// QA Lab Slack live scenario catalog. +import { randomUUID } from "node:crypto"; +import { + collectLiveTransportStandardScenarioCoverage, + selectLiveTransportScenarios, +} from "../shared/live-transport-scenarios.js"; +import { waitForSlackReaction } from "./slack-live.codex-approval.js"; +import { + SLACK_QA_REACTION_VERIFY_TIMEOUT_MS, + SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS, + SLACK_QA_LOG_TAIL_TIMEOUT_MS, + type SlackQaScenarioDefinition, +} from "./slack-live.contracts.js"; +import { + isExpectedSlackNativeChartMessage, + isExpectedSlackNativeTableMessage, + runSlackTableInvalidBlocksFallbackScenario, + waitForSlackStoredMessage, +} from "./slack-live.observations.js"; +import { + buildSlackChartMessageToolArgs, + renderSlackChartAccessibleText, + buildSlackTableMessageToolArgs, + renderSlackTableAccessibleText, + buildSlackProgressCommentaryRun, +} from "./slack-live.scenario-fixtures.js"; + +const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ + { + id: "slack-canary", + standardId: "canary", + title: "Slack canary echo", + timeoutMs: 45_000, + buildRun: (sutUserId) => { + const token = `SLACK_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + expectReply: true, + input: `<@${sutUserId}> reply with only this exact marker: ${token}`, + matchText: token, + }; + }, + }, + { + id: "slack-mention-gating", + standardId: "mention-gating", + title: "Slack unmentioned bot message does not trigger", + timeoutMs: 8_000, + buildRun: () => { + const token = `SLACK_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + expectReply: false, + input: `reply with only this exact marker: ${token}`, + matchText: token, + }; + }, + }, + { + id: "slack-allowlist-block", + standardId: "allowlist-block", + title: "Slack non-allowlisted sender does not trigger", + timeoutMs: 8_000, + configOverrides: { + allowFrom: ["U_OPENCLAW_QA_NEVER_ALLOWED"], + users: ["U_OPENCLAW_QA_NEVER_ALLOWED"], + }, + buildRun: (sutUserId) => { + const token = `SLACK_QA_BLOCK_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + expectReply: false, + input: `<@${sutUserId}> reply with only this exact marker: ${token}`, + matchText: token, + }; + }, + }, + { + id: "slack-channel-disabled-warning", + title: "Slack disabled channel warns and does not trigger", + timeoutMs: 8_000, + defaultEnabled: false, + configOverrides: { channelEnabled: false }, + buildRun: (sutUserId) => { + const marker = `SLACK_QA_DISABLED_${randomUUID().slice(0, 8).toUpperCase()}`; + let logCursor = 0; + return { + expectReply: false, + input: `<@${sutUserId}> reply with only this exact marker: ${marker}`, + matchText: marker, + preserveGatewayDebug: true, + beforeRun: async ({ gateway }) => { + const gatewayLogTail = (await gateway.call( + "logs.tail", + { limit: 1, maxBytes: 32_000 }, + { timeoutMs: SLACK_QA_LOG_TAIL_TIMEOUT_MS }, + )) as { cursor?: unknown }; + logCursor = typeof gatewayLogTail.cursor === "number" ? gatewayLogTail.cursor : 0; + }, + afterNoReply: async ({ gateway }) => { + const gatewayLogTail = (await gateway.call( + "logs.tail", + { cursor: logCursor, limit: 200, maxBytes: 256_000 }, + { timeoutMs: SLACK_QA_LOG_TAIL_TIMEOUT_MS }, + )) as { lines?: unknown }; + const gatewayLogLines = Array.isArray(gatewayLogTail.lines) + ? gatewayLogTail.lines.filter((line): line is string => typeof line === "string") + : []; + const expectedFields = [ + "Slack channel denied by configuration", + "channel_not_allowed", + "channel_disabled", + ]; + if ( + !gatewayLogLines.some((line) => expectedFields.every((field) => line.includes(field))) + ) { + throw new Error("disabled Slack channel did not emit the structured warning"); + } + return "structured disabled-channel warning observed"; + }, + }; + }, + }, + { + id: "slack-top-level-reply-shape", + standardId: "top-level-reply-shape", + title: "Slack top-level reply stays top-level", + timeoutMs: 45_000, + configOverrides: { replyToMode: "off" }, + buildRun: (sutUserId) => { + const token = `SLACK_QA_TOPLEVEL_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + expectReply: true, + input: `<@${sutUserId}> reply with only this exact marker: ${token}`, + matchText: token, + verify: (message) => { + if (message.thread_ts) { + throw new Error( + `expected top-level Slack reply without thread_ts; got ${message.thread_ts}`, + ); + } + }, + }; + }, + }, + { + id: "slack-progress-commentary-true", + title: "Slack progress commentary true is independent from tool progress", + defaultEnabled: false, + timeoutMs: 90_000, + configOverrides: { + progress: { commentary: true, toolProgress: false }, + }, + buildRun: (sutUserId) => + buildSlackProgressCommentaryRun(sutUserId, { + commentary: "draft", + toolProgress: "absent", + }), + }, + { + id: "slack-progress-commentary-false", + title: "Slack progress commentary false stays out of the progress draft", + defaultEnabled: false, + timeoutMs: 90_000, + configOverrides: { + progress: { commentary: false, toolProgress: false }, + }, + buildRun: (sutUserId) => + buildSlackProgressCommentaryRun(sutUserId, { + commentary: "absent", + toolProgress: "absent", + }), + }, + { + id: "slack-progress-commentary-omitted", + title: "Slack omitted progress commentary preserves the tool-progress default", + defaultEnabled: false, + timeoutMs: 90_000, + configOverrides: { + progress: { toolProgress: true }, + }, + buildRun: (sutUserId) => + buildSlackProgressCommentaryRun(sutUserId, { + commentary: "draft", + toolProgress: "draft", + }), + }, + { + id: "slack-progress-commentary-verbose-dedupe", + title: "Slack explicit commentary yields to durable verbose progress", + defaultEnabled: false, + timeoutMs: 90_000, + configOverrides: { + progress: { commentary: true, toolProgress: false, verboseDefault: "on" }, + }, + buildRun: (sutUserId) => + buildSlackProgressCommentaryRun(sutUserId, { + commentary: "standalone", + toolProgress: "standalone", + }), + }, + { + id: "slack-chart-presentation-native", + title: "Slack portable chart renders as a native data visualization", + timeoutMs: 90_000, + configOverrides: { messageTool: true }, + buildRun: (sutUserId) => { + const suffix = randomUUID().slice(0, 8).toUpperCase(); + const summaryText = `SLACK_QA_CHART_SUMMARY_${suffix}`; + const finalMarker = `SLACK_QA_CHART_DONE_${suffix}`; + const messageToolArgs = buildSlackChartMessageToolArgs(summaryText); + return { + expectReply: true, + input: [ + `<@${sutUserId}> Slack native chart QA check ${summaryText}.`, + `Call the message tool exactly once with these exact arguments: ${JSON.stringify(messageToolArgs)}.`, + `After the chart send succeeds, reply with only this exact marker: ${finalMarker}`, + ].join(" "), + matchText: finalMarker, + afterReply: async (_message, context) => { + await waitForSlackStoredMessage({ + channelId: context.channelId, + client: context.sutReadClient, + description: "message with native chart", + matchesMessage: (message) => + isExpectedSlackNativeChartMessage( + message, + renderSlackChartAccessibleText(summaryText), + ), + oldestTs: context.sentTs, + sutIdentity: context.sutIdentity, + timeoutMs: SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS, + }); + return "verified native data_visualization block and deterministic accessible text"; + }, + }; + }, + }, + { + id: "slack-table-presentation-native", + title: "Slack portable table renders as a native data table", + timeoutMs: 90_000, + configOverrides: { messageTool: true }, + buildRun: (sutUserId) => { + const suffix = randomUUID().slice(0, 8).toUpperCase(); + const summaryText = `SLACK_QA_TABLE_SUMMARY_${suffix}`; + const finalMarker = `SLACK_QA_TABLE_DONE_${suffix}`; + const messageToolArgs = buildSlackTableMessageToolArgs(summaryText); + return { + expectReply: true, + input: [ + `<@${sutUserId}> Slack native table QA check ${summaryText}.`, + `Call the message tool exactly once with these exact arguments: ${JSON.stringify(messageToolArgs)}.`, + `After the table send succeeds, reply with only this exact marker: ${finalMarker}`, + ].join(" "), + matchText: finalMarker, + afterReply: async (_message, context) => { + await waitForSlackStoredMessage({ + channelId: context.channelId, + client: context.sutReadClient, + description: "message with native table", + matchesMessage: (message) => + isExpectedSlackNativeTableMessage( + message, + renderSlackTableAccessibleText(summaryText), + ), + oldestTs: context.sentTs, + sutIdentity: context.sutIdentity, + timeoutMs: SLACK_QA_NATIVE_DATA_VERIFY_TIMEOUT_MS, + }); + return "verified native data_table block and deterministic accessible text"; + }, + }; + }, + }, + { + id: "slack-table-invalid-blocks-fallback", + title: "Slack rejects an over-limit native table and stores its complete fallback", + defaultEnabled: false, + timeoutMs: 45_000, + buildRun: () => ({ + kind: "direct-transport", + execute: runSlackTableInvalidBlocksFallbackScenario, + }), + }, + { + id: "slack-reaction-glyph-native", + title: "Slack message tool normalizes an emoji glyph reaction", + timeoutMs: 90_000, + configOverrides: { messageTool: true }, + buildRun: (sutUserId) => { + const token = `SLACK_QA_REACTION_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + expectReply: true, + input: [ + `<@${sutUserId}> use the message tool exactly once to react to this message.`, + 'Set action to "react", channel to "slack", and emoji to exactly "āœ…".', + "Do not substitute a shortcode.", + `After the reaction succeeds, reply with only this exact marker: ${token}`, + ].join(" "), + matchText: token, + afterReply: async (_message, context) => { + await waitForSlackReaction({ + channelId: context.channelId, + client: context.sutReadClient, + expectedReactionName: "white_check_mark", + messageId: context.sentTs, + sutUserId: context.sutIdentity.userId, + timeoutMs: SLACK_QA_REACTION_VERIFY_TIMEOUT_MS, + }); + return "verified SUT white_check_mark reaction from exact glyph instruction"; + }, + }; + }, + }, + { + id: "slack-approval-exec-native", + title: "Slack native exec approval prompt resolves", + timeoutMs: 60_000, + configOverrides: { + approvals: { + exec: true, + target: "channel", + }, + }, + buildRun: () => ({ + approvalKind: "exec", + decision: "allow-once", + kind: "approval", + token: `SLACK_QA_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), + }, + { + id: "slack-approval-plugin-native", + title: "Slack native plugin approval prompt resolves with exec approvals enabled", + timeoutMs: 60_000, + configOverrides: { + approvals: { + exec: true, + plugin: true, + target: "channel", + }, + }, + buildRun: () => ({ + approvalKind: "plugin", + decision: "allow-once", + kind: "approval", + token: `SLACK_QA_PLUGIN_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), + }, + { + id: "slack-codex-approval-exec-native", + title: "Slack native Codex command approval prompt resolves", + timeoutMs: 180_000, + configOverrides: { + approvals: { + exec: true, + plugin: true, + target: "channel", + }, + codexApproval: true, + }, + forcedRuntime: "codex", + buildRun: () => ({ + approvalKind: "plugin", + appServerMethod: "item/commandExecution/requestApproval", + decision: "allow-once", + kind: "codex-approval", + token: `SLACK_QA_CODEX_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), + }, + { + id: "slack-codex-approval-plugin-native", + title: "Slack native Codex file approval prompt resolves", + timeoutMs: 180_000, + configOverrides: { + approvals: { + exec: true, + plugin: true, + target: "channel", + }, + codexApproval: true, + }, + forcedRuntime: "codex", + buildRun: () => ({ + approvalKind: "plugin", + appServerMethod: "item/fileChange/requestApproval", + decision: "allow-once", + kind: "codex-approval", + token: `SLACK_QA_CODEX_FILE_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), + }, +]; + +export const SLACK_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({ + scenarios: SLACK_QA_SCENARIOS, +}); + +export function listSlackQaScenarioCatalog() { + return SLACK_QA_SCENARIOS.map((scenario) => ({ id: scenario.id })); +} + +export function findScenario(ids?: string[]) { + const selected = selectLiveTransportScenarios({ + ids, + laneLabel: "Slack", + scenarios: SLACK_QA_SCENARIOS, + }); + return ids?.length ? selected : selected.filter((scenario) => scenario.defaultEnabled !== false); +} diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-messages.ts b/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-messages.ts new file mode 100644 index 000000000000..81cc5d23895e --- /dev/null +++ b/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-messages.ts @@ -0,0 +1,98 @@ +import { + convertAnthropicMessagesToResponsesInput, + type ExtractedAssistantOutput, + extractFinalAssistantOutputFromEvents, + buildAnthropicMessageResponse, + buildAnthropicThinkingErrorResponse, + buildAnthropicThinkingErrorStreamEvents, + buildAnthropicMessageStreamEvents, +} from "./mock-anthropic-wire.js"; +// QA Lab Anthropic Messages request dispatcher. +import { + type ResponsesInputItem, + type StreamEvent, + type AnthropicMessagesRequest, + QA_ANTHROPIC_THINKING_ERROR_RECOVERY_PROMPT_RE, + type MockScenarioState, + type AnthropicStreamEvent, +} from "./mock-openai-contracts.js"; +import { buildAssistantEvents } from "./mock-openai-events.js"; +import { extractToolOutput, extractAllRequestTexts } from "./mock-openai-input.js"; +import { buildToolCallEventsWithArgs } from "./mock-openai-tooling.js"; + +export async function buildMessagesPayload( + body: AnthropicMessagesRequest, + scenarioState: MockScenarioState, + dispatchResponses: ( + body: Record, + scenarioState: MockScenarioState, + ) => Promise, +): Promise<{ + events: StreamEvent[]; + input: ResponsesInputItem[]; + extracted: ExtractedAssistantOutput; + responseBody: Record; + streamEvents: AnthropicStreamEvent[]; + model: string; +}> { + const messages = Array.isArray(body.messages) ? body.messages : []; + const input = convertAnthropicMessagesToResponsesInput({ + system: body.system, + messages, + }); + // Treat empty-string model the same as absent. A bare typeof check lets + // `""` leak through to `responseBody.model` and `lastRequest.model`, + // which then confuses parity consumers that assume the mock always + // echoes the real provider label. Normalize once and reuse everywhere. + const normalizedModel = + typeof body.model === "string" && body.model.trim() !== "" ? body.model : "claude-opus-4-8"; + // Dispatch through the same scenario logic the /v1/responses route uses. + // Preserve declared tools so route-specific adapters mirror what the + // real provider request made available to the model. + const dispatchBody: Record = { + input, + model: normalizedModel, + stream: false, + ...(Array.isArray(body.tools) ? { tools: body.tools } : {}), + }; + const allInputText = extractAllRequestTexts(input, dispatchBody); + if (QA_ANTHROPIC_THINKING_ERROR_RECOVERY_PROMPT_RE.test(allInputText)) { + const toolOutput = extractToolOutput(input); + const shouldEmitThinkingError = + toolOutput.length > 0 && scenarioState.anthropicThinkingErrorPhase === 0; + const events = + toolOutput.length === 0 + ? buildToolCallEventsWithArgs("read", { path: "QA_KICKOFF_TASK.md" }) + : shouldEmitThinkingError + ? (() => { + scenarioState.anthropicThinkingErrorPhase = 1; + return buildAssistantEvents(""); + })() + : buildAssistantEvents("ANTHROPIC-THINKING-ERROR-RECOVERED-OK"); + const extracted = extractFinalAssistantOutputFromEvents(events); + const responseBody = shouldEmitThinkingError + ? buildAnthropicThinkingErrorResponse({ model: normalizedModel }) + : buildAnthropicMessageResponse({ + model: normalizedModel, + extracted, + }); + const streamEvents = shouldEmitThinkingError + ? buildAnthropicThinkingErrorStreamEvents({ model: normalizedModel }) + : buildAnthropicMessageStreamEvents({ + model: normalizedModel, + extracted, + }); + return { events, input, extracted, responseBody, streamEvents, model: normalizedModel }; + } + const events = await dispatchResponses(dispatchBody, scenarioState); + const extracted = extractFinalAssistantOutputFromEvents(events); + const responseBody = buildAnthropicMessageResponse({ + model: normalizedModel, + extracted, + }); + const streamEvents = buildAnthropicMessageStreamEvents({ + model: normalizedModel, + extracted, + }); + return { events, input, extracted, responseBody, streamEvents, model: normalizedModel }; +} diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-wire.ts b/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-wire.ts new file mode 100644 index 000000000000..45bdd1099d24 --- /dev/null +++ b/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-wire.ts @@ -0,0 +1,422 @@ +// QA Lab Anthropic Messages wire conversion and response events. +import { + type ResponsesInputItem, + type StreamEvent, + type AnthropicMessageContentBlock, + type AnthropicMessage, + type AnthropicMessagesRequest, + type AnthropicStreamEvent, + countApproxTokens, +} from "./mock-openai-contracts.js"; + +// Anthropic Messages conversion preserves role and tool ordering while reusing +// the shared Responses scenario dispatcher for provider parity. + +function normalizeAnthropicSystemToString( + system: AnthropicMessagesRequest["system"], +): string | undefined { + if (typeof system === "string") { + return system.trim() || undefined; + } + if (Array.isArray(system)) { + const joined = system + .map((block) => (block?.type === "text" ? block.text : "")) + .filter(Boolean) + .join("\n") + .trim(); + return joined || undefined; + } + return undefined; +} + +function stringifyToolResultContent( + content: Extract["content"], +): string { + if (typeof content === "string") { + return content; + } + if (Array.isArray(content)) { + return content + .map((block) => (block?.type === "text" ? block.text : "")) + .filter(Boolean) + .join("\n"); + } + return ""; +} + +export function convertAnthropicMessagesToResponsesInput(params: { + system?: AnthropicMessagesRequest["system"]; + messages: AnthropicMessage[]; +}): ResponsesInputItem[] { + const items: ResponsesInputItem[] = []; + const systemText = normalizeAnthropicSystemToString(params.system); + if (systemText) { + items.push({ + role: "system", + content: [{ type: "input_text", text: systemText }], + }); + } + for (const message of params.messages) { + const content = message.content; + if (typeof content === "string") { + items.push({ + role: message.role, + content: [ + message.role === "assistant" + ? { type: "output_text", text: content } + : { type: "input_text", text: content }, + ], + }); + continue; + } + if (!Array.isArray(content)) { + continue; + } + // Buffer each block type so we can push in OpenAI-Responses order instead + // of the order they appear in the Anthropic content array. The parent + // role message must precede any function_call_output items from the same + // turn, otherwise extractToolOutput() (which scans for + // function_call_output AFTER the last user-role index) will not see the + // output and the downstream scenario dispatcher will behave as if no + // tool output was returned. Similarly, assistant tool_use blocks become + // function_call items that must follow the assistant text message they + // narrate. + const textPieces: Array<{ type: "input_text" | "output_text"; text: string }> = []; + const imagePieces: Array<{ type: "input_image"; image_url: string }> = []; + const toolResultItems: ResponsesInputItem[] = []; + const toolUseItems: ResponsesInputItem[] = []; + for (const block of content) { + if (!block || typeof block !== "object") { + continue; + } + if (block.type === "text") { + textPieces.push({ + type: message.role === "assistant" ? "output_text" : "input_text", + text: block.text ?? "", + }); + continue; + } + if (block.type === "image") { + // Mock only needs to count image inputs; a placeholder URL is fine. + imagePieces.push({ type: "input_image", image_url: "anthropic-mock:image" }); + continue; + } + if (block.type === "tool_result") { + const output = stringifyToolResultContent(block.content); + if (output.trim()) { + toolResultItems.push({ + type: "function_call_output", + call_id: block.tool_use_id, + output, + ...(block.is_error === true ? { is_error: true } : {}), + }); + } + continue; + } + if (block.type === "tool_use") { + // Mirror OpenAI's function_call output_item shape so downstream + // prompt extraction still sees "the assistant just emitted a tool + // call". The scenario dispatcher looks for tool_output on the next + // user turn, not the assistant's prior tool_use, so a minimal + // placeholder is enough. + toolUseItems.push({ + type: "function_call", + name: block.name, + arguments: JSON.stringify(block.input ?? {}), + call_id: block.id, + }); + continue; + } + } + if (textPieces.length > 0 || imagePieces.length > 0) { + const combinedContent: Array> = [...textPieces, ...imagePieces]; + items.push({ role: message.role, content: combinedContent }); + } + // Emit tool_use (assistant prior calls) and tool_result (user-side + // returns) AFTER the parent role message so extractLastUserText and + // extractToolOutput walk the array in the order they expect. For a + // tool_result-only user turn with no text/image blocks, the parent + // message is intentionally omitted ā€” the function_call_output itself + // represents the user's "return the tool output" turn. + for (const toolUse of toolUseItems) { + items.push(toolUse); + } + for (const toolResult of toolResultItems) { + items.push(toolResult); + } + } + return items; +} + +export type ExtractedAssistantOutput = { + text: string; + toolCalls: Array<{ id: string; name: string; input: Record }>; +}; + +export function extractFinalAssistantOutputFromEvents( + events: StreamEvent[], +): ExtractedAssistantOutput { + const toolCalls: ExtractedAssistantOutput["toolCalls"] = []; + let text = ""; + for (const event of events) { + if (event.type !== "response.output_item.done") { + continue; + } + const item = event.item as { + type?: unknown; + name?: unknown; + call_id?: unknown; + id?: unknown; + arguments?: unknown; + content?: unknown; + }; + if (item.type === "function_call" && typeof item.name === "string") { + let input: Record = {}; + if (typeof item.arguments === "string" && item.arguments.trim()) { + try { + const parsed = JSON.parse(item.arguments) as unknown; + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + input = parsed as Record; + } + } catch { + // keep empty input on malformed args ā€” mock dispatcher owns arg shape + } + } + toolCalls.push({ + id: typeof item.call_id === "string" ? item.call_id : `toolu_mock_${toolCalls.length + 1}`, + name: item.name, + input, + }); + continue; + } + if (item.type === "message" && Array.isArray(item.content)) { + for (const piece of item.content as Array<{ type?: unknown; text?: unknown }>) { + if (piece?.type === "output_text" && typeof piece.text === "string") { + text = piece.text; + } + } + } + } + return { text, toolCalls }; +} + +export function buildAnthropicMessageResponse(params: { + model: string; + extracted: ExtractedAssistantOutput; +}): Record { + const content: Array> = []; + if (params.extracted.text) { + content.push({ type: "text", text: params.extracted.text }); + } + for (const call of params.extracted.toolCalls) { + content.push({ + type: "tool_use", + id: call.id, + name: call.name, + input: call.input, + }); + } + if (content.length === 0) { + content.push({ type: "text", text: "" }); + } + const stopReason = params.extracted.toolCalls.length > 0 ? "tool_use" : "end_turn"; + const approxInputTokens = 64; + const approxOutputTokens = Math.max( + 16, + countApproxTokens(params.extracted.text) + params.extracted.toolCalls.length * 16, + ); + return { + id: `msg_mock_${Math.floor(Math.random() * 1_000_000).toString(16)}`, + type: "message", + role: "assistant", + model: params.model || "claude-opus-4-8", + content, + stop_reason: stopReason, + stop_sequence: null, + usage: { + input_tokens: approxInputTokens, + output_tokens: approxOutputTokens, + }, + }; +} + +const QA_ANTHROPIC_THINKING_ERROR_TEXT = + "QA replay-safe read completed, but the provider stream failed after signed thinking."; +const QA_ANTHROPIC_THINKING_ERROR_SIGNATURE = "qa_signed_thinking_block_91953"; +const QA_ANTHROPIC_THINKING_ERROR_MESSAGE = "QA injected provider stream failure"; + +export function buildAnthropicThinkingErrorResponse(params: { + model: string; +}): Record { + return { + type: "error", + error: { + type: "api_error", + message: QA_ANTHROPIC_THINKING_ERROR_MESSAGE, + }, + model: params.model || "claude-opus-4-8", + }; +} + +export function buildAnthropicThinkingErrorStreamEvents(params: { + model: string; +}): AnthropicStreamEvent[] { + const messageId = `msg_mock_${Math.floor(Math.random() * 1_000_000).toString(16)}`; + return [ + { + type: "message_start", + message: { + id: messageId, + type: "message", + role: "assistant", + model: params.model || "claude-opus-4-8", + content: [], + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: 64, + output_tokens: 0, + }, + }, + }, + { + type: "content_block_start", + index: 0, + content_block: { + type: "thinking", + thinking: "", + signature: "", + }, + }, + { + type: "content_block_delta", + index: 0, + delta: { + type: "thinking_delta", + thinking: QA_ANTHROPIC_THINKING_ERROR_TEXT, + }, + }, + { + type: "content_block_delta", + index: 0, + delta: { + type: "signature_delta", + signature: QA_ANTHROPIC_THINKING_ERROR_SIGNATURE, + }, + }, + { + type: "content_block_stop", + index: 0, + }, + { + type: "message_delta", + delta: {}, + usage: { + input_tokens: 64, + output_tokens: 1120, + }, + }, + { + type: "error", + error: { + type: "api_error", + message: QA_ANTHROPIC_THINKING_ERROR_MESSAGE, + }, + }, + ]; +} + +export function buildAnthropicMessageStreamEvents(params: { + model: string; + extracted: ExtractedAssistantOutput; +}): AnthropicStreamEvent[] { + const approxInputTokens = 64; + const approxOutputTokens = Math.max( + 16, + countApproxTokens(params.extracted.text) + params.extracted.toolCalls.length * 16, + ); + const messageId = `msg_mock_${Math.floor(Math.random() * 1_000_000).toString(16)}`; + const events: AnthropicStreamEvent[] = [ + { + type: "message_start", + message: { + id: messageId, + type: "message", + role: "assistant", + model: params.model || "claude-opus-4-8", + content: [], + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: approxInputTokens, + output_tokens: 0, + }, + }, + }, + ]; + let index = 0; + if (params.extracted.text || params.extracted.toolCalls.length === 0) { + events.push({ + type: "content_block_start", + index, + content_block: { + type: "text", + text: "", + }, + }); + if (params.extracted.text) { + events.push({ + type: "content_block_delta", + index, + delta: { + type: "text_delta", + text: params.extracted.text, + }, + }); + } + events.push({ + type: "content_block_stop", + index, + }); + index += 1; + } + for (const call of params.extracted.toolCalls) { + events.push({ + type: "content_block_start", + index, + content_block: { + type: "tool_use", + id: call.id, + name: call.name, + input: {}, + }, + }); + events.push({ + type: "content_block_delta", + index, + delta: { + type: "input_json_delta", + partial_json: JSON.stringify(call.input ?? {}), + }, + }); + events.push({ + type: "content_block_stop", + index, + }); + index += 1; + } + events.push({ + type: "message_delta", + delta: { + stop_reason: params.extracted.toolCalls.length > 0 ? "tool_use" : "end_turn", + }, + usage: { + input_tokens: approxInputTokens, + output_tokens: approxOutputTokens, + }, + }); + events.push({ + type: "message_stop", + }); + return events; +} diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-assistant-text.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-assistant-text.ts new file mode 100644 index 000000000000..6d0c978652a3 --- /dev/null +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-assistant-text.ts @@ -0,0 +1,360 @@ +// QA Lab mock provider assistant text fixtures. +import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; +import { + type ResponsesInputItem, + QA_STRANDED_FINAL_RECOVERY_PROMPT_RE, + QA_STRANDED_FINAL_RETRY_PROMPT_RE, + QA_SUBAGENT_DIRECT_FALLBACK_WORKER_RE, + buildStrandedFinalRecoveryText, + buildStrandedFinalRetryFailureText, + isStrandedFinalRetryFailureRequest, + QA_SUBAGENT_DIRECT_FALLBACK_MARKER, + QA_IMAGE_GENERATION_PROMPT_RE, + QA_SKILL_WORKSHOP_GIF_PROMPT_RE, + QA_TOOL_SEARCH_PROMPT_RE, + QA_TOOL_SEARCH_FAILURE_PROMPT_RE, + type MockScenarioState, +} from "./mock-openai-contracts.js"; +import { + extractExactReplyDirective, + extractFinishExactlyDirective, + extractExactMarkerDirective, + extractWhatsAppLocationMarkerDirective, + extractWhatsAppContactMarkerDirective, + extractWhatsAppStickerMarkerDirective, + shouldUseWhatsAppLocationMarker, + shouldUseWhatsAppContactMarker, + shouldUseWhatsAppStickerMarker, + extractToolErrorForNamedCall, + isHeartbeatPrompt, + readFirstMediaPath, +} from "./mock-openai-directives.js"; +import { + extractLastUserText, + extractToolOutput, + extractLatestToolOutput, + extractAllUserTexts, + extractAllRequestTexts, + extractLatestImageUserTurn, + parseToolOutputJson, +} from "./mock-openai-input.js"; +import { + extractRememberedFact, + extractOrbitCode, + extractActiveMemorySummary, + extractToolSearchTarget, + extractSnackPreference, +} from "./mock-openai-tooling.js"; +export function buildAssistantText( + input: ResponsesInputItem[], + body: Record, + scenarioState: MockScenarioState, +) { + const prompt = extractLastUserText(input); + const toolOutput = extractToolOutput(input); + const scenarioToolOutput = + toolOutput || + (/thread memory check|session memory ranking check|memory tools check|repo contract followthrough check/i.test( + extractAllRequestTexts(input, body), + ) + ? extractLatestToolOutput(input) + : ""); + const toolJson = parseToolOutputJson(scenarioToolOutput); + const userTexts = extractAllUserTexts(input); + const allInputText = extractAllRequestTexts(input, body); + const rememberedFact = extractRememberedFact(userTexts); + const model = typeof body.model === "string" ? body.model : ""; + const memorySnippet = + typeof toolJson?.text === "string" + ? toolJson.text + : Array.isArray(toolJson?.results) + ? JSON.stringify(toolJson.results) + : scenarioToolOutput; + const orbitCode = extractOrbitCode(memorySnippet) ?? extractOrbitCode(allInputText); + const mediaPath = + typeof toolJson?.details === "object" && + toolJson.details !== null && + !Array.isArray(toolJson.details) + ? readFirstMediaPath((toolJson.details as { media?: unknown }).media) + : ""; + const promptExactReplyDirective = extractExactReplyDirective(prompt); + const promptExactMarkerDirective = extractExactMarkerDirective(prompt); + const exactReplyDirective = promptExactReplyDirective ?? extractExactReplyDirective(allInputText); + const exactMarkerDirective = + promptExactMarkerDirective ?? extractExactMarkerDirective(allInputText); + const whatsAppLocationMarker = shouldUseWhatsAppLocationMarker(prompt) + ? extractWhatsAppLocationMarkerDirective(allInputText) + : ""; + const whatsAppContactMarker = shouldUseWhatsAppContactMarker(prompt) + ? extractWhatsAppContactMarkerDirective(allInputText) + : ""; + const whatsAppStickerMarker = shouldUseWhatsAppStickerMarker(prompt) + ? extractWhatsAppStickerMarkerDirective(allInputText) + : ""; + const finishExactlyDirective = + extractFinishExactlyDirective(prompt) ?? extractFinishExactlyDirective(allInputText); + const latestImageUserTurn = extractLatestImageUserTurn(input); + const activeMemorySummary = extractActiveMemorySummary(allInputText); + const snackPreference = extractSnackPreference(activeMemorySummary ?? memorySnippet); + const sessionsSpawnError = extractToolErrorForNamedCall({ + input, + name: "sessions_spawn", + toolJson, + }); + + if (/what was the qa canary code/i.test(prompt) && rememberedFact) { + return `Protocol note: the QA canary code was ${rememberedFact}.`; + } + if (sessionsSpawnError) { + return `Protocol note: sessions_spawn failed: ${sessionsSpawnError}`; + } + if (/remember this fact/i.test(prompt) && exactReplyDirective) { + return exactReplyDirective; + } + if (/remember this fact/i.test(prompt) && rememberedFact) { + return `Protocol note: acknowledged. I will remember ${rememberedFact}.`; + } + if (/memory unavailable check/i.test(prompt)) { + return "Protocol note: I checked the available runtime context but could not confirm the hidden memory-only fact, so I will not guess."; + } + if (isHeartbeatPrompt(prompt)) { + return "HEARTBEAT_OK"; + } + if ( + /roundtrip image inspection check/i.test(latestImageUserTurn.text) && + latestImageUserTurn.imageInputCount > 0 + ) { + return "Protocol note: the generated attachment shows the same QA lighthouse scene from the previous step."; + } + if ( + /image understanding check/i.test(latestImageUserTurn.text) && + latestImageUserTurn.imageInputCount > 0 + ) { + return "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom."; + } + if (whatsAppLocationMarker) { + return whatsAppLocationMarker; + } + if (whatsAppContactMarker) { + return whatsAppContactMarker; + } + if (whatsAppStickerMarker) { + return whatsAppStickerMarker; + } + if (/\bmarker\b/i.test(prompt) && promptExactMarkerDirective) { + return promptExactMarkerDirective; + } + if (/\bmarker\b/i.test(prompt) && promptExactReplyDirective) { + return promptExactReplyDirective; + } + if (/\bmarker\b/i.test(allInputText) && promptExactReplyDirective) { + return promptExactReplyDirective; + } + if (/\bmarker\b/i.test(allInputText) && exactMarkerDirective) { + return exactMarkerDirective; + } + if (/\bmarker\b/i.test(allInputText) && exactReplyDirective) { + return exactReplyDirective; + } + if (promptExactReplyDirective) { + return promptExactReplyDirective; + } + if (/visible skill marker/i.test(prompt)) { + return "VISIBLE-SKILL-OK"; + } + if (/hot install marker/i.test(prompt)) { + return "HOT-INSTALL-OK"; + } + if (/memory tools check/i.test(prompt) && orbitCode) { + return `Protocol note: I checked memory and the project codename is ${orbitCode}.`; + } + if (/silent snack recall check/i.test(prompt) && snackPreference) { + return `Protocol note: you usually want ${snackPreference} for QA movie night.`; + } + if (/silent snack recall check/i.test(prompt)) { + return "Protocol note: I do not have enough context to say what you usually want for QA movie night."; + } + if (/qa private final reply warning check/i.test(prompt)) { + return [ + "QA-STRANDED-85714 confirms this is a substantive private final reply that intentionally stays outside the message tool path for the warning check.", + "The response is long enough to exercise message_tool_only private-final detection while remaining private to the agent transcript.", + ].join(" "); + } + if (isStrandedFinalRetryFailureRequest(allInputText)) { + return buildStrandedFinalRetryFailureText(); + } + if (QA_STRANDED_FINAL_RECOVERY_PROMPT_RE.test(allInputText)) { + return QA_STRANDED_FINAL_RETRY_PROMPT_RE.test(allInputText) + ? "QA-STRANDED-85714" + : buildStrandedFinalRecoveryText(); + } + if (/tool continuity check/i.test(prompt) && toolOutput) { + return `Protocol note: model switch handoff confirmed on ${model || "the requested model"}. QA mission from QA_KICKOFF_TASK.md still applies: understand this OpenClaw repo from source + docs before acting.`; + } + if (toolOutput && promptExactReplyDirective) { + return promptExactReplyDirective; + } + if ((toolOutput || allInputText) && /repo contract followthrough check/i.test(allInputText)) { + const repoEvidenceText = [scenarioToolOutput, allInputText].filter(Boolean).join("\n"); + if ( + /successfully (?:wrote|created|updated|replaced)/i.test(repoEvidenceText) || + /status:\s*complete/i.test(repoEvidenceText) + ) { + return [ + "Read: AGENT.md, SOUL.md, FOLLOWTHROUGH_INPUT.md", + "Wrote: repo-contract-summary.txt", + "Status: complete", + ].join("\n"); + } + return [ + "Read: AGENT.md, SOUL.md, FOLLOWTHROUGH_INPUT.md", + "Wrote: repo-contract-summary.txt", + "Status: blocked", + ].join("\n"); + } + if (toolOutput && /personal task followthrough check/i.test(allInputText)) { + const taskEvidenceText = scenarioToolOutput; + if (/successfully (?:wrote|created|updated|replaced)/i.test(taskEvidenceText)) { + return [ + "Pending: maintainer feedback before publishing", + "Blocked: publishing needs explicit user approval", + "Done: local evidence captured in personal-task-status.txt", + ].join("\n"); + } + return [ + "Pending: maintainer feedback before publishing", + "Blocked: publishing needs explicit user approval", + "Done: blocked until personal-task-status.txt exists", + ].join("\n"); + } + if (/session memory ranking check/i.test(prompt) && orbitCode) { + return `Protocol note: I checked memory and the current Project Nebula codename is ${orbitCode}.`; + } + if (/thread memory check/i.test(allInputText) && orbitCode) { + return `Protocol note: I checked memory in-thread and the hidden thread codename is ${orbitCode}.`; + } + if (/switch(?:ing)? models?/i.test(prompt)) { + return `Protocol note: model switch acknowledged. Continuing on ${model || "the requested model"}.`; + } + if (QA_IMAGE_GENERATION_PROMPT_RE.test(allInputText) && mediaPath) { + return `Protocol note: generated the QA lighthouse image successfully. Attachment: ${mediaPath}`; + } + if (QA_SKILL_WORKSHOP_GIF_PROMPT_RE.test(prompt) && toolOutput) { + return [ + "Animated GIF QA checklist ready.", + "- Confirm true animation, not a static preview.", + "- Verify dimensions and product UI fit.", + "- Record attribution and license.", + "- Keep a local copy before using the asset.", + "- Re-open the copied file for final verification.", + ].join("\n"); + } + if ( + /interrupted by a gateway reload/i.test(prompt) && + /subagent recovery worker/i.test(allInputText) + ) { + return "RECOVERED-SUBAGENT-OK"; + } + if (/subagent recovery worker/i.test(prompt)) { + return "RECOVERED-SUBAGENT-OK"; + } + if (/fanout worker alpha/i.test(prompt)) { + return "ALPHA-OK"; + } + if (/fanout worker beta/i.test(prompt)) { + return "BETA-OK"; + } + if (QA_SUBAGENT_DIRECT_FALLBACK_WORKER_RE.test(prompt)) { + return QA_SUBAGENT_DIRECT_FALLBACK_MARKER; + } + if (/report the visible code/i.test(prompt) && /FORKED-CONTEXT-ALPHA/i.test(allInputText)) { + return "FORKED-CONTEXT-ALPHA"; + } + const fanoutCompleteReply = "subagent-1: ok\nsubagent-2: ok"; + if (scenarioState.subagentFanoutPhase === 2 && prompt) { + scenarioState.subagentFanoutPhase = 3; + return fanoutCompleteReply; + } + if ( + /forked subagent context qa check/i.test(prompt) && + /FORKED-CONTEXT-ALPHA/i.test(allInputText) + ) { + return [ + "Worked", + "- FORKED-CONTEXT-ALPHA", + "Evidence", + "- The forked child recovered the visible code from requester transcript context.", + "Blocked", + "- None.", + ].join("\n"); + } + if ( + toolOutput && + (/delegate (?:one |a )bounded qa task/i.test(allInputText) || + /subagent handoff/i.test(allInputText)) + ) { + const compact = toolOutput.replace(/\s+/g, " ").trim() || "no delegated output"; + return `Delegated task:\n- Inspect the QA workspace via a bounded subagent.\nResult:\n- ${compact}\nEvidence:\n- The child result was folded back into the main thread exactly once.`; + } + if (toolOutput && /worked, failed, blocked|worked\/failed\/blocked|follow-up/i.test(prompt)) { + return `Worked:\n- Read seeded QA material.\n- Expanded the report structure.\nFailed:\n- None observed in mock mode.\nBlocked:\n- No live provider evidence in this lane.\nFollow-up:\n- Re-run with a real model for qualitative coverage.`; + } + if (toolOutput && /lobster invaders/i.test(prompt)) { + if (toolOutput.includes("QA mission") || toolOutput.includes("Testing")) { + return ""; + } + return `Protocol note: Lobster Invaders built at lobster-invaders.html.`; + } + if ( + toolOutput && + (/compaction retry mutating tool check/i.test(allInputText) || + /compaction-retry-summary\.txt/i.test(toolOutput)) + ) { + if ( + toolOutput.includes("Replay safety: unsafe after write.") || + /compaction-retry-summary\.txt/i.test(toolOutput) || + /successfully (?:wrote|replaced)/i.test(toolOutput) || + /\bwrote\b.*\bcompaction-retry-summary\.txt\b/i.test(toolOutput) + ) { + return "Protocol note: replay unsafe after write."; + } + return ""; + } + if ( + toolOutput && + (QA_TOOL_SEARCH_PROMPT_RE.test(allInputText) || + QA_TOOL_SEARCH_FAILURE_PROMPT_RE.test(allInputText)) + ) { + const targetTool = extractToolSearchTarget(allInputText); + if (targetTool && toolOutput.includes(targetTool) && toolOutput.includes("FAKE_PLUGIN_OK")) { + return `FAKE_PLUGIN_OK ${targetTool}`; + } + } + if ( + toolOutput && + /(worked, failed, blocked|worked\/failed\/blocked|source and docs)/i.test(allInputText) + ) { + return [ + "Worked:", + "- Read all three seeded files: repo/qa/scenarios/index.yaml, repo/extensions/qa-lab/src/suite.ts, and repo/docs/help/testing.md.", + "- Extra QA scenario candidates: config restart capability flip and image generation roundtrip.", + "Failed:", + "- None observed in mock mode.", + "Blocked:", + "- No live provider evidence in this lane.", + "Follow-up:", + "- Re-run with a real model for qualitative coverage.", + ].join("\n"); + } + if (toolOutput) { + const snippet = truncateUtf16Safe(toolOutput.replace(/\s+/g, " ").trim(), 220); + return `Protocol note: I reviewed the requested material. Evidence snippet: ${snippet || "no content"}`; + } + if (finishExactlyDirective) { + return finishExactlyDirective; + } + if (prompt) { + return `Protocol note: acknowledged. Continue with the QA scenario plan and report worked, failed, and blocked items.`; + } + return "Protocol note: mock OpenAI server ready."; +} diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts new file mode 100644 index 000000000000..302abcc827b9 --- /dev/null +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts @@ -0,0 +1,432 @@ +// QA Lab mock provider contracts, wire helpers, and scenario constants. +import type { IncomingMessage, ServerResponse } from "node:http"; +import { setTimeout as sleep } from "node:timers/promises"; +import { readRequestBodyWithLimit } from "openclaw/plugin-sdk/webhook-ingress"; +import { writeJson } from "../shared/http-json.js"; + +export type ResponsesInputItem = Record; + +export type StreamEvent = + | { type: "response.output_item.added"; item: Record } + | { + type: "response.output_text.delta"; + item_id: string; + output_index: number; + content_index: number; + delta: string; + } + | { + type: "response.output_text.done"; + item_id: string; + output_index: number; + content_index: number; + text: string; + } + | { type: "response.function_call_arguments.delta"; delta: string } + | { type: "response.output_item.done"; item: Record } + | { + type: "response.completed"; + response: { + id: string; + status: "completed"; + output: Array>; + usage: { + input_tokens: number; + output_tokens: number; + total_tokens: number; + }; + }; + }; + +/** + * Provider variant tag for `body.model`. The mock previously ignored + * `body.model` for dispatch and only echoed it in the prose output, which + * made the parity gate tautological when run against the mock alone + * (both providers produced identical scenario plans by construction). + * Tagging requests with a normalized variant lets individual scenario + * branches opt into provider-specific behavior while the rest of the + * dispatcher stays shared, and lets `/debug/requests` consumers verify + * which provider lane a given request came from without re-parsing the + * raw model string. + * + * Policy: + * - `openai/*`, `gpt-*`, `o1-*`, anything starting with `gpt-` ā†’ `"openai"` + * - `anthropic/*`, `claude-*` ā†’ `"anthropic"` + * - Everything else (including empty strings) ā†’ `"unknown"` + * + * The `/v1/messages` route always feeds `body.model` straight through, + * so an Anthropic request with an `openai/gpt-5.6-luna` model string is still + * classified as `"openai"`. That matches the parity program's convention + * where the provider label is the source of truth, not the HTTP route. + */ +type MockOpenAiProviderVariant = "openai" | "anthropic" | "unknown"; + +export function resolveProviderVariant(model: string | undefined): MockOpenAiProviderVariant { + if (typeof model !== "string") { + return "unknown"; + } + const trimmed = model.trim().toLowerCase(); + if (trimmed.length === 0) { + return "unknown"; + } + // Prefer the explicit `provider/model` or `provider:model` prefix when + // the caller supplied one ā€” that's the most reliable signal. + const separatorMatch = /^([^/:]+)[/:]/.exec(trimmed); + const provider = separatorMatch?.[1] ?? trimmed; + if (provider === "openai") { + return "openai"; + } + if (provider === "anthropic" || provider === "claude-cli") { + return "anthropic"; + } + // Fall back to model-name prefix matching for bare model strings like + // `gpt-5.6-luna` or `claude-opus-4-8`. + if (/^(?:gpt-|o1-|openai-)/.test(trimmed)) { + return "openai"; + } + if (/^(?:claude-|anthropic-)/.test(trimmed)) { + return "anthropic"; + } + return "unknown"; +} + +export type MockOpenAiRequestSnapshot = { + cursor: number; + raw: string; + body: Record; + prompt: string; + allInputText: string; + instructions?: string; + toolOutput: string; + model: string; + providerVariant: MockOpenAiProviderVariant; + imageInputCount: number; + plannedToolCallId?: string; + plannedToolName?: string; + plannedToolArgs?: Record; + toolOutputCallId?: string; + toolOutputStructuredError?: true; +}; + +export type MockOpenAiRequestSnapshotInput = Omit; + +// Runtime-context delimiters are owned by src/agents/internal-runtime-context.ts. +// This mock mirrors the wire shape so delimiter drift fails through QA timeouts. +export const INTERNAL_RUNTIME_CONTEXT_BEGIN = "<<>>"; +export const INTERNAL_RUNTIME_CONTEXT_END = "<<>>"; + +// Anthropic /v1/messages request/response shapes the mock actually needs. +// This is a subset of the real Anthropic Messages API ā€” just enough so the +// QA suite can run its parity pack against a "baseline" Anthropic provider +// without needing real API keys. The scenarios drive their dispatch through +// the shared mock scenario logic (buildResponsesPayload), with `model` +// preserved so provider-aware branches can intentionally diverge. +export type AnthropicMessageContentBlock = + | { type: "text"; text: string } + | { + type: "tool_use"; + id: string; + name: string; + input: Record; + } + | { + type: "tool_result"; + tool_use_id: string; + is_error?: boolean; + content: string | Array<{ type: "text"; text: string }>; + } + | { type: "image"; source: Record }; + +export type AnthropicMessage = { + role: "user" | "assistant"; + content: string | AnthropicMessageContentBlock[]; +}; + +export type AnthropicMessagesRequest = { + model?: string; + max_tokens?: number; + system?: string | Array<{ type: "text"; text: string }>; + messages?: AnthropicMessage[]; + tools?: Array>; + stream?: boolean; +}; + +export const TINY_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO7Z0nQAAAAASUVORK5CYII="; +export const QA_REASONING_ONLY_RECOVERY_PROMPT_RE = /reasoning-only continuation qa check/i; +export const QA_REASONING_ONLY_SIDE_EFFECT_PROMPT_RE = /reasoning-only after write safety check/i; +export const QA_ANTHROPIC_THINKING_ERROR_RECOVERY_PROMPT_RE = /anthropic thinking error qa check/i; +export const QA_THINKING_VISIBILITY_OFF_PROMPT_RE = /qa thinking visibility check off/i; +export const QA_THINKING_VISIBILITY_MAX_PROMPT_RE = /qa thinking visibility check max/i; +export const QA_EMPTY_RESPONSE_RECOVERY_PROMPT_RE = /empty response continuation qa check/i; +export const QA_EMPTY_RESPONSE_EXHAUSTION_PROMPT_RE = /empty response exhaustion qa check/i; +export const QA_STREAMING_PROMPT_RE = /(?:partial|quiet) streaming qa check/i; +export const QA_FINAL_ONLY_MARKER_STREAMING_PROMPT_RE = /final-only marker streaming qa check/i; +export const QA_BLOCK_STREAMING_PROMPT_RE = /block streaming qa check/i; +export const QA_TOOL_PROGRESS_ERROR_PROMPT_RE = /tool progress error qa check/i; +export const QA_TOOL_PROGRESS_PROMPT_RE = /tool progress qa check/i; +export const QA_GROUP_VISIBLE_REPLY_TOOL_PROMPT_RE = /qa group visible reply tool check/i; +export const QA_A2A_MESSAGE_TOOL_MIRROR_PROMPT_RE = /qa a2a message-tool mirror check/i; +export const QA_GROUP_MESSAGE_UNAVAILABLE_FALLBACK_PROMPT_RE = + /qa group message unavailable fallback check/i; +export const QA_STRANDED_FINAL_RECOVERY_PROMPT_RE = /qa stranded final recovery check/i; +const QA_STRANDED_FINAL_RETRY_FAILURE_PROMPT_RE = /qa stranded final retry failure check/i; +export const QA_STRANDED_FINAL_RETRY_PROMPT_RE = /you did not call message\(action=send\)/i; +const QA_STRANDED_FINAL_RETRY_FAILURE_MARKER = "QA-STRANDED-RETRY-FAIL-RAW"; +export const QA_TELEGRAM_CURRENT_SESSION_STATUS_PROMPT_RE = + /telegram current session_status qa check/i; +export const QA_TELEGRAM_STREAM_SINGLE_MARKER = "QA-TELEGRAM-STREAM-SINGLE-OK"; +export const QA_TELEGRAM_LONG_FINAL_THREE_CHUNK_PROMPT_RE = + /telegram long final three chunk qa check/i; +export const QA_TELEGRAM_LONG_FINAL_PROMPT_RE = /telegram long final qa check/i; +export const QA_WHATSAPP_LONG_FINAL_PROMPT_RE = /whatsapp long final qa check/i; +export const QA_SLACK_CHART_PRESENTATION_PROMPT_RE = + /Slack native chart QA check\s+(SLACK_QA_CHART_SUMMARY_[A-Z0-9]+)[\s\S]*?reply with only this exact marker:\s*(SLACK_QA_CHART_DONE_[A-Z0-9]+)/i; +export const QA_WHATSAPP_AGENT_MESSAGE_ACTION_REACT_PROMPT_RE = + /react to this whatsapp(?: group)? message with thumbs up for qa action check\s+(?:WHATSAPP_QA_AGENT_REACT|WHATSAPP_QA_GROUP_AGENT_REACT)_[A-Z0-9]+/i; +export const QA_WHATSAPP_AGENT_MESSAGE_ACTION_UPLOAD_PROMPT_RE = + /upload-file action to send a PNG with caption\s+((?:WHATSAPP_QA_AGENT_UPLOAD|WHATSAPP_QA_GROUP_AGENT_UPLOAD)_[A-Z0-9]+)/i; +export const QA_WHATSAPP_PENDING_HISTORY_TRIGGER_MARKER_RE = + /\bWHATSAPP_QA_PENDING_HISTORY_TRIGGER_([A-Z0-9]+)\b/u; +export const QA_WHATSAPP_PENDING_HISTORY_STRUCTURED_LABEL = + "Chat history since last reply (untrusted, for context):"; +export const QA_WHATSAPP_BROADCAST_PROMPT_RE = + /\bopenclawqa broadcast fanout check\s+([A-Z0-9_]+)\b/i; +export const QA_WHATSAPP_RUNTIME_AGENT_RE = /\bRuntime:\s*[^\n]*\bagent=([A-Za-z0-9_-]+)/i; +export const QA_WHATSAPP_ACTIVATION_ALWAYS_MARKER_RE = + /\bWHATSAPP_QA_ACTIVATION_ALWAYS_([A-Z0-9]+)\b/u; +export const QA_WHATSAPP_REPLY_TO_BOT_SEED_MARKER_RE = + /\bWHATSAPP_QA_REPLY_TO_BOT_SEED_[A-Z0-9]+\b/u; +export const QA_WHATSAPP_REPLY_TO_BOT_TRIGGER_MARKER_RE = + /\bWHATSAPP_QA_REPLY_TO_BOT_TRIGGER_[A-Z0-9]+\b/u; +export const QA_WHATSAPP_BATCHED_FINAL_MARKER_RE = /\bWHATSAPP_QA_BATCHED_FINAL_([A-Z0-9]+)\b/u; +export const QA_SUBAGENT_DIRECT_FALLBACK_PROMPT_RE = /subagent direct fallback qa check/i; +export const QA_SUBAGENT_DIRECT_FALLBACK_WORKER_RE = /subagent direct fallback worker/i; + +export function buildStrandedFinalRecoveryText(): string { + return [ + "QA-STRANDED-85714 confirms this is a substantive private final reply that initially skipped the message tool.", + "The reply is intentionally long enough to exercise message_tool_only stranded-final recovery before the retry delivers it visibly.", + ].join(" "); +} + +export function buildStrandedFinalRetryFailureText(): string { + return [ + "QA-STRANDED-RETRY-FAIL-RAW confirms this retry also produced a substantive private final reply instead of calling the message tool.", + "This text must remain private so the gateway can deliver only its sanitized failure diagnostic to the source chat.", + ].join(" "); +} + +export function isStrandedFinalRetryFailureRequest(allInputText: string): boolean { + return ( + QA_STRANDED_FINAL_RETRY_FAILURE_PROMPT_RE.test(allInputText) || + (QA_STRANDED_FINAL_RETRY_PROMPT_RE.test(allInputText) && + allInputText.includes(QA_STRANDED_FINAL_RETRY_FAILURE_MARKER)) + ); +} +export const QA_SUBAGENT_DIRECT_FALLBACK_MARKER = "QA-SUBAGENT-DIRECT-FALLBACK-OK"; +export const QA_NATIVE_STOP_DELAY_PROMPT_RE = + /subagent recovery worker native command target proof\.\s*wait until stopped\./i; +export const QA_NATIVE_STOP_DELAY_MS = 180_000; +export const QA_IMAGE_GENERATION_PROMPT_RE = + /image generation check|capability flip image check|\/tool\s+image_generate/i; +export const QA_REASONING_ONLY_RETRY_NEEDLE = + "recorded reasoning but did not produce a user-visible answer"; +export const QA_EMPTY_RESPONSE_RETRY_NEEDLE = + "The previous attempt did not produce a user-visible answer."; +export const QA_SKILL_WORKSHOP_GIF_PROMPT_RE = + /externally sourced animated GIF asset|animated GIF asset in a product UI/i; +export const QA_SKILL_WORKSHOP_REVIEW_PROMPT_RE = /Review transcript for durable skill updates/i; +export const QA_RELEASE_AUDIT_PROMPT_RE = /release readiness audit for the small project/i; +export const QA_TOOL_SEARCH_PROMPT_RE = /tool search qa check/i; +export const QA_TOOL_SEARCH_FAILURE_PROMPT_RE = /tool search qa failure/i; +export const QA_MCP_CODE_MODE_PROMPT_RE = /mcp code mode qa check/i; +export const QA_RESTART_CODE_MODE_WAIT_PROMPT_RE = /code mode restart wait qa check/i; +export const QA_RESTART_RECOVERY_PROMPT_RE = /previous turn was interrupted by a gateway restart/i; +const QA_AUDIO_TRANSCRIPTION_TEXT = + "Reply with only this exact marker: WHATSAPP_QA_AUDIO_TRANSCRIPT_OK"; +const QA_GROUP_AUDIO_TRANSCRIPTION_TEXT = + "openclawqa reply with only this exact marker after group audio preflight: WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_OK"; +const QA_GROUP_AUDIO_TRIGGER_SENTINEL = "OPENCLAW_QA_GROUP_AUDIO_TRIGGER"; +export const QA_MCP_CODE_MODE_API_FILE_PROMPT_RE = /mcp code mode api file qa check/i; + +export type MockScenarioState = { + anthropicThinkingErrorPhase: number; + subagentFanoutPhase: number; + subagentHandoffSpawned: boolean; +}; + +export function sourceDiscoveryReadPathForProvider(providerVariant: MockOpenAiProviderVariant) { + return providerVariant === "anthropic" + ? "repo/docs/help/testing.md" + : "repo/qa/scenarios/index.yaml"; +} + +export function subagentHandoffTaskForProvider(providerVariant: MockOpenAiProviderVariant) { + return providerVariant === "anthropic" + ? "Inspect the QA docs fixture and return one concise protocol note." + : "Inspect the QA workspace and return one concise protocol note."; +} + +export function subagentFanoutTaskForProvider( + providerVariant: MockOpenAiProviderVariant, + worker: "alpha" | "beta", +) { + const marker = worker === "alpha" ? "ALPHA-OK" : "BETA-OK"; + const scope = providerVariant === "anthropic" ? "the QA docs fixture" : "the QA workspace"; + return `Fanout worker ${worker}: inspect ${scope} and finish with exactly ${marker}.`; +} + +const MOCK_OPENAI_MAX_BODY_BYTES = 16 * 1024 * 1024; +const MOCK_OPENAI_BODY_TIMEOUT_MS = 30_000; +export const MOCK_OPENAI_DEBUG_REQUEST_LIMIT = 2_000; + +export function readBody(req: IncomingMessage): Promise { + return readRequestBodyWithLimit(req, { + maxBytes: MOCK_OPENAI_MAX_BODY_BYTES, + timeoutMs: MOCK_OPENAI_BODY_TIMEOUT_MS, + }); +} + +export function parseJsonObjectBody(raw: string): Record | null { + try { + const parsed = raw ? (JSON.parse(raw) as unknown) : {}; + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : null; + } catch { + return null; + } +} + +export function writeOpenAiMalformedJsonError(res: ServerResponse, label: string) { + writeJson(res, 400, { + error: { + type: "invalid_request_error", + message: `Malformed JSON body for ${label} request.`, + }, + }); +} + +export function transcriptionTextForAudioRequest(rawBody: string) { + if (rawBody.includes(QA_GROUP_AUDIO_TRIGGER_SENTINEL)) { + return QA_GROUP_AUDIO_TRANSCRIPTION_TEXT; + } + return QA_AUDIO_TRANSCRIPTION_TEXT; +} + +export function writeSse(res: ServerResponse, events: StreamEvent[]) { + const body = `${events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join("")}data: [DONE]\n\n`; + res.writeHead(200, { + "content-type": "text/event-stream", + "cache-control": "no-store", + connection: "keep-alive", + "content-length": Buffer.byteLength(body), + }); + res.end(body); +} + +export function isRemoteCompactionV2Request(input: ResponsesInputItem[]) { + // Codex sends compaction through /responses with a trigger item. Keep it + // outside scenario dispatch so maintenance calls never become tool evidence. + return input.some((item) => item.type === "compaction_trigger"); +} + +export function buildRemoteCompactionV2Events(): [ + Extract, + Extract, +] { + const item = { + type: "compaction", + encrypted_content: "QA_MOCK_REMOTE_COMPACTION_SUMMARY", + }; + return [ + { type: "response.output_item.done", item }, + { + type: "response.completed", + response: { + id: "resp_mock_compaction_1", + status: "completed", + output: [item], + usage: { input_tokens: 64, output_tokens: 16, total_tokens: 80 }, + }, + }, + ]; +} + +export async function writeSseWithPreviewPause( + res: ServerResponse, + events: StreamEvent[], + pauseMs: number, +) { + const completionIndex = events.findIndex((event) => event.type === "response.output_text.done"); + if (completionIndex < 0) { + writeSse(res, events); + return; + } + res.writeHead(200, { + "content-type": "text/event-stream", + "cache-control": "no-store", + connection: "keep-alive", + }); + for (const event of events.slice(0, completionIndex)) { + res.write(`data: ${JSON.stringify(event)}\n\n`); + } + await sleep(pauseMs); + for (const event of events.slice(completionIndex)) { + res.write(`data: ${JSON.stringify(event)}\n\n`); + } + res.end("data: [DONE]\n\n"); +} + +export type AnthropicStreamEvent = Record & { + type: string; +}; + +export function writeAnthropicSse(res: ServerResponse, events: AnthropicStreamEvent[]) { + const body = events + .map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`) + .join(""); + res.writeHead(200, { + "content-type": "text/event-stream", + "cache-control": "no-store", + connection: "keep-alive", + "content-length": Buffer.byteLength(body), + }); + res.end(body); +} + +export function countApproxTokens(text: string) { + const trimmed = text.trim(); + if (!trimmed) { + return 0; + } + return Math.max(1, Math.ceil(trimmed.length / 4)); +} + +export function extractEmbeddingInputTexts(input: unknown): string[] { + if (typeof input === "string") { + return [input]; + } + if (Array.isArray(input)) { + return input.flatMap((entry) => extractEmbeddingInputTexts(entry)); + } + if ( + input && + typeof input === "object" && + typeof (input as { text?: unknown }).text === "string" + ) { + return [(input as { text: string }).text]; + } + return []; +} + +export function buildDeterministicEmbedding(text: string, dimensions = 16) { + const values = Array.from({ length: dimensions }, () => 0); + for (let index = 0; index < text.length; index += 1) { + const embeddingIndex = index % dimensions; + values[embeddingIndex] = (values[embeddingIndex] ?? 0) + text.charCodeAt(index) / 255; + } + const magnitude = Math.hypot(...values) || 1; + return values.map((value) => Number((value / magnitude).toFixed(8))); +} diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-directives.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-directives.ts new file mode 100644 index 000000000000..8573e769b44f --- /dev/null +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-directives.ts @@ -0,0 +1,323 @@ +// QA Lab mock provider prompt directives and tool declarations. +import { escapeRegExp } from "openclaw/plugin-sdk/text-utility-runtime"; +import { + type ResponsesInputItem, + QA_A2A_MESSAGE_TOOL_MIRROR_PROMPT_RE, + QA_TOOL_SEARCH_PROMPT_RE, + QA_TOOL_SEARCH_FAILURE_PROMPT_RE, +} from "./mock-openai-contracts.js"; +import { extractInstructionsText } from "./mock-openai-input.js"; +function extractLastCapture(text: string, pattern: RegExp) { + let lastMatch: RegExpExecArray | null = null; + const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`; + const globalPattern = new RegExp(pattern.source, flags); + for (let match = globalPattern.exec(text); match; match = globalPattern.exec(text)) { + lastMatch = match; + } + return lastMatch?.[1]?.trim() || null; +} + +function extractCaptures(text: string, pattern: RegExp) { + const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`; + const globalPattern = new RegExp(pattern.source, flags); + return Array.from(text.matchAll(globalPattern), (match) => match[1]?.trim()).filter(Boolean); +} + +export function extractLastMatchingUserText(texts: string[], pattern: RegExp) { + for (let index = texts.length - 1; index >= 0; index -= 1) { + const text = texts[index] ?? ""; + if (pattern.test(text)) { + return text; + } + } + return ""; +} + +export function extractExactReplyDirective(text: string) { + const backtickedMatch = extractLastCapture(text, /reply(?: with)? exactly\s+`([^`]+)`/i); + if (backtickedMatch) { + return backtickedMatch; + } + return ( + extractLastCapture(text, /reply(?: with)? exactly:\s*([^\n]+)/i) ?? + extractLastCapture(text, /reply(?: with)? exactly\s+(?!with\b)([^\s`.,;:!?]+)/i) + ); +} + +export function extractFinishExactlyDirective(text: string) { + const backtickedMatch = extractLastCapture(text, /finish with exactly\s+`([^`]+)`/i); + if (backtickedMatch) { + return backtickedMatch; + } + return extractLastCapture(text, /finish with exactly\s+([^\s`.,;:!?]+)/i); +} + +export function extractExactMarkerDirective(text: string) { + const backtickedMatch = extractLastCapture(text, /exact marker\b[^:\n]{0,120}:\s*`([^`]+)`/i); + if (backtickedMatch) { + return backtickedMatch; + } + return extractLastCapture( + text, + /exact marker\b[^:\n]{0,120}:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i, + ); +} + +export function extractWhatsAppLocationMarkerDirective(text: string) { + return extractLastCapture( + text, + /WhatsApp location marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i, + ); +} + +export function extractWhatsAppContactMarkerDirective(text: string) { + return extractLastCapture(text, /WhatsApp contact marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i); +} + +export function extractWhatsAppStickerMarkerDirective(text: string) { + return extractLastCapture(text, /WhatsApp sticker marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i); +} + +export function shouldUseWhatsAppLocationMarker(prompt: string) { + return /(?:^|[\n:]\s*)šŸ“\s*37\.774900,\s*-122\.419400\b/u.test(prompt.trim()); +} + +export function shouldUseWhatsAppContactMarker(prompt: string) { + return /(?:^|[\n:]\s*))/iu.test(prompt.trim()); +} + +export function shouldUseWhatsAppStickerMarker(prompt: string) { + return /(?:^|[\n:]\s*)(?:\s|$)/iu.test(prompt.trim()); +} + +function extractLabeledMarkerDirective(text: string, label: string) { + const escapedLabel = escapeRegExp(label); + const backtickedMatch = extractLastCapture( + text, + new RegExp(`${escapedLabel}:\\s*\`([^\\\`]+)\``, "i"), + ); + if (backtickedMatch) { + return backtickedMatch; + } + return extractLastCapture( + text, + new RegExp(`${escapedLabel}:\\s*([^\\s\\\`.,;:!?]+(?:-[^\\s\\\`.,;:!?]+)*)`, "i"), + ); +} + +export function extractBlockStreamingMarkerDirectives(text: string) { + const firstLabeledMarker = extractLabeledMarkerDirective(text, "first exact marker"); + const secondLabeledMarker = extractLabeledMarkerDirective(text, "second exact marker"); + if (firstLabeledMarker && secondLabeledMarker) { + return { + first: firstLabeledMarker, + second: secondLabeledMarker, + }; + } + + const markers = extractCaptures(text, /exact marker\b[^:\n]{0,120}:\s*`([^`]+)`/i); + if (markers.length < 2) { + return null; + } + const [first, second] = markers.slice(-2); + return first && second + ? { + first, + second, + } + : null; +} + +function extractQuotedToolArg(text: string, name: string) { + const escapedName = escapeRegExp(name); + return extractLastCapture(text, new RegExp(`\\b${escapedName}\\s*=\\s*"([^"]+)"`, "i")); +} + +function extractBareToolArg(text: string, name: string) { + const escapedName = escapeRegExp(name); + return extractLastCapture(text, new RegExp(`\\b${escapedName}\\s*=\\s*([^\\s\\\`.,;:!?]+)`, "i")); +} + +export function hasDeclaredTool(body: Record, name: string) { + const tools = Array.isArray(body.tools) ? body.tools : []; + const dynamicTools = Array.isArray(body.dynamicTools) ? body.dynamicTools : []; + if ( + [...tools, ...dynamicTools].some((tool) => toolDefinitionMentionsName(tool, name)) || + instructionTextMentionsToolName(extractInstructionsText(body), name) + ) { + return true; + } + return false; +} + +export function hasToolDefinition(body: Record, name: string) { + const tools = Array.isArray(body.tools) ? body.tools : []; + const dynamicTools = Array.isArray(body.dynamicTools) ? body.dynamicTools : []; + return [...tools, ...dynamicTools].some((tool) => toolDefinitionMentionsName(tool, name)); +} + +function toolDefinitionMentionsName(value: unknown, name: string, depth = 0): boolean { + if (depth > 6 || !value || typeof value !== "object") { + return false; + } + if (Array.isArray(value)) { + return value.some((item) => toolDefinitionMentionsName(item, name, depth + 1)); + } + const record = value as Record; + for (const key of ["name", "tool", "functionName"]) { + if (record[key] === name) { + return true; + } + } + return Object.values(record).some((item) => toolDefinitionMentionsName(item, name, depth + 1)); +} + +function instructionTextMentionsToolName(text: string, name: string) { + if (!text) { + return false; + } + const escapedName = escapeRegExp(name); + return new RegExp(`(^|[^A-Za-z0-9_])${escapedName}([^A-Za-z0-9_]|$)`).test(text); +} + +export function isQaToolSearchFixture(text: string) { + return QA_TOOL_SEARCH_PROMPT_RE.test(text) || QA_TOOL_SEARCH_FAILURE_PROMPT_RE.test(text); +} + +export function buildExplicitSessionsSpawnArgs(text: string): Record | null { + if (!/\bsessions_spawn\b/i.test(text)) { + return null; + } + const task = extractQuotedToolArg(text, "task"); + if (!task) { + return null; + } + const label = extractQuotedToolArg(text, "label") ?? extractBareToolArg(text, "label"); + const mode = extractBareToolArg(text, "mode")?.toLowerCase(); + const context = extractBareToolArg(text, "context")?.toLowerCase(); + return { + task, + ...(label ? { label } : {}), + ...(extractBareToolArg(text, "thread")?.toLowerCase() === "true" ? { thread: true } : {}), + ...(mode === "session" || mode === "run" ? { mode } : {}), + ...(context === "fork" || context === "isolated" ? { context } : {}), + }; +} + +export function buildQaA2aMessageToolMirrorSessionsSendArgs( + text: string, +): Record | null { + if (!QA_A2A_MESSAGE_TOOL_MIRROR_PROMPT_RE.test(text)) { + return null; + } + const sessionKey = + extractQuotedToolArg(text, "sessionKey") ?? extractBareToolArg(text, "sessionKey"); + if (!sessionKey) { + return null; + } + const marker = + extractExactMarkerDirective(text) ?? + extractExactReplyDirective(text) ?? + "QA-A2A-MESSAGE-TOOL-MIRROR-OK"; + return { + sessionKey, + message: `qa group visible reply tool check. Use the visible room reply path. exact marker: \`${marker}\``, + timeoutSeconds: 0, + }; +} + +export function extractToolErrorForNamedCall(params: { + input: ResponsesInputItem[]; + name: string; + toolJson: Record | null; +}) { + const error = typeof params.toolJson?.error === "string" ? params.toolJson.error.trim() : ""; + if (!error) { + return undefined; + } + const namedFunctionCall = params.input.some( + (item) => item.type === "function_call" && item.name === params.name, + ); + if (namedFunctionCall) { + return error; + } + return undefined; +} + +export function hasToolErrorOutput(toolJson: Record | null, toolOutput: string) { + if (typeof toolJson?.error === "string" && toolJson.error.trim()) { + return true; + } + if ( + typeof toolJson?.status === "string" && + /\b(?:error|failed|failure)\b/i.test(toolJson.status) + ) { + return true; + } + return /\b(?:error|failed|failure|not found|no such file|enoent)\b/i.test(toolOutput); +} + +export function extractSessionStatusSessionKey( + toolJson: Record | null, + toolOutput: string, +) { + const details = toolJson?.details; + if (details && typeof details === "object") { + const sessionKey = (details as { sessionKey?: unknown }).sessionKey; + if (typeof sessionKey === "string" && sessionKey.trim()) { + return sessionKey.trim(); + } + } + const topLevelSessionKey = toolJson?.sessionKey; + if (typeof topLevelSessionKey === "string" && topLevelSessionKey.trim()) { + return topLevelSessionKey.trim(); + } + const statusLineSessionKey = /(?:^|\n)[^\n]*Session:\s*([^\sā€¢\n]+)/u.exec(toolOutput)?.[1]; + if (statusLineSessionKey?.trim()) { + return statusLineSessionKey.trim(); + } + return /"sessionKey"\s*:\s*"([^"]+)"/.exec(toolOutput)?.[1]?.trim() ?? ""; +} + +export function isHeartbeatPrompt(text: string) { + const trimmed = text.trim(); + if (!trimmed || /remember this fact/i.test(trimmed)) { + return false; + } + return /(?:^|\n)Read HEARTBEAT\.md if it exists\b/i.test(trimmed); +} + +export function readFirstMediaPath(value: unknown): string { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return ""; + } + const media = value as { + mediaUrl?: unknown; + mediaUrls?: unknown; + path?: unknown; + filePath?: unknown; + attachments?: unknown; + }; + for (const candidate of [media.mediaUrl, media.path, media.filePath]) { + if (typeof candidate === "string" && candidate.trim()) { + return candidate.trim(); + } + } + if (Array.isArray(media.mediaUrls)) { + const mediaUrl = media.mediaUrls.find( + (candidate) => typeof candidate === "string" && candidate.trim(), + ); + if (typeof mediaUrl === "string" && mediaUrl.trim()) { + return mediaUrl.trim(); + } + } + if (Array.isArray(media.attachments)) { + for (const attachment of media.attachments) { + const mediaPath = readFirstMediaPath(attachment); + if (mediaPath) { + return mediaPath; + } + } + } + return ""; +} diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-events.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-events.ts new file mode 100644 index 000000000000..194c1dc1d6b8 --- /dev/null +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-events.ts @@ -0,0 +1,422 @@ +// QA Lab mock provider output event builders. + +import type { StreamEvent } from "./mock-openai-contracts.js"; +import { + readTargetFromPrompt, + buildMockFunctionCall, + buildToolCallEventsWithArgs, +} from "./mock-openai-tooling.js"; +export function buildToolCallEvents(prompt: string): StreamEvent[] { + const targetPath = readTargetFromPrompt(prompt); + return buildToolCallEventsWithArgs("read", { path: targetPath }); +} + +export function buildReleaseAuditJson() { + return `${JSON.stringify( + { + verified: false, + findings: [ + { + id: "REL-GATEWAY-417", + source: "src/gateway/reconnect.ts", + status: "retry jitter verified, resume token fallback still needs manual spot check", + verified: true, + }, + { + id: "REL-CHANNEL-238", + source: "src/channels/delivery.ts", + status: "thread replies preserve ordering, root-channel fallback needs handoff note", + verified: true, + }, + { + id: "REL-CRON-904", + source: "src/scheduling/cron.ts", + status: "single-run lock verified for restart wakeups", + verified: true, + }, + { + id: "REL-MEMORY-552", + source: "src/memory/recall.ts", + status: + "fallback summary survives empty memory search; ranking sample needs second reviewer", + verified: true, + }, + { + id: "REL-PLUGIN-319", + source: "src/plugins/runtime.ts", + status: "bundled runtime manifest loads cleanly after restart", + verified: true, + }, + { + id: "REL-INSTALL-846", + source: "install/update.ts", + status: "update smoke passed from previous stable tag", + verified: true, + }, + { + id: "REL-DOCS-611", + source: "docs/operator-notes.md", + status: + "docs mention reconnect, cron, memory, plugin, and installer checks; channel ordering and UI notes need maintainer handoff", + verified: true, + }, + { + id: "REL-UI-BLOCKED", + source: "ui/control-panel.ts", + status: "blocked: source file was referenced by checklist but missing from the fixture", + verified: false, + }, + ], + }, + null, + 2, + )}\n`; +} + +export function buildReleaseHandoffMarkdown() { + return [ + "# Release Handoff", + "", + "Ready:", + "- REL-GATEWAY-417: gateway reconnect handling checked in `src/gateway/reconnect.ts`.", + "- REL-CRON-904: cron duplicate prevention checked in `src/scheduling/cron.ts`.", + "- REL-PLUGIN-319: plugin runtime loading checked in `src/plugins/runtime.ts`.", + "- REL-INSTALL-846: installer update path checked in `install/update.ts`.", + "", + "Follow-up:", + "- REL-CHANNEL-238: channel delivery ordering needs maintainer handoff.", + "- REL-MEMORY-552: memory recall fallback ranking sample needs a second reviewer.", + "- REL-DOCS-611: docs update status needs channel ordering and UI notes.", + "- `ui/control-panel.ts` is blocked/not found in the fixture.", + "", + ].join("\n"); +} + +export function extractPlannedToolName(events: StreamEvent[]) { + for (const event of events) { + if (event.type !== "response.output_item.done") { + continue; + } + const item = event.item as { type?: unknown; name?: unknown }; + if (item.type === "function_call" && typeof item.name === "string") { + return item.name; + } + } + return undefined; +} + +export function extractPlannedToolCallId(events: StreamEvent[]) { + for (const event of events) { + if (event.type !== "response.output_item.done") { + continue; + } + const item = event.item as { type?: unknown; call_id?: unknown }; + if (item.type === "function_call" && typeof item.call_id === "string") { + return item.call_id; + } + } + return undefined; +} + +export function extractPlannedToolArgs(events: StreamEvent[]) { + for (const event of events) { + if (event.type !== "response.output_item.done") { + continue; + } + const item = event.item as { type?: unknown; arguments?: unknown }; + if (item.type !== "function_call" || typeof item.arguments !== "string") { + continue; + } + try { + const parsed = JSON.parse(item.arguments); + return parsed && typeof parsed === "object" ? (parsed as Record) : undefined; + } catch { + return undefined; + } + } + return undefined; +} + +type MockAssistantMessageSpec = { + id: string; + phase?: "commentary" | "final_answer"; + streamDeltas?: string[]; + text: string; +}; + +export function splitMockStreamingText(text: string, parts = 3) { + if (text.length <= 1) { + return [text]; + } + const chunkSize = Math.max(1, Math.ceil(text.length / parts)); + const chunks: string[] = []; + for (let index = 0; index < text.length; index += chunkSize) { + chunks.push(text.slice(index, index + chunkSize)); + } + return chunks.length > 1 ? chunks : [text.slice(0, 1), text.slice(1)]; +} + +export function buildQaLongFinalText({ + endMarker = "TELEGRAM-LONG-FINAL-END", + segmentPrefix = "telegram-long-final-segment", + segmentCount = 42, + startMarker = "TELEGRAM-LONG-FINAL-BEGIN", +}: { + endMarker?: string; + segmentPrefix?: string; + segmentCount?: number; + startMarker?: string; +} = {}) { + const body = Array.from( + { length: segmentCount }, + (_, index) => `${segmentPrefix}-${String(index + 1).padStart(3, "0")} ${"x".repeat(54)}`, + ).join("\n"); + return `${startMarker}\n${body}\n${endMarker}`; +} + +function buildAssistantOutputItem(spec: MockAssistantMessageSpec) { + return { + type: "message", + id: spec.id, + role: "assistant", + status: "completed", + ...(spec.phase ? { phase: spec.phase } : {}), + content: [{ type: "output_text", text: spec.text, annotations: [] }], + } as const; +} + +function appendAssistantMessageEvents(events: StreamEvent[], spec: MockAssistantMessageSpec) { + events.push({ + type: "response.output_item.added", + item: { + type: "message", + id: spec.id, + role: "assistant", + ...(spec.phase ? { phase: spec.phase } : {}), + content: [], + status: "in_progress", + }, + }); + for (const delta of spec.streamDeltas ?? []) { + events.push({ + type: "response.output_text.delta", + item_id: spec.id, + output_index: 0, + content_index: 0, + delta, + }); + } + if ((spec.streamDeltas ?? []).length > 0) { + events.push({ + type: "response.output_text.done", + item_id: spec.id, + output_index: 0, + content_index: 0, + text: spec.text, + }); + } + events.push({ + type: "response.output_item.done", + item: buildAssistantOutputItem(spec), + }); +} + +export function buildAssistantThenToolCallEvents( + spec: MockAssistantMessageSpec, + name: string, + args: Record, +): StreamEvent[] { + const call = buildMockFunctionCall(name, args); + const message = buildAssistantOutputItem(spec); + const events: StreamEvent[] = []; + appendAssistantMessageEvents(events, spec); + events.push({ + type: "response.output_item.added", + item: { + type: "function_call", + id: call.itemId, + call_id: call.callId, + name, + arguments: "", + }, + }); + events.push({ type: "response.function_call_arguments.delta", delta: call.serialized }); + events.push({ + type: "response.output_item.done", + item: call.item, + }); + events.push({ + type: "response.completed", + response: { + id: call.responseId, + status: "completed", + output: [message, call.item], + usage: { input_tokens: 64, output_tokens: 32, total_tokens: 96 }, + }, + }); + return events; +} + +export function buildAssistantEvents( + specsOrText: MockAssistantMessageSpec[] | string, +): StreamEvent[] { + const specs = + typeof specsOrText === "string" + ? [ + { + id: "msg_mock_1", + text: specsOrText, + }, + ] + : specsOrText; + const renderedSpecs = specs.map((spec) => ({ spec, item: buildAssistantOutputItem(spec) })); + const output = renderedSpecs.map(({ item }) => item); + const events: StreamEvent[] = []; + + for (const [outputIndex, { spec, item }] of renderedSpecs.entries()) { + events.push({ + type: "response.output_item.added", + item: { + type: "message", + id: spec.id, + role: "assistant", + ...(spec.phase ? { phase: spec.phase } : {}), + content: [], + status: "in_progress", + }, + }); + for (const delta of spec.streamDeltas ?? []) { + events.push({ + type: "response.output_text.delta", + item_id: spec.id, + output_index: outputIndex, + content_index: 0, + delta, + }); + } + if ((spec.streamDeltas ?? []).length > 0) { + events.push({ + type: "response.output_text.done", + item_id: spec.id, + output_index: outputIndex, + content_index: 0, + text: spec.text, + }); + } + events.push({ + type: "response.output_item.done", + item, + }); + } + + events.push({ + type: "response.completed", + response: { + id: "resp_mock_msg_1", + status: "completed", + output, + usage: { input_tokens: 64, output_tokens: 24, total_tokens: 88 }, + }, + }); + return events; +} + +export function buildReasoningOnlyEvents(summaryText: string, id: string): StreamEvent[] { + const reasoningItem = { + type: "reasoning", + id, + summary: [{ text: summaryText }], + } as const; + return [ + { + type: "response.output_item.added", + item: { + type: "reasoning", + id, + summary: [], + }, + }, + { + type: "response.output_item.done", + item: reasoningItem, + }, + { + type: "response.completed", + response: { + id: `resp_${id}`, + status: "completed", + output: [reasoningItem], + usage: { input_tokens: 64, output_tokens: 8, total_tokens: 72 }, + }, + }, + ]; +} + +export function buildReasoningAndAssistantEvents(params: { + reasoningId: string; + answerText: string; + answerId?: string; +}): StreamEvent[] { + const reasoningItem = { + type: "reasoning", + id: params.reasoningId, + summary: [], + } as const; + const answerItem = buildAssistantOutputItem({ + id: params.answerId ?? "msg_mock_reasoned_answer", + phase: "final_answer", + text: params.answerText, + }); + return [ + { + type: "response.output_item.added", + item: { + type: "reasoning", + id: params.reasoningId, + summary: [], + }, + }, + { + type: "response.output_item.done", + item: reasoningItem, + }, + { + type: "response.output_item.added", + item: { + type: "message", + id: answerItem.id, + role: "assistant", + phase: "final_answer", + content: [], + status: "in_progress", + }, + }, + { + type: "response.output_text.delta", + item_id: answerItem.id, + output_index: 1, + content_index: 0, + delta: params.answerText, + }, + { + type: "response.output_text.done", + item_id: answerItem.id, + output_index: 1, + content_index: 0, + text: params.answerText, + }, + { + type: "response.output_item.done", + item: answerItem, + }, + { + type: "response.completed", + response: { + id: `resp_${params.reasoningId}`, + status: "completed", + output: [reasoningItem, answerItem], + usage: { input_tokens: 64, output_tokens: 16, total_tokens: 80 }, + }, + }, + ]; +} diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-input.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-input.ts new file mode 100644 index 000000000000..f54847dab63f --- /dev/null +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-input.ts @@ -0,0 +1,470 @@ +// QA Lab mock provider input and tool-output extraction. +import { escapeRegExp } from "openclaw/plugin-sdk/text-utility-runtime"; +import { + type ResponsesInputItem, + INTERNAL_RUNTIME_CONTEXT_BEGIN, + INTERNAL_RUNTIME_CONTEXT_END, + QA_WHATSAPP_PENDING_HISTORY_TRIGGER_MARKER_RE, + QA_WHATSAPP_PENDING_HISTORY_STRUCTURED_LABEL, + QA_WHATSAPP_BROADCAST_PROMPT_RE, + QA_WHATSAPP_RUNTIME_AGENT_RE, + QA_WHATSAPP_ACTIVATION_ALWAYS_MARKER_RE, + QA_WHATSAPP_REPLY_TO_BOT_SEED_MARKER_RE, + QA_WHATSAPP_REPLY_TO_BOT_TRIGGER_MARKER_RE, + QA_WHATSAPP_BATCHED_FINAL_MARKER_RE, +} from "./mock-openai-contracts.js"; +export function extractLastUserText(input: ResponsesInputItem[]) { + for (const item of input.toReversed()) { + if (item.role !== "user" || !Array.isArray(item.content)) { + continue; + } + const text = extractInputText(item.content); + if (text && !isInternalRuntimeContextCarrierText(text)) { + return text; + } + } + return ""; +} + +function findLastUserIndex(input: ResponsesInputItem[]) { + return input.findLastIndex( + (item) => + item.role === "user" && + Array.isArray(item.content) && + !isInternalRuntimeContextCarrierText(extractInputText(item.content)), + ); +} + +function isInternalRuntimeContextCarrierText(text: string) { + const trimmed = text.trim(); + return ( + trimmed.includes(INTERNAL_RUNTIME_CONTEXT_BEGIN) && + trimmed.endsWith(INTERNAL_RUNTIME_CONTEXT_END) + ); +} + +function isToolOutputContinuationText(text: string) { + const trimmed = text.trim(); + if (!trimmed) { + return false; + } + return ( + /^(?:continue|keep going|resume|retry|carry on)(?:[.!?])?$/i.test(trimmed) || + /\b(?:continue|continuation|compaction|post-compaction|retry|resume)\b/i.test(trimmed) + ); +} + +function stringifyFunctionCallOutput(output: unknown): string { + if (typeof output === "string") { + return output; + } + if (Array.isArray(output)) { + return output + .map((entry) => { + if (typeof entry === "string") { + return entry; + } + if (!entry || typeof entry !== "object") { + return ""; + } + const record = entry as Record; + if (typeof record.text === "string") { + return record.text; + } + if (typeof record.output_text === "string") { + return record.output_text; + } + if (typeof record.content === "string") { + return record.content; + } + return ""; + }) + .filter(Boolean) + .join("\n"); + } + if (output && typeof output === "object") { + const record = output as Record; + if (typeof record.text === "string") { + return record.text; + } + if (typeof record.output_text === "string") { + return record.output_text; + } + if (typeof record.content === "string") { + return record.content; + } + try { + return JSON.stringify(output); + } catch { + return ""; + } + } + return ""; +} + +function extractFunctionCallOutputText(item: ResponsesInputItem) { + if (item.type !== "function_call_output") { + return ""; + } + return stringifyFunctionCallOutput(item.output); +} + +function extractFunctionCallOutputCallId(item: ResponsesInputItem) { + if (item.type !== "function_call_output") { + return ""; + } + const record = item as { + call_id?: unknown; + tool_call_id?: unknown; + tool_use_id?: unknown; + }; + return ( + [record.call_id, record.tool_call_id, record.tool_use_id].find( + (value): value is string => typeof value === "string" && value.trim().length > 0, + ) ?? "" + ); +} + +function functionCallOutputIsStructuredError(item: ResponsesInputItem) { + if (item.type !== "function_call_output") { + return false; + } + return item.is_error === true || item.isError === true; +} + +export function extractToolOutput(input: ResponsesInputItem[]) { + const lastUserIndex = findLastUserIndex(input); + for (const item of input.slice(lastUserIndex + 1).toReversed()) { + const output = extractFunctionCallOutputText(item); + if (output) { + return output; + } + } + for (const [candidateIndex, candidateItem] of Array.from(input.entries()).toReversed()) { + const output = extractFunctionCallOutputText(candidateItem); + if (output) { + const laterUserTexts = input + .slice(candidateIndex + 1) + .filter((laterItem) => laterItem.role === "user" && Array.isArray(laterItem.content)) + .map((laterItem) => extractInputText(laterItem.content as unknown[])) + .filter(Boolean); + if ( + laterUserTexts.length > 0 && + laterUserTexts.every((text) => isToolOutputContinuationText(text)) + ) { + return output; + } + continue; + } + } + return ""; +} + +export function extractToolOutputStructuredError(input: ResponsesInputItem[]) { + const lastUserIndex = findLastUserIndex(input); + for (const item of input.slice(lastUserIndex + 1).toReversed()) { + const output = extractFunctionCallOutputText(item); + if (output) { + return functionCallOutputIsStructuredError(item); + } + } + for (const [candidateIndex, candidateItem] of Array.from(input.entries()).toReversed()) { + const output = extractFunctionCallOutputText(candidateItem); + if (output) { + const laterUserTexts = input + .slice(candidateIndex + 1) + .filter((laterItem) => laterItem.role === "user" && Array.isArray(laterItem.content)) + .map((laterItem) => extractInputText(laterItem.content as unknown[])) + .filter(Boolean); + if ( + laterUserTexts.length > 0 && + laterUserTexts.every((text) => isToolOutputContinuationText(text)) + ) { + return functionCallOutputIsStructuredError(candidateItem); + } + } + } + return false; +} + +export function extractToolOutputCallId(input: ResponsesInputItem[]) { + const lastUserIndex = findLastUserIndex(input); + for (const item of input.slice(lastUserIndex + 1).toReversed()) { + const output = extractFunctionCallOutputText(item); + if (output) { + return extractFunctionCallOutputCallId(item); + } + } + for (const [candidateIndex, candidateItem] of Array.from(input.entries()).toReversed()) { + const output = extractFunctionCallOutputText(candidateItem); + if (output) { + const laterUserTexts = input + .slice(candidateIndex + 1) + .filter((laterItem) => laterItem.role === "user" && Array.isArray(laterItem.content)) + .map((laterItem) => extractInputText(laterItem.content as unknown[])) + .filter(Boolean); + if ( + laterUserTexts.length > 0 && + laterUserTexts.every((text) => isToolOutputContinuationText(text)) + ) { + return extractFunctionCallOutputCallId(candidateItem); + } + } + } + return ""; +} + +export function extractLatestToolOutput(input: ResponsesInputItem[]) { + for (const item of input.toReversed()) { + const output = extractFunctionCallOutputText(item); + if (output) { + return output; + } + } + return ""; +} + +export function extractAllToolOutputText(input: ResponsesInputItem[]) { + return input + .map((item) => extractFunctionCallOutputText(item)) + .filter(Boolean) + .join("\n"); +} + +export function extractUserTextAfterLatestToolOutput(input: ResponsesInputItem[]) { + const latestToolOutputIndex = input.findLastIndex((item) => + Boolean(extractFunctionCallOutputText(item)), + ); + if (latestToolOutputIndex < 0) { + return ""; + } + return input + .slice(latestToolOutputIndex + 1) + .filter((item) => item.role === "user" && Array.isArray(item.content)) + .map((item) => extractInputText(item.content as unknown[])) + .filter(Boolean) + .join("\n"); +} + +function extractInputText(content: unknown[]): string { + return content + .filter( + (entry): entry is { type: "input_text"; text: string } => + Boolean(entry) && + typeof entry === "object" && + (entry as { type?: unknown }).type === "input_text" && + typeof (entry as { text?: unknown }).text === "string", + ) + .map((entry) => entry.text) + .join("\n") + .trim(); +} + +export function extractAllUserTexts(input: ResponsesInputItem[]) { + const texts: string[] = []; + for (const item of input) { + if (item.role !== "user" || !Array.isArray(item.content)) { + continue; + } + const text = extractInputText(item.content); + if (text) { + texts.push(text); + } + } + return texts; +} + +export function extractSystemInputText(input: ResponsesInputItem[]) { + const texts: string[] = []; + for (const item of input) { + if (item.role !== "system") { + continue; + } + if (typeof item.content === "string" && item.content.trim()) { + texts.push(item.content.trim()); + continue; + } + if (!Array.isArray(item.content)) { + continue; + } + const text = extractInputText(item.content); + if (text) { + texts.push(text); + } + } + return texts.join("\n"); +} + +export function extractAllInputTexts(input: ResponsesInputItem[]) { + const texts: string[] = []; + for (const item of input) { + if (typeof item.output === "string" && item.output.trim()) { + texts.push(item.output.trim()); + } + if (!Array.isArray(item.content)) { + continue; + } + const text = extractInputText(item.content); + if (text) { + texts.push(text); + } + } + return texts.join("\n"); +} + +export function extractInstructionsText(body: Record) { + return typeof body.instructions === "string" ? body.instructions.trim() : ""; +} + +export function extractAllRequestTexts(input: ResponsesInputItem[], body: Record) { + const texts: string[] = []; + const instructions = extractInstructionsText(body); + if (instructions) { + texts.push(instructions); + } + const inputText = extractAllInputTexts(input); + if (inputText) { + texts.push(inputText); + } + return texts.join("\n"); +} + +export function buildWhatsAppPendingHistoryReply(allInputText: string) { + const triggerMatch = QA_WHATSAPP_PENDING_HISTORY_TRIGGER_MARKER_RE.exec(allInputText); + if (!triggerMatch?.[1]) { + return undefined; + } + const suffix = triggerMatch[1]; + const beforeTrigger = allInputText.slice(0, triggerMatch.index); + const priorGroupContext = extractStructuredWhatsAppPendingHistoryContext(beforeTrigger); + const quietMarkerPattern = new RegExp(`\\bWHATSAPP_QA_PENDING_HISTORY_QUIET_${suffix}\\b`, "u"); + const contextSentinelPattern = new RegExp( + `\\bWHATSAPP_QA_PENDING_HISTORY_CONTEXT_ONLY_${suffix}\\b`, + "u", + ); + if ( + !quietMarkerPattern.test(priorGroupContext) || + !contextSentinelPattern.test(priorGroupContext) + ) { + return "WHATSAPP_QA_PENDING_HISTORY_MISSING_CONTEXT"; + } + return `WHATSAPP_QA_PENDING_HISTORY_OK_${suffix}`; +} + +function extractStructuredWhatsAppPendingHistoryContext(beforeTrigger: string) { + const blockRe = new RegExp( + `${escapeRegExp(QA_WHATSAPP_PENDING_HISTORY_STRUCTURED_LABEL)}\\n((?:(?!\\n\\n)[\\s\\S])+)\\n\\n`, + "gu", + ); + return Array.from(beforeTrigger.matchAll(blockRe), (match) => match[1]?.trim()) + .filter((block): block is string => Boolean(block)) + .join("\n"); +} + +export function buildWhatsAppBroadcastReply(allInputText: string) { + const promptMatch = QA_WHATSAPP_BROADCAST_PROMPT_RE.exec(allInputText); + const token = promptMatch?.[1]; + if (!token) { + return undefined; + } + const agentId = QA_WHATSAPP_RUNTIME_AGENT_RE.exec(allInputText)?.[1]; + if (agentId === "main") { + return `${token}_MAIN`; + } + if (agentId === "qa-second") { + return `${token}_SECOND`; + } + return "WHATSAPP_QA_BROADCAST_AGENT_CONTEXT_MISSING"; +} + +export function buildWhatsAppGroupDispatchReply(allInputText: string) { + const activationMatch = QA_WHATSAPP_ACTIVATION_ALWAYS_MARKER_RE.exec(allInputText); + if (activationMatch?.[1]) { + return `WHATSAPP_QA_ACTIVATION_ALWAYS_${activationMatch[1]}`; + } + const triggerMatch = QA_WHATSAPP_REPLY_TO_BOT_TRIGGER_MARKER_RE.exec(allInputText); + if (triggerMatch?.[0]) { + return triggerMatch[0]; + } + return QA_WHATSAPP_REPLY_TO_BOT_SEED_MARKER_RE.exec(allInputText)?.[0]; +} + +export function buildWhatsAppBatchedReply(allInputText: string) { + const finalMatch = QA_WHATSAPP_BATCHED_FINAL_MARKER_RE.exec(allInputText); + const suffix = finalMatch?.[1]; + if (!suffix) { + return undefined; + } + const firstMarker = `WHATSAPP_QA_BATCHED_FIRST_${suffix}`; + if (!allInputText.includes(firstMarker)) { + return `WHATSAPP_QA_BATCHED_MISSING_CONTEXT_${suffix}`; + } + return finalMatch[0]; +} + +export function countImageInputs(value: unknown): number { + const seen = new WeakSet(); + const stack = [value]; + let count = 0; + let visited = 0; + while (stack.length > 0 && visited < 50_000) { + visited += 1; + const current = stack.pop(); + if (Array.isArray(current)) { + for (const entry of current) { + stack.push(entry); + } + continue; + } + if (!current || typeof current !== "object") { + continue; + } + if (seen.has(current)) { + continue; + } + seen.add(current); + const record = current as Record; + const type = typeof record.type === "string" ? record.type : ""; + if (type === "input_image" || type === "image" || type === "image_url" || type === "media") { + count += 1; + } + stack.push(record.content, record.image_url, record.source); + } + return count; +} + +export function extractLatestImageUserTurn(input: ResponsesInputItem[]) { + const latestUserIndex = findLastUserIndex(input); + if (latestUserIndex < 0) { + return { text: "", imageInputCount: 0 }; + } + + const latestUserItem = input[latestUserIndex]; + if (!latestUserItem) { + return { text: "", imageInputCount: 0 }; + } + + const imageTurnItems = [latestUserItem]; + const imageInputCount = countImageInputs(imageTurnItems.map((item) => item.content)); + if (imageInputCount === 0) { + return { text: "", imageInputCount: 0 }; + } + return { + text: imageTurnItems + .map((item) => extractInputText(item.content as unknown[])) + .filter(Boolean) + .join("\n"), + imageInputCount, + }; +} + +export function parseToolOutputJson(toolOutput: string): Record | null { + if (!toolOutput.trim()) { + return null; + } + try { + return JSON.parse(toolOutput) as Record; + } catch { + return null; + } +} diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-tooling.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-tooling.ts new file mode 100644 index 000000000000..81c2c2fadc2a --- /dev/null +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-tooling.ts @@ -0,0 +1,246 @@ +// QA Lab mock provider tool planning and memory fixtures. +import { createHash } from "node:crypto"; +import { QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY } from "../../qa-web-search-provider.js"; +import type { StreamEvent } from "./mock-openai-contracts.js"; + +let mockFunctionCallSequence = 0; + +function normalizePromptPathCandidate(candidate: string) { + const trimmed = candidate.trim().replace(/^`+|`+$/g, ""); + if (!trimmed) { + return null; + } + const normalized = trimmed.replace(/^\.\//, ""); + if ( + normalized.includes("/") || + /\.(?:md|json|ts|tsx|js|mjs|cjs|txt|yaml|yml)$/i.test(normalized) + ) { + return normalized; + } + return null; +} + +export function readTargetFromPrompt(prompt: string) { + const backtickedMatches = Array.from(prompt.matchAll(/`([^`]+)`/g)) + .map((match) => normalizePromptPathCandidate(match[1] ?? "")) + .filter((value): value is string => Boolean(value)); + if (backtickedMatches.length > 0) { + return backtickedMatches[0]; + } + + const quotedMatches = Array.from(prompt.matchAll(/"([^"]+)"/g)) + .map((match) => normalizePromptPathCandidate(match[1] ?? "")) + .filter((value): value is string => Boolean(value)); + if (quotedMatches.length > 0) { + return quotedMatches[0]; + } + + const repoScoped = /\b(?:repo\/[^\s`",)]+|QA_[A-Z_]+\.md)\b/.exec(prompt)?.[0]?.trim(); + if (repoScoped) { + return repoScoped; + } + + const loosePath = /\b[A-Za-z0-9._-]+\.(?:md|json|ts|tsx|js|mjs|cjs|txt|yaml|yml)\b/i + .exec(prompt)?.[0] + ?.trim(); + if (loosePath) { + return loosePath; + } + + if (/\bdocs?\b/i.test(prompt)) { + return "repo/docs/help/testing.md"; + } + if (/\bscenario|kickoff|qa\b/i.test(prompt)) { + return "QA_KICKOFF_TASK.md"; + } + return "repo/package.json"; +} + +export function execCommandFromToolProgressPrompt(prompt: string) { + return ( + /call the exec tool exactly once with this exact command before answering:\s*`([^`]+)`/i + .exec(prompt)?.[1] + ?.trim() || null + ); +} + +export function buildMockFunctionCall(name: string, args: Record) { + const serialized = JSON.stringify(args); + const callSuffix = createHash("sha256") + .update(name) + .update("\0") + .update(serialized) + .digest("hex") + .slice(0, 10); + const sequence = ++mockFunctionCallSequence; + const uniqueSuffix = `${callSuffix}_${sequence}`; + const callId = `call_mock_${name}_${uniqueSuffix}`; + const itemId = `fc_mock_${name}_${uniqueSuffix}`; + const item = { + type: "function_call", + id: itemId, + call_id: callId, + name, + arguments: serialized, + }; + return { + callId, + item, + itemId, + responseId: `resp_mock_${name}_${uniqueSuffix}`, + serialized, + }; +} + +export function buildToolCallEventsWithArgs( + name: string, + args: Record, +): StreamEvent[] { + const call = buildMockFunctionCall(name, args); + return [ + { + type: "response.output_item.added", + item: { + type: "function_call", + id: call.itemId, + call_id: call.callId, + name, + arguments: "", + }, + }, + { type: "response.function_call_arguments.delta", delta: call.serialized }, + { + type: "response.output_item.done", + item: call.item, + }, + { + type: "response.completed", + response: { + id: call.responseId, + status: "completed", + output: [call.item], + usage: { input_tokens: 64, output_tokens: 16, total_tokens: 80 }, + }, + }, + ]; +} + +export function extractRememberedFact(userTexts: string[]) { + for (const text of userTexts) { + const qaCanaryMatch = /\bqa canary code is\s+([A-Za-z0-9-]+)/i.exec(text); + if (qaCanaryMatch?.[1]) { + return qaCanaryMatch[1]; + } + } + for (const text of userTexts) { + const match = /remember(?: this fact for later)?:\s*([A-Za-z0-9-]+)/i.exec(text); + if (match?.[1]) { + return match[1]; + } + } + return null; +} + +export function extractOrbitCode(text: string) { + return /\bORBIT-\d+\b/i.exec(text)?.[0]?.toUpperCase() ?? null; +} + +function decodeXmlEntities(text: string) { + return text + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll("&", "&") + .replaceAll(""", '"') + .replaceAll("'", "'"); +} + +export function extractActiveMemorySummary(text: string) { + const match = /\s*([\s\S]*?)\s*<\/active_memory_plugin>/i.exec(text); + return match?.[1] ? decodeXmlEntities(match[1]).trim() : null; +} + +export function extractToolSearchTarget(text: string): string | null { + const match = /\btarget=([A-Za-z0-9_.:-]+)\b/.exec(text); + return match?.[1]?.trim() || null; +} + +export function buildQaToolSearchArgs( + targetTool: string, + failureMode: boolean, +): Record { + if (failureMode && targetTool === "web_search") { + return { query: QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY }; + } + if (failureMode) { + return { __qaFailureMode: "denied-input" }; + } + if (targetTool === "exec") { + return { command: "echo runtime-tool-fixture", timeout: 5 }; + } + if (targetTool === "read") { + return { path: "QA_KICKOFF_TASK.md" }; + } + if (targetTool === "write") { + return { path: "runtime-tool-fixture-write.txt", content: "runtime tool fixture\n" }; + } + if (targetTool === "edit") { + return { + path: "runtime-tool-fixture-edit.txt", + edits: [{ oldText: "before edit\n", newText: "after edit\n" }], + }; + } + if (targetTool === "apply_patch") { + return { + input: [ + "*** Begin Patch", + "*** Add File: runtime-tool-fixture-patch.txt", + "+runtime patch", + "*** End Patch", + "", + ].join("\n"), + }; + } + if (targetTool === "web_search") { + return { query: "OpenClaw runtime parity fixed query", count: 1 }; + } + if (targetTool === "web_fetch") { + return { url: "https://example.com/", maxChars: 500 }; + } + if (targetTool === "image_generate") { + return { prompt: "QA lighthouse runtime parity fixture", filename: "runtime-tool-fixture" }; + } + if (targetTool === "tts") { + return { text: "Runtime parity voice fixture." }; + } + if (targetTool === "message") { + return { action: "send", message: "runtime parity message fixture" }; + } + if (targetTool === "session_status") { + return { sessionKey: "current" }; + } + if (targetTool === "sessions_spawn") { + return { + task: "Runtime tool fixture subagent: reply exactly RUNTIME-TOOL-FIXTURE.", + label: "runtime-tool-fixture", + mode: "run", + thread: false, + }; + } + if (targetTool === "memory_recall") { + return { query: "runtime parity memory fixture" }; + } + return { marker: "normal" }; +} + +export function isActiveMemorySubagentPrompt(text: string) { + return text.includes("You are a memory search agent."); +} + +export function extractSnackPreference(text: string) { + const normalized = text.replace(/\s+/g, " ").trim(); + const match = + /(lemon pepper wings(?:\s+with\s+blue cheese)?|blue cheese(?:\s+with\s+lemon pepper wings)?)/i.exec( + normalized, + ); + return match?.[0]?.trim() ?? null; +} diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index 37c79ccb9a39..9103d2488de5 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -1,2168 +1,146 @@ -// Qa Lab plugin module implements server behavior. -import { createHash } from "node:crypto"; -import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +// QA Lab mock Responses dispatcher, HTTP transport, and debug endpoints. +import { createServer } from "node:http"; import { setTimeout as sleep } from "node:timers/promises"; -import { escapeRegExp, truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; -import { readRequestBodyWithLimit } from "openclaw/plugin-sdk/webhook-ingress"; import { closeQaHttpServer } from "../../bus-server.js"; -import { QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY } from "../../qa-web-search-provider.js"; import { parseQaDebugRequestCursor } from "../shared/debug-request-cursor.js"; import { writeJson } from "../shared/http-json.js"; import { listMockOpenAiServerModelIds } from "../shared/mock-model-config.js"; - -type ResponsesInputItem = Record; - -let mockFunctionCallSequence = 0; - -type StreamEvent = - | { type: "response.output_item.added"; item: Record } - | { - type: "response.output_text.delta"; - item_id: string; - output_index: number; - content_index: number; - delta: string; - } - | { - type: "response.output_text.done"; - item_id: string; - output_index: number; - content_index: number; - text: string; - } - | { type: "response.function_call_arguments.delta"; delta: string } - | { type: "response.output_item.done"; item: Record } - | { - type: "response.completed"; - response: { - id: string; - status: "completed"; - output: Array>; - usage: { - input_tokens: number; - output_tokens: number; - total_tokens: number; - }; - }; - }; - -/** - * Provider variant tag for `body.model`. The mock previously ignored - * `body.model` for dispatch and only echoed it in the prose output, which - * made the parity gate tautological when run against the mock alone - * (both providers produced identical scenario plans by construction). - * Tagging requests with a normalized variant lets individual scenario - * branches opt into provider-specific behavior while the rest of the - * dispatcher stays shared, and lets `/debug/requests` consumers verify - * which provider lane a given request came from without re-parsing the - * raw model string. - * - * Policy: - * - `openai/*`, `gpt-*`, `o1-*`, anything starting with `gpt-` ā†’ `"openai"` - * - `anthropic/*`, `claude-*` ā†’ `"anthropic"` - * - Everything else (including empty strings) ā†’ `"unknown"` - * - * The `/v1/messages` route always feeds `body.model` straight through, - * so an Anthropic request with an `openai/gpt-5.6-luna` model string is still - * classified as `"openai"`. That matches the parity program's convention - * where the provider label is the source of truth, not the HTTP route. - */ -type MockOpenAiProviderVariant = "openai" | "anthropic" | "unknown"; - -function resolveProviderVariant(model: string | undefined): MockOpenAiProviderVariant { - if (typeof model !== "string") { - return "unknown"; - } - const trimmed = model.trim().toLowerCase(); - if (trimmed.length === 0) { - return "unknown"; - } - // Prefer the explicit `provider/model` or `provider:model` prefix when - // the caller supplied one ā€” that's the most reliable signal. - const separatorMatch = /^([^/:]+)[/:]/.exec(trimmed); - const provider = separatorMatch?.[1] ?? trimmed; - if (provider === "openai") { - return "openai"; - } - if (provider === "anthropic" || provider === "claude-cli") { - return "anthropic"; - } - // Fall back to model-name prefix matching for bare model strings like - // `gpt-5.6-luna` or `claude-opus-4-8`. - if (/^(?:gpt-|o1-|openai-)/.test(trimmed)) { - return "openai"; - } - if (/^(?:claude-|anthropic-)/.test(trimmed)) { - return "anthropic"; - } - return "unknown"; -} - -type MockOpenAiRequestSnapshot = { - cursor: number; - raw: string; - body: Record; - prompt: string; - allInputText: string; - instructions?: string; - toolOutput: string; - model: string; - providerVariant: MockOpenAiProviderVariant; - imageInputCount: number; - plannedToolCallId?: string; - plannedToolName?: string; - plannedToolArgs?: Record; - toolOutputCallId?: string; - toolOutputStructuredError?: true; -}; - -type MockOpenAiRequestSnapshotInput = Omit; - -// Runtime-context delimiters are owned by src/agents/internal-runtime-context.ts. -// This mock mirrors the wire shape so delimiter drift fails through QA timeouts. -const INTERNAL_RUNTIME_CONTEXT_BEGIN = "<<>>"; -const INTERNAL_RUNTIME_CONTEXT_END = "<<>>"; - -// Anthropic /v1/messages request/response shapes the mock actually needs. -// This is a subset of the real Anthropic Messages API ā€” just enough so the -// QA suite can run its parity pack against a "baseline" Anthropic provider -// without needing real API keys. The scenarios drive their dispatch through -// the shared mock scenario logic (buildResponsesPayload), with `model` -// preserved so provider-aware branches can intentionally diverge. -type AnthropicMessageContentBlock = - | { type: "text"; text: string } - | { - type: "tool_use"; - id: string; - name: string; - input: Record; - } - | { - type: "tool_result"; - tool_use_id: string; - is_error?: boolean; - content: string | Array<{ type: "text"; text: string }>; - } - | { type: "image"; source: Record }; - -type AnthropicMessage = { - role: "user" | "assistant"; - content: string | AnthropicMessageContentBlock[]; -}; - -type AnthropicMessagesRequest = { - model?: string; - max_tokens?: number; - system?: string | Array<{ type: "text"; text: string }>; - messages?: AnthropicMessage[]; - tools?: Array>; - stream?: boolean; -}; - -const TINY_PNG_BASE64 = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO7Z0nQAAAAASUVORK5CYII="; -const QA_REASONING_ONLY_RECOVERY_PROMPT_RE = /reasoning-only continuation qa check/i; -const QA_REASONING_ONLY_SIDE_EFFECT_PROMPT_RE = /reasoning-only after write safety check/i; -const QA_ANTHROPIC_THINKING_ERROR_RECOVERY_PROMPT_RE = /anthropic thinking error qa check/i; -const QA_THINKING_VISIBILITY_OFF_PROMPT_RE = /qa thinking visibility check off/i; -const QA_THINKING_VISIBILITY_MAX_PROMPT_RE = /qa thinking visibility check max/i; -const QA_EMPTY_RESPONSE_RECOVERY_PROMPT_RE = /empty response continuation qa check/i; -const QA_EMPTY_RESPONSE_EXHAUSTION_PROMPT_RE = /empty response exhaustion qa check/i; -const QA_STREAMING_PROMPT_RE = /(?:partial|quiet) streaming qa check/i; -const QA_FINAL_ONLY_MARKER_STREAMING_PROMPT_RE = /final-only marker streaming qa check/i; -const QA_BLOCK_STREAMING_PROMPT_RE = /block streaming qa check/i; -const QA_TOOL_PROGRESS_ERROR_PROMPT_RE = /tool progress error qa check/i; -const QA_TOOL_PROGRESS_PROMPT_RE = /tool progress qa check/i; -const QA_GROUP_VISIBLE_REPLY_TOOL_PROMPT_RE = /qa group visible reply tool check/i; -const QA_A2A_MESSAGE_TOOL_MIRROR_PROMPT_RE = /qa a2a message-tool mirror check/i; -const QA_GROUP_MESSAGE_UNAVAILABLE_FALLBACK_PROMPT_RE = - /qa group message unavailable fallback check/i; -const QA_STRANDED_FINAL_RECOVERY_PROMPT_RE = /qa stranded final recovery check/i; -const QA_STRANDED_FINAL_RETRY_FAILURE_PROMPT_RE = /qa stranded final retry failure check/i; -const QA_STRANDED_FINAL_RETRY_PROMPT_RE = /you did not call message\(action=send\)/i; -const QA_STRANDED_FINAL_RETRY_FAILURE_MARKER = "QA-STRANDED-RETRY-FAIL-RAW"; -const QA_TELEGRAM_CURRENT_SESSION_STATUS_PROMPT_RE = /telegram current session_status qa check/i; -const QA_TELEGRAM_STREAM_SINGLE_MARKER = "QA-TELEGRAM-STREAM-SINGLE-OK"; -const QA_TELEGRAM_LONG_FINAL_THREE_CHUNK_PROMPT_RE = /telegram long final three chunk qa check/i; -const QA_TELEGRAM_LONG_FINAL_PROMPT_RE = /telegram long final qa check/i; -const QA_WHATSAPP_LONG_FINAL_PROMPT_RE = /whatsapp long final qa check/i; -const QA_SLACK_CHART_PRESENTATION_PROMPT_RE = - /Slack native chart QA check\s+(SLACK_QA_CHART_SUMMARY_[A-Z0-9]+)[\s\S]*?reply with only this exact marker:\s*(SLACK_QA_CHART_DONE_[A-Z0-9]+)/i; -const QA_WHATSAPP_AGENT_MESSAGE_ACTION_REACT_PROMPT_RE = - /react to this whatsapp(?: group)? message with thumbs up for qa action check\s+(?:WHATSAPP_QA_AGENT_REACT|WHATSAPP_QA_GROUP_AGENT_REACT)_[A-Z0-9]+/i; -const QA_WHATSAPP_AGENT_MESSAGE_ACTION_UPLOAD_PROMPT_RE = - /upload-file action to send a PNG with caption\s+((?:WHATSAPP_QA_AGENT_UPLOAD|WHATSAPP_QA_GROUP_AGENT_UPLOAD)_[A-Z0-9]+)/i; -const QA_WHATSAPP_PENDING_HISTORY_TRIGGER_MARKER_RE = - /\bWHATSAPP_QA_PENDING_HISTORY_TRIGGER_([A-Z0-9]+)\b/u; -const QA_WHATSAPP_PENDING_HISTORY_STRUCTURED_LABEL = - "Chat history since last reply (untrusted, for context):"; -const QA_WHATSAPP_BROADCAST_PROMPT_RE = /\bopenclawqa broadcast fanout check\s+([A-Z0-9_]+)\b/i; -const QA_WHATSAPP_RUNTIME_AGENT_RE = /\bRuntime:\s*[^\n]*\bagent=([A-Za-z0-9_-]+)/i; -const QA_WHATSAPP_ACTIVATION_ALWAYS_MARKER_RE = /\bWHATSAPP_QA_ACTIVATION_ALWAYS_([A-Z0-9]+)\b/u; -const QA_WHATSAPP_REPLY_TO_BOT_SEED_MARKER_RE = /\bWHATSAPP_QA_REPLY_TO_BOT_SEED_[A-Z0-9]+\b/u; -const QA_WHATSAPP_REPLY_TO_BOT_TRIGGER_MARKER_RE = - /\bWHATSAPP_QA_REPLY_TO_BOT_TRIGGER_[A-Z0-9]+\b/u; -const QA_WHATSAPP_BATCHED_FINAL_MARKER_RE = /\bWHATSAPP_QA_BATCHED_FINAL_([A-Z0-9]+)\b/u; -const QA_SUBAGENT_DIRECT_FALLBACK_PROMPT_RE = /subagent direct fallback qa check/i; -const QA_SUBAGENT_DIRECT_FALLBACK_WORKER_RE = /subagent direct fallback worker/i; - -function buildStrandedFinalRecoveryText(): string { - return [ - "QA-STRANDED-85714 confirms this is a substantive private final reply that initially skipped the message tool.", - "The reply is intentionally long enough to exercise message_tool_only stranded-final recovery before the retry delivers it visibly.", - ].join(" "); -} - -function buildStrandedFinalRetryFailureText(): string { - return [ - "QA-STRANDED-RETRY-FAIL-RAW confirms this retry also produced a substantive private final reply instead of calling the message tool.", - "This text must remain private so the gateway can deliver only its sanitized failure diagnostic to the source chat.", - ].join(" "); -} - -function isStrandedFinalRetryFailureRequest(allInputText: string): boolean { - return ( - QA_STRANDED_FINAL_RETRY_FAILURE_PROMPT_RE.test(allInputText) || - (QA_STRANDED_FINAL_RETRY_PROMPT_RE.test(allInputText) && - allInputText.includes(QA_STRANDED_FINAL_RETRY_FAILURE_MARKER)) - ); -} -const QA_SUBAGENT_DIRECT_FALLBACK_MARKER = "QA-SUBAGENT-DIRECT-FALLBACK-OK"; -const QA_NATIVE_STOP_DELAY_PROMPT_RE = - /subagent recovery worker native command target proof\.\s*wait until stopped\./i; -const QA_NATIVE_STOP_DELAY_MS = 180_000; -const QA_IMAGE_GENERATION_PROMPT_RE = - /image generation check|capability flip image check|\/tool\s+image_generate/i; -const QA_REASONING_ONLY_RETRY_NEEDLE = - "recorded reasoning but did not produce a user-visible answer"; -const QA_EMPTY_RESPONSE_RETRY_NEEDLE = - "The previous attempt did not produce a user-visible answer."; -const QA_SKILL_WORKSHOP_GIF_PROMPT_RE = - /externally sourced animated GIF asset|animated GIF asset in a product UI/i; -const QA_SKILL_WORKSHOP_REVIEW_PROMPT_RE = /Review transcript for durable skill updates/i; -const QA_RELEASE_AUDIT_PROMPT_RE = /release readiness audit for the small project/i; -const QA_TOOL_SEARCH_PROMPT_RE = /tool search qa check/i; -const QA_TOOL_SEARCH_FAILURE_PROMPT_RE = /tool search qa failure/i; -const QA_MCP_CODE_MODE_PROMPT_RE = /mcp code mode qa check/i; -const QA_RESTART_CODE_MODE_WAIT_PROMPT_RE = /code mode restart wait qa check/i; -const QA_RESTART_RECOVERY_PROMPT_RE = /previous turn was interrupted by a gateway restart/i; -const QA_AUDIO_TRANSCRIPTION_TEXT = - "Reply with only this exact marker: WHATSAPP_QA_AUDIO_TRANSCRIPT_OK"; -const QA_GROUP_AUDIO_TRANSCRIPTION_TEXT = - "openclawqa reply with only this exact marker after group audio preflight: WHATSAPP_QA_GROUP_AUDIO_TRANSCRIPT_OK"; -const QA_GROUP_AUDIO_TRIGGER_SENTINEL = "OPENCLAW_QA_GROUP_AUDIO_TRIGGER"; -const QA_MCP_CODE_MODE_API_FILE_PROMPT_RE = /mcp code mode api file qa check/i; - -type MockScenarioState = { - anthropicThinkingErrorPhase: number; - subagentFanoutPhase: number; - subagentHandoffSpawned: boolean; -}; - -function sourceDiscoveryReadPathForProvider(providerVariant: MockOpenAiProviderVariant) { - return providerVariant === "anthropic" - ? "repo/docs/help/testing.md" - : "repo/qa/scenarios/index.yaml"; -} - -function subagentHandoffTaskForProvider(providerVariant: MockOpenAiProviderVariant) { - return providerVariant === "anthropic" - ? "Inspect the QA docs fixture and return one concise protocol note." - : "Inspect the QA workspace and return one concise protocol note."; -} - -function subagentFanoutTaskForProvider( - providerVariant: MockOpenAiProviderVariant, - worker: "alpha" | "beta", -) { - const marker = worker === "alpha" ? "ALPHA-OK" : "BETA-OK"; - const scope = providerVariant === "anthropic" ? "the QA docs fixture" : "the QA workspace"; - return `Fanout worker ${worker}: inspect ${scope} and finish with exactly ${marker}.`; -} - -const MOCK_OPENAI_MAX_BODY_BYTES = 16 * 1024 * 1024; -const MOCK_OPENAI_BODY_TIMEOUT_MS = 30_000; -const MOCK_OPENAI_DEBUG_REQUEST_LIMIT = 2_000; - -function readBody(req: IncomingMessage): Promise { - return readRequestBodyWithLimit(req, { - maxBytes: MOCK_OPENAI_MAX_BODY_BYTES, - timeoutMs: MOCK_OPENAI_BODY_TIMEOUT_MS, - }); -} - -function parseJsonObjectBody(raw: string): Record | null { - try { - const parsed = raw ? (JSON.parse(raw) as unknown) : {}; - return parsed && typeof parsed === "object" && !Array.isArray(parsed) - ? (parsed as Record) - : null; - } catch { - return null; - } -} - -function writeOpenAiMalformedJsonError(res: ServerResponse, label: string) { - writeJson(res, 400, { - error: { - type: "invalid_request_error", - message: `Malformed JSON body for ${label} request.`, - }, - }); -} - -function transcriptionTextForAudioRequest(rawBody: string) { - if (rawBody.includes(QA_GROUP_AUDIO_TRIGGER_SENTINEL)) { - return QA_GROUP_AUDIO_TRANSCRIPTION_TEXT; - } - return QA_AUDIO_TRANSCRIPTION_TEXT; -} - -function writeSse(res: ServerResponse, events: StreamEvent[]) { - const body = `${events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join("")}data: [DONE]\n\n`; - res.writeHead(200, { - "content-type": "text/event-stream", - "cache-control": "no-store", - connection: "keep-alive", - "content-length": Buffer.byteLength(body), - }); - res.end(body); -} - -function isRemoteCompactionV2Request(input: ResponsesInputItem[]) { - // Codex sends compaction through /responses with a trigger item. Keep it - // outside scenario dispatch so maintenance calls never become tool evidence. - return input.some((item) => item.type === "compaction_trigger"); -} - -function buildRemoteCompactionV2Events(): [ - Extract, - Extract, -] { - const item = { - type: "compaction", - encrypted_content: "QA_MOCK_REMOTE_COMPACTION_SUMMARY", - }; - return [ - { type: "response.output_item.done", item }, - { - type: "response.completed", - response: { - id: "resp_mock_compaction_1", - status: "completed", - output: [item], - usage: { input_tokens: 64, output_tokens: 16, total_tokens: 80 }, - }, - }, - ]; -} - -async function writeSseWithPreviewPause( - res: ServerResponse, - events: StreamEvent[], - pauseMs: number, -) { - const completionIndex = events.findIndex((event) => event.type === "response.output_text.done"); - if (completionIndex < 0) { - writeSse(res, events); - return; - } - res.writeHead(200, { - "content-type": "text/event-stream", - "cache-control": "no-store", - connection: "keep-alive", - }); - for (const event of events.slice(0, completionIndex)) { - res.write(`data: ${JSON.stringify(event)}\n\n`); - } - await sleep(pauseMs); - for (const event of events.slice(completionIndex)) { - res.write(`data: ${JSON.stringify(event)}\n\n`); - } - res.end("data: [DONE]\n\n"); -} - -type AnthropicStreamEvent = Record & { - type: string; -}; - -function writeAnthropicSse(res: ServerResponse, events: AnthropicStreamEvent[]) { - const body = events - .map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`) - .join(""); - res.writeHead(200, { - "content-type": "text/event-stream", - "cache-control": "no-store", - connection: "keep-alive", - "content-length": Buffer.byteLength(body), - }); - res.end(body); -} - -function countApproxTokens(text: string) { - const trimmed = text.trim(); - if (!trimmed) { - return 0; - } - return Math.max(1, Math.ceil(trimmed.length / 4)); -} - -function extractEmbeddingInputTexts(input: unknown): string[] { - if (typeof input === "string") { - return [input]; - } - if (Array.isArray(input)) { - return input.flatMap((entry) => extractEmbeddingInputTexts(entry)); - } - if ( - input && - typeof input === "object" && - typeof (input as { text?: unknown }).text === "string" - ) { - return [(input as { text: string }).text]; - } - return []; -} - -function buildDeterministicEmbedding(text: string, dimensions = 16) { - const values = Array.from({ length: dimensions }, () => 0); - for (let index = 0; index < text.length; index += 1) { - const embeddingIndex = index % dimensions; - values[embeddingIndex] = (values[embeddingIndex] ?? 0) + text.charCodeAt(index) / 255; - } - const magnitude = Math.hypot(...values) || 1; - return values.map((value) => Number((value / magnitude).toFixed(8))); -} - -function extractLastUserText(input: ResponsesInputItem[]) { - for (const item of input.toReversed()) { - if (item.role !== "user" || !Array.isArray(item.content)) { - continue; - } - const text = extractInputText(item.content); - if (text && !isInternalRuntimeContextCarrierText(text)) { - return text; - } - } - return ""; -} - -function findLastUserIndex(input: ResponsesInputItem[]) { - return input.findLastIndex( - (item) => - item.role === "user" && - Array.isArray(item.content) && - !isInternalRuntimeContextCarrierText(extractInputText(item.content)), - ); -} - -function isInternalRuntimeContextCarrierText(text: string) { - const trimmed = text.trim(); - return ( - trimmed.includes(INTERNAL_RUNTIME_CONTEXT_BEGIN) && - trimmed.endsWith(INTERNAL_RUNTIME_CONTEXT_END) - ); -} - -function isToolOutputContinuationText(text: string) { - const trimmed = text.trim(); - if (!trimmed) { - return false; - } - return ( - /^(?:continue|keep going|resume|retry|carry on)(?:[.!?])?$/i.test(trimmed) || - /\b(?:continue|continuation|compaction|post-compaction|retry|resume)\b/i.test(trimmed) - ); -} - -function stringifyFunctionCallOutput(output: unknown): string { - if (typeof output === "string") { - return output; - } - if (Array.isArray(output)) { - return output - .map((entry) => { - if (typeof entry === "string") { - return entry; - } - if (!entry || typeof entry !== "object") { - return ""; - } - const record = entry as Record; - if (typeof record.text === "string") { - return record.text; - } - if (typeof record.output_text === "string") { - return record.output_text; - } - if (typeof record.content === "string") { - return record.content; - } - return ""; - }) - .filter(Boolean) - .join("\n"); - } - if (output && typeof output === "object") { - const record = output as Record; - if (typeof record.text === "string") { - return record.text; - } - if (typeof record.output_text === "string") { - return record.output_text; - } - if (typeof record.content === "string") { - return record.content; - } - try { - return JSON.stringify(output); - } catch { - return ""; - } - } - return ""; -} - -function extractFunctionCallOutputText(item: ResponsesInputItem) { - if (item.type !== "function_call_output") { - return ""; - } - return stringifyFunctionCallOutput(item.output); -} - -function extractFunctionCallOutputCallId(item: ResponsesInputItem) { - if (item.type !== "function_call_output") { - return ""; - } - const record = item as { - call_id?: unknown; - tool_call_id?: unknown; - tool_use_id?: unknown; - }; - return ( - [record.call_id, record.tool_call_id, record.tool_use_id].find( - (value): value is string => typeof value === "string" && value.trim().length > 0, - ) ?? "" - ); -} - -function functionCallOutputIsStructuredError(item: ResponsesInputItem) { - if (item.type !== "function_call_output") { - return false; - } - return item.is_error === true || item.isError === true; -} - -function extractToolOutput(input: ResponsesInputItem[]) { - const lastUserIndex = findLastUserIndex(input); - for (const item of input.slice(lastUserIndex + 1).toReversed()) { - const output = extractFunctionCallOutputText(item); - if (output) { - return output; - } - } - for (const [candidateIndex, candidateItem] of Array.from(input.entries()).toReversed()) { - const output = extractFunctionCallOutputText(candidateItem); - if (output) { - const laterUserTexts = input - .slice(candidateIndex + 1) - .filter((laterItem) => laterItem.role === "user" && Array.isArray(laterItem.content)) - .map((laterItem) => extractInputText(laterItem.content as unknown[])) - .filter(Boolean); - if ( - laterUserTexts.length > 0 && - laterUserTexts.every((text) => isToolOutputContinuationText(text)) - ) { - return output; - } - continue; - } - } - return ""; -} - -function extractToolOutputStructuredError(input: ResponsesInputItem[]) { - const lastUserIndex = findLastUserIndex(input); - for (const item of input.slice(lastUserIndex + 1).toReversed()) { - const output = extractFunctionCallOutputText(item); - if (output) { - return functionCallOutputIsStructuredError(item); - } - } - for (const [candidateIndex, candidateItem] of Array.from(input.entries()).toReversed()) { - const output = extractFunctionCallOutputText(candidateItem); - if (output) { - const laterUserTexts = input - .slice(candidateIndex + 1) - .filter((laterItem) => laterItem.role === "user" && Array.isArray(laterItem.content)) - .map((laterItem) => extractInputText(laterItem.content as unknown[])) - .filter(Boolean); - if ( - laterUserTexts.length > 0 && - laterUserTexts.every((text) => isToolOutputContinuationText(text)) - ) { - return functionCallOutputIsStructuredError(candidateItem); - } - } - } - return false; -} - -function extractToolOutputCallId(input: ResponsesInputItem[]) { - const lastUserIndex = findLastUserIndex(input); - for (const item of input.slice(lastUserIndex + 1).toReversed()) { - const output = extractFunctionCallOutputText(item); - if (output) { - return extractFunctionCallOutputCallId(item); - } - } - for (const [candidateIndex, candidateItem] of Array.from(input.entries()).toReversed()) { - const output = extractFunctionCallOutputText(candidateItem); - if (output) { - const laterUserTexts = input - .slice(candidateIndex + 1) - .filter((laterItem) => laterItem.role === "user" && Array.isArray(laterItem.content)) - .map((laterItem) => extractInputText(laterItem.content as unknown[])) - .filter(Boolean); - if ( - laterUserTexts.length > 0 && - laterUserTexts.every((text) => isToolOutputContinuationText(text)) - ) { - return extractFunctionCallOutputCallId(candidateItem); - } - } - } - return ""; -} - -function extractLatestToolOutput(input: ResponsesInputItem[]) { - for (const item of input.toReversed()) { - const output = extractFunctionCallOutputText(item); - if (output) { - return output; - } - } - return ""; -} - -function extractAllToolOutputText(input: ResponsesInputItem[]) { - return input - .map((item) => extractFunctionCallOutputText(item)) - .filter(Boolean) - .join("\n"); -} - -function extractUserTextAfterLatestToolOutput(input: ResponsesInputItem[]) { - const latestToolOutputIndex = input.findLastIndex((item) => - Boolean(extractFunctionCallOutputText(item)), - ); - if (latestToolOutputIndex < 0) { - return ""; - } - return input - .slice(latestToolOutputIndex + 1) - .filter((item) => item.role === "user" && Array.isArray(item.content)) - .map((item) => extractInputText(item.content as unknown[])) - .filter(Boolean) - .join("\n"); -} - -function extractInputText(content: unknown[]): string { - return content - .filter( - (entry): entry is { type: "input_text"; text: string } => - Boolean(entry) && - typeof entry === "object" && - (entry as { type?: unknown }).type === "input_text" && - typeof (entry as { text?: unknown }).text === "string", - ) - .map((entry) => entry.text) - .join("\n") - .trim(); -} - -function extractAllUserTexts(input: ResponsesInputItem[]) { - const texts: string[] = []; - for (const item of input) { - if (item.role !== "user" || !Array.isArray(item.content)) { - continue; - } - const text = extractInputText(item.content); - if (text) { - texts.push(text); - } - } - return texts; -} - -function extractSystemInputText(input: ResponsesInputItem[]) { - const texts: string[] = []; - for (const item of input) { - if (item.role !== "system") { - continue; - } - if (typeof item.content === "string" && item.content.trim()) { - texts.push(item.content.trim()); - continue; - } - if (!Array.isArray(item.content)) { - continue; - } - const text = extractInputText(item.content); - if (text) { - texts.push(text); - } - } - return texts.join("\n"); -} - -function extractAllInputTexts(input: ResponsesInputItem[]) { - const texts: string[] = []; - for (const item of input) { - if (typeof item.output === "string" && item.output.trim()) { - texts.push(item.output.trim()); - } - if (!Array.isArray(item.content)) { - continue; - } - const text = extractInputText(item.content); - if (text) { - texts.push(text); - } - } - return texts.join("\n"); -} - -function extractInstructionsText(body: Record) { - return typeof body.instructions === "string" ? body.instructions.trim() : ""; -} - -function extractAllRequestTexts(input: ResponsesInputItem[], body: Record) { - const texts: string[] = []; - const instructions = extractInstructionsText(body); - if (instructions) { - texts.push(instructions); - } - const inputText = extractAllInputTexts(input); - if (inputText) { - texts.push(inputText); - } - return texts.join("\n"); -} - -function buildWhatsAppPendingHistoryReply(allInputText: string) { - const triggerMatch = QA_WHATSAPP_PENDING_HISTORY_TRIGGER_MARKER_RE.exec(allInputText); - if (!triggerMatch?.[1]) { - return undefined; - } - const suffix = triggerMatch[1]; - const beforeTrigger = allInputText.slice(0, triggerMatch.index); - const priorGroupContext = extractStructuredWhatsAppPendingHistoryContext(beforeTrigger); - const quietMarkerPattern = new RegExp(`\\bWHATSAPP_QA_PENDING_HISTORY_QUIET_${suffix}\\b`, "u"); - const contextSentinelPattern = new RegExp( - `\\bWHATSAPP_QA_PENDING_HISTORY_CONTEXT_ONLY_${suffix}\\b`, - "u", - ); - if ( - !quietMarkerPattern.test(priorGroupContext) || - !contextSentinelPattern.test(priorGroupContext) - ) { - return "WHATSAPP_QA_PENDING_HISTORY_MISSING_CONTEXT"; - } - return `WHATSAPP_QA_PENDING_HISTORY_OK_${suffix}`; -} - -function extractStructuredWhatsAppPendingHistoryContext(beforeTrigger: string) { - const blockRe = new RegExp( - `${escapeRegExp(QA_WHATSAPP_PENDING_HISTORY_STRUCTURED_LABEL)}\\n((?:(?!\\n\\n)[\\s\\S])+)\\n\\n`, - "gu", - ); - return Array.from(beforeTrigger.matchAll(blockRe), (match) => match[1]?.trim()) - .filter((block): block is string => Boolean(block)) - .join("\n"); -} - -function buildWhatsAppBroadcastReply(allInputText: string) { - const promptMatch = QA_WHATSAPP_BROADCAST_PROMPT_RE.exec(allInputText); - const token = promptMatch?.[1]; - if (!token) { - return undefined; - } - const agentId = QA_WHATSAPP_RUNTIME_AGENT_RE.exec(allInputText)?.[1]; - if (agentId === "main") { - return `${token}_MAIN`; - } - if (agentId === "qa-second") { - return `${token}_SECOND`; - } - return "WHATSAPP_QA_BROADCAST_AGENT_CONTEXT_MISSING"; -} - -function buildWhatsAppGroupDispatchReply(allInputText: string) { - const activationMatch = QA_WHATSAPP_ACTIVATION_ALWAYS_MARKER_RE.exec(allInputText); - if (activationMatch?.[1]) { - return `WHATSAPP_QA_ACTIVATION_ALWAYS_${activationMatch[1]}`; - } - const triggerMatch = QA_WHATSAPP_REPLY_TO_BOT_TRIGGER_MARKER_RE.exec(allInputText); - if (triggerMatch?.[0]) { - return triggerMatch[0]; - } - return QA_WHATSAPP_REPLY_TO_BOT_SEED_MARKER_RE.exec(allInputText)?.[0]; -} - -function buildWhatsAppBatchedReply(allInputText: string) { - const finalMatch = QA_WHATSAPP_BATCHED_FINAL_MARKER_RE.exec(allInputText); - const suffix = finalMatch?.[1]; - if (!suffix) { - return undefined; - } - const firstMarker = `WHATSAPP_QA_BATCHED_FIRST_${suffix}`; - if (!allInputText.includes(firstMarker)) { - return `WHATSAPP_QA_BATCHED_MISSING_CONTEXT_${suffix}`; - } - return finalMatch[0]; -} - -function countImageInputs(value: unknown): number { - const seen = new WeakSet(); - const stack = [value]; - let count = 0; - let visited = 0; - while (stack.length > 0 && visited < 50_000) { - visited += 1; - const current = stack.pop(); - if (Array.isArray(current)) { - for (const entry of current) { - stack.push(entry); - } - continue; - } - if (!current || typeof current !== "object") { - continue; - } - if (seen.has(current)) { - continue; - } - seen.add(current); - const record = current as Record; - const type = typeof record.type === "string" ? record.type : ""; - if (type === "input_image" || type === "image" || type === "image_url" || type === "media") { - count += 1; - } - stack.push(record.content, record.image_url, record.source); - } - return count; -} - -function extractLatestImageUserTurn(input: ResponsesInputItem[]) { - const latestUserIndex = findLastUserIndex(input); - if (latestUserIndex < 0) { - return { text: "", imageInputCount: 0 }; - } - - const latestUserItem = input[latestUserIndex]; - if (!latestUserItem) { - return { text: "", imageInputCount: 0 }; - } - - const imageTurnItems = [latestUserItem]; - const imageInputCount = countImageInputs(imageTurnItems.map((item) => item.content)); - if (imageInputCount === 0) { - return { text: "", imageInputCount: 0 }; - } - return { - text: imageTurnItems - .map((item) => extractInputText(item.content as unknown[])) - .filter(Boolean) - .join("\n"), - imageInputCount, - }; -} - -function parseToolOutputJson(toolOutput: string): Record | null { - if (!toolOutput.trim()) { - return null; - } - try { - return JSON.parse(toolOutput) as Record; - } catch { - return null; - } -} - -function normalizePromptPathCandidate(candidate: string) { - const trimmed = candidate.trim().replace(/^`+|`+$/g, ""); - if (!trimmed) { - return null; - } - const normalized = trimmed.replace(/^\.\//, ""); - if ( - normalized.includes("/") || - /\.(?:md|json|ts|tsx|js|mjs|cjs|txt|yaml|yml)$/i.test(normalized) - ) { - return normalized; - } - return null; -} - -function readTargetFromPrompt(prompt: string) { - const backtickedMatches = Array.from(prompt.matchAll(/`([^`]+)`/g)) - .map((match) => normalizePromptPathCandidate(match[1] ?? "")) - .filter((value): value is string => Boolean(value)); - if (backtickedMatches.length > 0) { - return backtickedMatches[0]; - } - - const quotedMatches = Array.from(prompt.matchAll(/"([^"]+)"/g)) - .map((match) => normalizePromptPathCandidate(match[1] ?? "")) - .filter((value): value is string => Boolean(value)); - if (quotedMatches.length > 0) { - return quotedMatches[0]; - } - - const repoScoped = /\b(?:repo\/[^\s`",)]+|QA_[A-Z_]+\.md)\b/.exec(prompt)?.[0]?.trim(); - if (repoScoped) { - return repoScoped; - } - - const loosePath = /\b[A-Za-z0-9._-]+\.(?:md|json|ts|tsx|js|mjs|cjs|txt|yaml|yml)\b/i - .exec(prompt)?.[0] - ?.trim(); - if (loosePath) { - return loosePath; - } - - if (/\bdocs?\b/i.test(prompt)) { - return "repo/docs/help/testing.md"; - } - if (/\bscenario|kickoff|qa\b/i.test(prompt)) { - return "QA_KICKOFF_TASK.md"; - } - return "repo/package.json"; -} - -function execCommandFromToolProgressPrompt(prompt: string) { - return ( - /call the exec tool exactly once with this exact command before answering:\s*`([^`]+)`/i - .exec(prompt)?.[1] - ?.trim() || null - ); -} - -function buildMockFunctionCall(name: string, args: Record) { - const serialized = JSON.stringify(args); - const callSuffix = createHash("sha256") - .update(name) - .update("\0") - .update(serialized) - .digest("hex") - .slice(0, 10); - const sequence = ++mockFunctionCallSequence; - const uniqueSuffix = `${callSuffix}_${sequence}`; - const callId = `call_mock_${name}_${uniqueSuffix}`; - const itemId = `fc_mock_${name}_${uniqueSuffix}`; - const item = { - type: "function_call", - id: itemId, - call_id: callId, - name, - arguments: serialized, - }; - return { - callId, - item, - itemId, - responseId: `resp_mock_${name}_${uniqueSuffix}`, - serialized, - }; -} - -function buildToolCallEventsWithArgs(name: string, args: Record): StreamEvent[] { - const call = buildMockFunctionCall(name, args); - return [ - { - type: "response.output_item.added", - item: { - type: "function_call", - id: call.itemId, - call_id: call.callId, - name, - arguments: "", - }, - }, - { type: "response.function_call_arguments.delta", delta: call.serialized }, - { - type: "response.output_item.done", - item: call.item, - }, - { - type: "response.completed", - response: { - id: call.responseId, - status: "completed", - output: [call.item], - usage: { input_tokens: 64, output_tokens: 16, total_tokens: 80 }, - }, - }, - ]; -} - -function extractRememberedFact(userTexts: string[]) { - for (const text of userTexts) { - const qaCanaryMatch = /\bqa canary code is\s+([A-Za-z0-9-]+)/i.exec(text); - if (qaCanaryMatch?.[1]) { - return qaCanaryMatch[1]; - } - } - for (const text of userTexts) { - const match = /remember(?: this fact for later)?:\s*([A-Za-z0-9-]+)/i.exec(text); - if (match?.[1]) { - return match[1]; - } - } - return null; -} - -function extractOrbitCode(text: string) { - return /\bORBIT-\d+\b/i.exec(text)?.[0]?.toUpperCase() ?? null; -} - -function decodeXmlEntities(text: string) { - return text - .replaceAll("<", "<") - .replaceAll(">", ">") - .replaceAll("&", "&") - .replaceAll(""", '"') - .replaceAll("'", "'"); -} - -function extractActiveMemorySummary(text: string) { - const match = /\s*([\s\S]*?)\s*<\/active_memory_plugin>/i.exec(text); - return match?.[1] ? decodeXmlEntities(match[1]).trim() : null; -} - -function extractToolSearchTarget(text: string): string | null { - const match = /\btarget=([A-Za-z0-9_.:-]+)\b/.exec(text); - return match?.[1]?.trim() || null; -} - -function buildQaToolSearchArgs(targetTool: string, failureMode: boolean): Record { - if (failureMode && targetTool === "web_search") { - return { query: QA_LAB_WEB_SEARCH_DENIED_INPUT_QUERY }; - } - if (failureMode) { - return { __qaFailureMode: "denied-input" }; - } - if (targetTool === "exec") { - return { command: "echo runtime-tool-fixture", timeout: 5 }; - } - if (targetTool === "read") { - return { path: "QA_KICKOFF_TASK.md" }; - } - if (targetTool === "write") { - return { path: "runtime-tool-fixture-write.txt", content: "runtime tool fixture\n" }; - } - if (targetTool === "edit") { - return { - path: "runtime-tool-fixture-edit.txt", - edits: [{ oldText: "before edit\n", newText: "after edit\n" }], - }; - } - if (targetTool === "apply_patch") { - return { - input: [ - "*** Begin Patch", - "*** Add File: runtime-tool-fixture-patch.txt", - "+runtime patch", - "*** End Patch", - "", - ].join("\n"), - }; - } - if (targetTool === "web_search") { - return { query: "OpenClaw runtime parity fixed query", count: 1 }; - } - if (targetTool === "web_fetch") { - return { url: "https://example.com/", maxChars: 500 }; - } - if (targetTool === "image_generate") { - return { prompt: "QA lighthouse runtime parity fixture", filename: "runtime-tool-fixture" }; - } - if (targetTool === "tts") { - return { text: "Runtime parity voice fixture." }; - } - if (targetTool === "message") { - return { action: "send", message: "runtime parity message fixture" }; - } - if (targetTool === "session_status") { - return { sessionKey: "current" }; - } - if (targetTool === "sessions_spawn") { - return { - task: "Runtime tool fixture subagent: reply exactly RUNTIME-TOOL-FIXTURE.", - label: "runtime-tool-fixture", - mode: "run", - thread: false, - }; - } - if (targetTool === "memory_recall") { - return { query: "runtime parity memory fixture" }; - } - return { marker: "normal" }; -} - -function isActiveMemorySubagentPrompt(text: string) { - return text.includes("You are a memory search agent."); -} - -function extractSnackPreference(text: string) { - const normalized = text.replace(/\s+/g, " ").trim(); - const match = - /(lemon pepper wings(?:\s+with\s+blue cheese)?|blue cheese(?:\s+with\s+lemon pepper wings)?)/i.exec( - normalized, - ); - return match?.[0]?.trim() ?? null; -} - -function extractLastCapture(text: string, pattern: RegExp) { - let lastMatch: RegExpExecArray | null = null; - const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`; - const globalPattern = new RegExp(pattern.source, flags); - for (let match = globalPattern.exec(text); match; match = globalPattern.exec(text)) { - lastMatch = match; - } - return lastMatch?.[1]?.trim() || null; -} - -function extractCaptures(text: string, pattern: RegExp) { - const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`; - const globalPattern = new RegExp(pattern.source, flags); - return Array.from(text.matchAll(globalPattern), (match) => match[1]?.trim()).filter(Boolean); -} - -function extractLastMatchingUserText(texts: string[], pattern: RegExp) { - for (let index = texts.length - 1; index >= 0; index -= 1) { - const text = texts[index] ?? ""; - if (pattern.test(text)) { - return text; - } - } - return ""; -} - -function extractExactReplyDirective(text: string) { - const backtickedMatch = extractLastCapture(text, /reply(?: with)? exactly\s+`([^`]+)`/i); - if (backtickedMatch) { - return backtickedMatch; - } - return ( - extractLastCapture(text, /reply(?: with)? exactly:\s*([^\n]+)/i) ?? - extractLastCapture(text, /reply(?: with)? exactly\s+(?!with\b)([^\s`.,;:!?]+)/i) - ); -} - -function extractFinishExactlyDirective(text: string) { - const backtickedMatch = extractLastCapture(text, /finish with exactly\s+`([^`]+)`/i); - if (backtickedMatch) { - return backtickedMatch; - } - return extractLastCapture(text, /finish with exactly\s+([^\s`.,;:!?]+)/i); -} - -function extractExactMarkerDirective(text: string) { - const backtickedMatch = extractLastCapture(text, /exact marker\b[^:\n]{0,120}:\s*`([^`]+)`/i); - if (backtickedMatch) { - return backtickedMatch; - } - return extractLastCapture( - text, - /exact marker\b[^:\n]{0,120}:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i, - ); -} - -function extractWhatsAppLocationMarkerDirective(text: string) { - return extractLastCapture( - text, - /WhatsApp location marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i, - ); -} - -function extractWhatsAppContactMarkerDirective(text: string) { - return extractLastCapture(text, /WhatsApp contact marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i); -} - -function extractWhatsAppStickerMarkerDirective(text: string) { - return extractLastCapture(text, /WhatsApp sticker marker:\s*([^\s`.,;:!?]+(?:-[^\s`.,;:!?]+)*)/i); -} - -function shouldUseWhatsAppLocationMarker(prompt: string) { - return /(?:^|[\n:]\s*)šŸ“\s*37\.774900,\s*-122\.419400\b/u.test(prompt.trim()); -} - -function shouldUseWhatsAppContactMarker(prompt: string) { - return /(?:^|[\n:]\s*))/iu.test(prompt.trim()); -} - -function shouldUseWhatsAppStickerMarker(prompt: string) { - return /(?:^|[\n:]\s*)(?:\s|$)/iu.test(prompt.trim()); -} - -function extractLabeledMarkerDirective(text: string, label: string) { - const escapedLabel = escapeRegExp(label); - const backtickedMatch = extractLastCapture( - text, - new RegExp(`${escapedLabel}:\\s*\`([^\\\`]+)\``, "i"), - ); - if (backtickedMatch) { - return backtickedMatch; - } - return extractLastCapture( - text, - new RegExp(`${escapedLabel}:\\s*([^\\s\\\`.,;:!?]+(?:-[^\\s\\\`.,;:!?]+)*)`, "i"), - ); -} - -function extractBlockStreamingMarkerDirectives(text: string) { - const firstLabeledMarker = extractLabeledMarkerDirective(text, "first exact marker"); - const secondLabeledMarker = extractLabeledMarkerDirective(text, "second exact marker"); - if (firstLabeledMarker && secondLabeledMarker) { - return { - first: firstLabeledMarker, - second: secondLabeledMarker, - }; - } - - const markers = extractCaptures(text, /exact marker\b[^:\n]{0,120}:\s*`([^`]+)`/i); - if (markers.length < 2) { - return null; - } - const [first, second] = markers.slice(-2); - return first && second - ? { - first, - second, - } - : null; -} - -function extractQuotedToolArg(text: string, name: string) { - const escapedName = escapeRegExp(name); - return extractLastCapture(text, new RegExp(`\\b${escapedName}\\s*=\\s*"([^"]+)"`, "i")); -} - -function extractBareToolArg(text: string, name: string) { - const escapedName = escapeRegExp(name); - return extractLastCapture(text, new RegExp(`\\b${escapedName}\\s*=\\s*([^\\s\\\`.,;:!?]+)`, "i")); -} - -function hasDeclaredTool(body: Record, name: string) { - const tools = Array.isArray(body.tools) ? body.tools : []; - const dynamicTools = Array.isArray(body.dynamicTools) ? body.dynamicTools : []; - if ( - [...tools, ...dynamicTools].some((tool) => toolDefinitionMentionsName(tool, name)) || - instructionTextMentionsToolName(extractInstructionsText(body), name) - ) { - return true; - } - return false; -} - -function hasToolDefinition(body: Record, name: string) { - const tools = Array.isArray(body.tools) ? body.tools : []; - const dynamicTools = Array.isArray(body.dynamicTools) ? body.dynamicTools : []; - return [...tools, ...dynamicTools].some((tool) => toolDefinitionMentionsName(tool, name)); -} - -function toolDefinitionMentionsName(value: unknown, name: string, depth = 0): boolean { - if (depth > 6 || !value || typeof value !== "object") { - return false; - } - if (Array.isArray(value)) { - return value.some((item) => toolDefinitionMentionsName(item, name, depth + 1)); - } - const record = value as Record; - for (const key of ["name", "tool", "functionName"]) { - if (record[key] === name) { - return true; - } - } - return Object.values(record).some((item) => toolDefinitionMentionsName(item, name, depth + 1)); -} - -function instructionTextMentionsToolName(text: string, name: string) { - if (!text) { - return false; - } - const escapedName = escapeRegExp(name); - return new RegExp(`(^|[^A-Za-z0-9_])${escapedName}([^A-Za-z0-9_]|$)`).test(text); -} - -function isQaToolSearchFixture(text: string) { - return QA_TOOL_SEARCH_PROMPT_RE.test(text) || QA_TOOL_SEARCH_FAILURE_PROMPT_RE.test(text); -} - -function buildExplicitSessionsSpawnArgs(text: string): Record | null { - if (!/\bsessions_spawn\b/i.test(text)) { - return null; - } - const task = extractQuotedToolArg(text, "task"); - if (!task) { - return null; - } - const label = extractQuotedToolArg(text, "label") ?? extractBareToolArg(text, "label"); - const mode = extractBareToolArg(text, "mode")?.toLowerCase(); - const context = extractBareToolArg(text, "context")?.toLowerCase(); - return { - task, - ...(label ? { label } : {}), - ...(extractBareToolArg(text, "thread")?.toLowerCase() === "true" ? { thread: true } : {}), - ...(mode === "session" || mode === "run" ? { mode } : {}), - ...(context === "fork" || context === "isolated" ? { context } : {}), - }; -} - -function buildQaA2aMessageToolMirrorSessionsSendArgs(text: string): Record | null { - if (!QA_A2A_MESSAGE_TOOL_MIRROR_PROMPT_RE.test(text)) { - return null; - } - const sessionKey = - extractQuotedToolArg(text, "sessionKey") ?? extractBareToolArg(text, "sessionKey"); - if (!sessionKey) { - return null; - } - const marker = - extractExactMarkerDirective(text) ?? - extractExactReplyDirective(text) ?? - "QA-A2A-MESSAGE-TOOL-MIRROR-OK"; - return { - sessionKey, - message: `qa group visible reply tool check. Use the visible room reply path. exact marker: \`${marker}\``, - timeoutSeconds: 0, - }; -} - -function extractToolErrorForNamedCall(params: { - input: ResponsesInputItem[]; - name: string; - toolJson: Record | null; -}) { - const error = typeof params.toolJson?.error === "string" ? params.toolJson.error.trim() : ""; - if (!error) { - return undefined; - } - const namedFunctionCall = params.input.some( - (item) => item.type === "function_call" && item.name === params.name, - ); - if (namedFunctionCall) { - return error; - } - return undefined; -} - -function hasToolErrorOutput(toolJson: Record | null, toolOutput: string) { - if (typeof toolJson?.error === "string" && toolJson.error.trim()) { - return true; - } - if ( - typeof toolJson?.status === "string" && - /\b(?:error|failed|failure)\b/i.test(toolJson.status) - ) { - return true; - } - return /\b(?:error|failed|failure|not found|no such file|enoent)\b/i.test(toolOutput); -} - -function extractSessionStatusSessionKey( - toolJson: Record | null, - toolOutput: string, -) { - const details = toolJson?.details; - if (details && typeof details === "object") { - const sessionKey = (details as { sessionKey?: unknown }).sessionKey; - if (typeof sessionKey === "string" && sessionKey.trim()) { - return sessionKey.trim(); - } - } - const topLevelSessionKey = toolJson?.sessionKey; - if (typeof topLevelSessionKey === "string" && topLevelSessionKey.trim()) { - return topLevelSessionKey.trim(); - } - const statusLineSessionKey = /(?:^|\n)[^\n]*Session:\s*([^\sā€¢\n]+)/u.exec(toolOutput)?.[1]; - if (statusLineSessionKey?.trim()) { - return statusLineSessionKey.trim(); - } - return /"sessionKey"\s*:\s*"([^"]+)"/.exec(toolOutput)?.[1]?.trim() ?? ""; -} - -function isHeartbeatPrompt(text: string) { - const trimmed = text.trim(); - if (!trimmed || /remember this fact/i.test(trimmed)) { - return false; - } - return /(?:^|\n)Read HEARTBEAT\.md if it exists\b/i.test(trimmed); -} - -function readFirstMediaPath(value: unknown): string { - if (!value || typeof value !== "object" || Array.isArray(value)) { - return ""; - } - const media = value as { - mediaUrl?: unknown; - mediaUrls?: unknown; - path?: unknown; - filePath?: unknown; - attachments?: unknown; - }; - for (const candidate of [media.mediaUrl, media.path, media.filePath]) { - if (typeof candidate === "string" && candidate.trim()) { - return candidate.trim(); - } - } - if (Array.isArray(media.mediaUrls)) { - const mediaUrl = media.mediaUrls.find( - (candidate) => typeof candidate === "string" && candidate.trim(), - ); - if (typeof mediaUrl === "string" && mediaUrl.trim()) { - return mediaUrl.trim(); - } - } - if (Array.isArray(media.attachments)) { - for (const attachment of media.attachments) { - const mediaPath = readFirstMediaPath(attachment); - if (mediaPath) { - return mediaPath; - } - } - } - return ""; -} - -function buildAssistantText( - input: ResponsesInputItem[], - body: Record, - scenarioState: MockScenarioState, -) { - const prompt = extractLastUserText(input); - const toolOutput = extractToolOutput(input); - const scenarioToolOutput = - toolOutput || - (/thread memory check|session memory ranking check|memory tools check|repo contract followthrough check/i.test( - extractAllRequestTexts(input, body), - ) - ? extractLatestToolOutput(input) - : ""); - const toolJson = parseToolOutputJson(scenarioToolOutput); - const userTexts = extractAllUserTexts(input); - const allInputText = extractAllRequestTexts(input, body); - const rememberedFact = extractRememberedFact(userTexts); - const model = typeof body.model === "string" ? body.model : ""; - const memorySnippet = - typeof toolJson?.text === "string" - ? toolJson.text - : Array.isArray(toolJson?.results) - ? JSON.stringify(toolJson.results) - : scenarioToolOutput; - const orbitCode = extractOrbitCode(memorySnippet) ?? extractOrbitCode(allInputText); - const mediaPath = - typeof toolJson?.details === "object" && - toolJson.details !== null && - !Array.isArray(toolJson.details) - ? readFirstMediaPath((toolJson.details as { media?: unknown }).media) - : ""; - const promptExactReplyDirective = extractExactReplyDirective(prompt); - const promptExactMarkerDirective = extractExactMarkerDirective(prompt); - const exactReplyDirective = promptExactReplyDirective ?? extractExactReplyDirective(allInputText); - const exactMarkerDirective = - promptExactMarkerDirective ?? extractExactMarkerDirective(allInputText); - const whatsAppLocationMarker = shouldUseWhatsAppLocationMarker(prompt) - ? extractWhatsAppLocationMarkerDirective(allInputText) - : ""; - const whatsAppContactMarker = shouldUseWhatsAppContactMarker(prompt) - ? extractWhatsAppContactMarkerDirective(allInputText) - : ""; - const whatsAppStickerMarker = shouldUseWhatsAppStickerMarker(prompt) - ? extractWhatsAppStickerMarkerDirective(allInputText) - : ""; - const finishExactlyDirective = - extractFinishExactlyDirective(prompt) ?? extractFinishExactlyDirective(allInputText); - const latestImageUserTurn = extractLatestImageUserTurn(input); - const activeMemorySummary = extractActiveMemorySummary(allInputText); - const snackPreference = extractSnackPreference(activeMemorySummary ?? memorySnippet); - const sessionsSpawnError = extractToolErrorForNamedCall({ - input, - name: "sessions_spawn", - toolJson, - }); - - if (/what was the qa canary code/i.test(prompt) && rememberedFact) { - return `Protocol note: the QA canary code was ${rememberedFact}.`; - } - if (sessionsSpawnError) { - return `Protocol note: sessions_spawn failed: ${sessionsSpawnError}`; - } - if (/remember this fact/i.test(prompt) && exactReplyDirective) { - return exactReplyDirective; - } - if (/remember this fact/i.test(prompt) && rememberedFact) { - return `Protocol note: acknowledged. I will remember ${rememberedFact}.`; - } - if (/memory unavailable check/i.test(prompt)) { - return "Protocol note: I checked the available runtime context but could not confirm the hidden memory-only fact, so I will not guess."; - } - if (isHeartbeatPrompt(prompt)) { - return "HEARTBEAT_OK"; - } - if ( - /roundtrip image inspection check/i.test(latestImageUserTurn.text) && - latestImageUserTurn.imageInputCount > 0 - ) { - return "Protocol note: the generated attachment shows the same QA lighthouse scene from the previous step."; - } - if ( - /image understanding check/i.test(latestImageUserTurn.text) && - latestImageUserTurn.imageInputCount > 0 - ) { - return "Protocol note: the attached image is split horizontally, with red on top and blue on the bottom."; - } - if (whatsAppLocationMarker) { - return whatsAppLocationMarker; - } - if (whatsAppContactMarker) { - return whatsAppContactMarker; - } - if (whatsAppStickerMarker) { - return whatsAppStickerMarker; - } - if (/\bmarker\b/i.test(prompt) && promptExactMarkerDirective) { - return promptExactMarkerDirective; - } - if (/\bmarker\b/i.test(prompt) && promptExactReplyDirective) { - return promptExactReplyDirective; - } - if (/\bmarker\b/i.test(allInputText) && promptExactReplyDirective) { - return promptExactReplyDirective; - } - if (/\bmarker\b/i.test(allInputText) && exactMarkerDirective) { - return exactMarkerDirective; - } - if (/\bmarker\b/i.test(allInputText) && exactReplyDirective) { - return exactReplyDirective; - } - if (promptExactReplyDirective) { - return promptExactReplyDirective; - } - if (/visible skill marker/i.test(prompt)) { - return "VISIBLE-SKILL-OK"; - } - if (/hot install marker/i.test(prompt)) { - return "HOT-INSTALL-OK"; - } - if (/memory tools check/i.test(prompt) && orbitCode) { - return `Protocol note: I checked memory and the project codename is ${orbitCode}.`; - } - if (/silent snack recall check/i.test(prompt) && snackPreference) { - return `Protocol note: you usually want ${snackPreference} for QA movie night.`; - } - if (/silent snack recall check/i.test(prompt)) { - return "Protocol note: I do not have enough context to say what you usually want for QA movie night."; - } - if (/qa private final reply warning check/i.test(prompt)) { - return [ - "QA-STRANDED-85714 confirms this is a substantive private final reply that intentionally stays outside the message tool path for the warning check.", - "The response is long enough to exercise message_tool_only private-final detection while remaining private to the agent transcript.", - ].join(" "); - } - if (isStrandedFinalRetryFailureRequest(allInputText)) { - return buildStrandedFinalRetryFailureText(); - } - if (QA_STRANDED_FINAL_RECOVERY_PROMPT_RE.test(allInputText)) { - return QA_STRANDED_FINAL_RETRY_PROMPT_RE.test(allInputText) - ? "QA-STRANDED-85714" - : buildStrandedFinalRecoveryText(); - } - if (/tool continuity check/i.test(prompt) && toolOutput) { - return `Protocol note: model switch handoff confirmed on ${model || "the requested model"}. QA mission from QA_KICKOFF_TASK.md still applies: understand this OpenClaw repo from source + docs before acting.`; - } - if (toolOutput && promptExactReplyDirective) { - return promptExactReplyDirective; - } - if ((toolOutput || allInputText) && /repo contract followthrough check/i.test(allInputText)) { - const repoEvidenceText = [scenarioToolOutput, allInputText].filter(Boolean).join("\n"); - if ( - /successfully (?:wrote|created|updated|replaced)/i.test(repoEvidenceText) || - /status:\s*complete/i.test(repoEvidenceText) - ) { - return [ - "Read: AGENT.md, SOUL.md, FOLLOWTHROUGH_INPUT.md", - "Wrote: repo-contract-summary.txt", - "Status: complete", - ].join("\n"); - } - return [ - "Read: AGENT.md, SOUL.md, FOLLOWTHROUGH_INPUT.md", - "Wrote: repo-contract-summary.txt", - "Status: blocked", - ].join("\n"); - } - if (toolOutput && /personal task followthrough check/i.test(allInputText)) { - const taskEvidenceText = scenarioToolOutput; - if (/successfully (?:wrote|created|updated|replaced)/i.test(taskEvidenceText)) { - return [ - "Pending: maintainer feedback before publishing", - "Blocked: publishing needs explicit user approval", - "Done: local evidence captured in personal-task-status.txt", - ].join("\n"); - } - return [ - "Pending: maintainer feedback before publishing", - "Blocked: publishing needs explicit user approval", - "Done: blocked until personal-task-status.txt exists", - ].join("\n"); - } - if (/session memory ranking check/i.test(prompt) && orbitCode) { - return `Protocol note: I checked memory and the current Project Nebula codename is ${orbitCode}.`; - } - if (/thread memory check/i.test(allInputText) && orbitCode) { - return `Protocol note: I checked memory in-thread and the hidden thread codename is ${orbitCode}.`; - } - if (/switch(?:ing)? models?/i.test(prompt)) { - return `Protocol note: model switch acknowledged. Continuing on ${model || "the requested model"}.`; - } - if (QA_IMAGE_GENERATION_PROMPT_RE.test(allInputText) && mediaPath) { - return `Protocol note: generated the QA lighthouse image successfully. Attachment: ${mediaPath}`; - } - if (QA_SKILL_WORKSHOP_GIF_PROMPT_RE.test(prompt) && toolOutput) { - return [ - "Animated GIF QA checklist ready.", - "- Confirm true animation, not a static preview.", - "- Verify dimensions and product UI fit.", - "- Record attribution and license.", - "- Keep a local copy before using the asset.", - "- Re-open the copied file for final verification.", - ].join("\n"); - } - if ( - /interrupted by a gateway reload/i.test(prompt) && - /subagent recovery worker/i.test(allInputText) - ) { - return "RECOVERED-SUBAGENT-OK"; - } - if (/subagent recovery worker/i.test(prompt)) { - return "RECOVERED-SUBAGENT-OK"; - } - if (/fanout worker alpha/i.test(prompt)) { - return "ALPHA-OK"; - } - if (/fanout worker beta/i.test(prompt)) { - return "BETA-OK"; - } - if (QA_SUBAGENT_DIRECT_FALLBACK_WORKER_RE.test(prompt)) { - return QA_SUBAGENT_DIRECT_FALLBACK_MARKER; - } - if (/report the visible code/i.test(prompt) && /FORKED-CONTEXT-ALPHA/i.test(allInputText)) { - return "FORKED-CONTEXT-ALPHA"; - } - const fanoutCompleteReply = "subagent-1: ok\nsubagent-2: ok"; - if (scenarioState.subagentFanoutPhase === 2 && prompt) { - scenarioState.subagentFanoutPhase = 3; - return fanoutCompleteReply; - } - if ( - /forked subagent context qa check/i.test(prompt) && - /FORKED-CONTEXT-ALPHA/i.test(allInputText) - ) { - return [ - "Worked", - "- FORKED-CONTEXT-ALPHA", - "Evidence", - "- The forked child recovered the visible code from requester transcript context.", - "Blocked", - "- None.", - ].join("\n"); - } - if ( - toolOutput && - (/delegate (?:one |a )bounded qa task/i.test(allInputText) || - /subagent handoff/i.test(allInputText)) - ) { - const compact = toolOutput.replace(/\s+/g, " ").trim() || "no delegated output"; - return `Delegated task:\n- Inspect the QA workspace via a bounded subagent.\nResult:\n- ${compact}\nEvidence:\n- The child result was folded back into the main thread exactly once.`; - } - if (toolOutput && /worked, failed, blocked|worked\/failed\/blocked|follow-up/i.test(prompt)) { - return `Worked:\n- Read seeded QA material.\n- Expanded the report structure.\nFailed:\n- None observed in mock mode.\nBlocked:\n- No live provider evidence in this lane.\nFollow-up:\n- Re-run with a real model for qualitative coverage.`; - } - if (toolOutput && /lobster invaders/i.test(prompt)) { - if (toolOutput.includes("QA mission") || toolOutput.includes("Testing")) { - return ""; - } - return `Protocol note: Lobster Invaders built at lobster-invaders.html.`; - } - if ( - toolOutput && - (/compaction retry mutating tool check/i.test(allInputText) || - /compaction-retry-summary\.txt/i.test(toolOutput)) - ) { - if ( - toolOutput.includes("Replay safety: unsafe after write.") || - /compaction-retry-summary\.txt/i.test(toolOutput) || - /successfully (?:wrote|replaced)/i.test(toolOutput) || - /\bwrote\b.*\bcompaction-retry-summary\.txt\b/i.test(toolOutput) - ) { - return "Protocol note: replay unsafe after write."; - } - return ""; - } - if ( - toolOutput && - (QA_TOOL_SEARCH_PROMPT_RE.test(allInputText) || - QA_TOOL_SEARCH_FAILURE_PROMPT_RE.test(allInputText)) - ) { - const targetTool = extractToolSearchTarget(allInputText); - if (targetTool && toolOutput.includes(targetTool) && toolOutput.includes("FAKE_PLUGIN_OK")) { - return `FAKE_PLUGIN_OK ${targetTool}`; - } - } - if ( - toolOutput && - /(worked, failed, blocked|worked\/failed\/blocked|source and docs)/i.test(allInputText) - ) { - return [ - "Worked:", - "- Read all three seeded files: repo/qa/scenarios/index.yaml, repo/extensions/qa-lab/src/suite.ts, and repo/docs/help/testing.md.", - "- Extra QA scenario candidates: config restart capability flip and image generation roundtrip.", - "Failed:", - "- None observed in mock mode.", - "Blocked:", - "- No live provider evidence in this lane.", - "Follow-up:", - "- Re-run with a real model for qualitative coverage.", - ].join("\n"); - } - if (toolOutput) { - const snippet = truncateUtf16Safe(toolOutput.replace(/\s+/g, " ").trim(), 220); - return `Protocol note: I reviewed the requested material. Evidence snippet: ${snippet || "no content"}`; - } - if (finishExactlyDirective) { - return finishExactlyDirective; - } - if (prompt) { - return `Protocol note: acknowledged. Continue with the QA scenario plan and report worked, failed, and blocked items.`; - } - return "Protocol note: mock OpenAI server ready."; -} - -function buildToolCallEvents(prompt: string): StreamEvent[] { - const targetPath = readTargetFromPrompt(prompt); - return buildToolCallEventsWithArgs("read", { path: targetPath }); -} - -function buildReleaseAuditJson() { - return `${JSON.stringify( - { - verified: false, - findings: [ - { - id: "REL-GATEWAY-417", - source: "src/gateway/reconnect.ts", - status: "retry jitter verified, resume token fallback still needs manual spot check", - verified: true, - }, - { - id: "REL-CHANNEL-238", - source: "src/channels/delivery.ts", - status: "thread replies preserve ordering, root-channel fallback needs handoff note", - verified: true, - }, - { - id: "REL-CRON-904", - source: "src/scheduling/cron.ts", - status: "single-run lock verified for restart wakeups", - verified: true, - }, - { - id: "REL-MEMORY-552", - source: "src/memory/recall.ts", - status: - "fallback summary survives empty memory search; ranking sample needs second reviewer", - verified: true, - }, - { - id: "REL-PLUGIN-319", - source: "src/plugins/runtime.ts", - status: "bundled runtime manifest loads cleanly after restart", - verified: true, - }, - { - id: "REL-INSTALL-846", - source: "install/update.ts", - status: "update smoke passed from previous stable tag", - verified: true, - }, - { - id: "REL-DOCS-611", - source: "docs/operator-notes.md", - status: - "docs mention reconnect, cron, memory, plugin, and installer checks; channel ordering and UI notes need maintainer handoff", - verified: true, - }, - { - id: "REL-UI-BLOCKED", - source: "ui/control-panel.ts", - status: "blocked: source file was referenced by checklist but missing from the fixture", - verified: false, - }, - ], - }, - null, - 2, - )}\n`; -} - -function buildReleaseHandoffMarkdown() { - return [ - "# Release Handoff", - "", - "Ready:", - "- REL-GATEWAY-417: gateway reconnect handling checked in `src/gateway/reconnect.ts`.", - "- REL-CRON-904: cron duplicate prevention checked in `src/scheduling/cron.ts`.", - "- REL-PLUGIN-319: plugin runtime loading checked in `src/plugins/runtime.ts`.", - "- REL-INSTALL-846: installer update path checked in `install/update.ts`.", - "", - "Follow-up:", - "- REL-CHANNEL-238: channel delivery ordering needs maintainer handoff.", - "- REL-MEMORY-552: memory recall fallback ranking sample needs a second reviewer.", - "- REL-DOCS-611: docs update status needs channel ordering and UI notes.", - "- `ui/control-panel.ts` is blocked/not found in the fixture.", - "", - ].join("\n"); -} - -function extractPlannedToolName(events: StreamEvent[]) { - for (const event of events) { - if (event.type !== "response.output_item.done") { - continue; - } - const item = event.item as { type?: unknown; name?: unknown }; - if (item.type === "function_call" && typeof item.name === "string") { - return item.name; - } - } - return undefined; -} - -function extractPlannedToolCallId(events: StreamEvent[]) { - for (const event of events) { - if (event.type !== "response.output_item.done") { - continue; - } - const item = event.item as { type?: unknown; call_id?: unknown }; - if (item.type === "function_call" && typeof item.call_id === "string") { - return item.call_id; - } - } - return undefined; -} - -function extractPlannedToolArgs(events: StreamEvent[]) { - for (const event of events) { - if (event.type !== "response.output_item.done") { - continue; - } - const item = event.item as { type?: unknown; arguments?: unknown }; - if (item.type !== "function_call" || typeof item.arguments !== "string") { - continue; - } - try { - const parsed = JSON.parse(item.arguments); - return parsed && typeof parsed === "object" ? (parsed as Record) : undefined; - } catch { - return undefined; - } - } - return undefined; -} - -type MockAssistantMessageSpec = { - id: string; - phase?: "commentary" | "final_answer"; - streamDeltas?: string[]; - text: string; -}; - -function splitMockStreamingText(text: string, parts = 3) { - if (text.length <= 1) { - return [text]; - } - const chunkSize = Math.max(1, Math.ceil(text.length / parts)); - const chunks: string[] = []; - for (let index = 0; index < text.length; index += chunkSize) { - chunks.push(text.slice(index, index + chunkSize)); - } - return chunks.length > 1 ? chunks : [text.slice(0, 1), text.slice(1)]; -} - -function buildQaLongFinalText({ - endMarker = "TELEGRAM-LONG-FINAL-END", - segmentPrefix = "telegram-long-final-segment", - segmentCount = 42, - startMarker = "TELEGRAM-LONG-FINAL-BEGIN", -}: { - endMarker?: string; - segmentPrefix?: string; - segmentCount?: number; - startMarker?: string; -} = {}) { - const body = Array.from( - { length: segmentCount }, - (_, index) => `${segmentPrefix}-${String(index + 1).padStart(3, "0")} ${"x".repeat(54)}`, - ).join("\n"); - return `${startMarker}\n${body}\n${endMarker}`; -} - -function buildAssistantOutputItem(spec: MockAssistantMessageSpec) { - return { - type: "message", - id: spec.id, - role: "assistant", - status: "completed", - ...(spec.phase ? { phase: spec.phase } : {}), - content: [{ type: "output_text", text: spec.text, annotations: [] }], - } as const; -} - -function appendAssistantMessageEvents(events: StreamEvent[], spec: MockAssistantMessageSpec) { - events.push({ - type: "response.output_item.added", - item: { - type: "message", - id: spec.id, - role: "assistant", - ...(spec.phase ? { phase: spec.phase } : {}), - content: [], - status: "in_progress", - }, - }); - for (const delta of spec.streamDeltas ?? []) { - events.push({ - type: "response.output_text.delta", - item_id: spec.id, - output_index: 0, - content_index: 0, - delta, - }); - } - if ((spec.streamDeltas ?? []).length > 0) { - events.push({ - type: "response.output_text.done", - item_id: spec.id, - output_index: 0, - content_index: 0, - text: spec.text, - }); - } - events.push({ - type: "response.output_item.done", - item: buildAssistantOutputItem(spec), - }); -} - -function buildAssistantThenToolCallEvents( - spec: MockAssistantMessageSpec, - name: string, - args: Record, -): StreamEvent[] { - const call = buildMockFunctionCall(name, args); - const message = buildAssistantOutputItem(spec); - const events: StreamEvent[] = []; - appendAssistantMessageEvents(events, spec); - events.push({ - type: "response.output_item.added", - item: { - type: "function_call", - id: call.itemId, - call_id: call.callId, - name, - arguments: "", - }, - }); - events.push({ type: "response.function_call_arguments.delta", delta: call.serialized }); - events.push({ - type: "response.output_item.done", - item: call.item, - }); - events.push({ - type: "response.completed", - response: { - id: call.responseId, - status: "completed", - output: [message, call.item], - usage: { input_tokens: 64, output_tokens: 32, total_tokens: 96 }, - }, - }); - return events; -} - -function buildAssistantEvents(specsOrText: MockAssistantMessageSpec[] | string): StreamEvent[] { - const specs = - typeof specsOrText === "string" - ? [ - { - id: "msg_mock_1", - text: specsOrText, - }, - ] - : specsOrText; - const renderedSpecs = specs.map((spec) => ({ spec, item: buildAssistantOutputItem(spec) })); - const output = renderedSpecs.map(({ item }) => item); - const events: StreamEvent[] = []; - - for (const [outputIndex, { spec, item }] of renderedSpecs.entries()) { - events.push({ - type: "response.output_item.added", - item: { - type: "message", - id: spec.id, - role: "assistant", - ...(spec.phase ? { phase: spec.phase } : {}), - content: [], - status: "in_progress", - }, - }); - for (const delta of spec.streamDeltas ?? []) { - events.push({ - type: "response.output_text.delta", - item_id: spec.id, - output_index: outputIndex, - content_index: 0, - delta, - }); - } - if ((spec.streamDeltas ?? []).length > 0) { - events.push({ - type: "response.output_text.done", - item_id: spec.id, - output_index: outputIndex, - content_index: 0, - text: spec.text, - }); - } - events.push({ - type: "response.output_item.done", - item, - }); - } - - events.push({ - type: "response.completed", - response: { - id: "resp_mock_msg_1", - status: "completed", - output, - usage: { input_tokens: 64, output_tokens: 24, total_tokens: 88 }, - }, - }); - return events; -} - -function buildReasoningOnlyEvents(summaryText: string, id: string): StreamEvent[] { - const reasoningItem = { - type: "reasoning", - id, - summary: [{ text: summaryText }], - } as const; - return [ - { - type: "response.output_item.added", - item: { - type: "reasoning", - id, - summary: [], - }, - }, - { - type: "response.output_item.done", - item: reasoningItem, - }, - { - type: "response.completed", - response: { - id: `resp_${id}`, - status: "completed", - output: [reasoningItem], - usage: { input_tokens: 64, output_tokens: 8, total_tokens: 72 }, - }, - }, - ]; -} - -function buildReasoningAndAssistantEvents(params: { - reasoningId: string; - answerText: string; - answerId?: string; -}): StreamEvent[] { - const reasoningItem = { - type: "reasoning", - id: params.reasoningId, - summary: [], - } as const; - const answerItem = buildAssistantOutputItem({ - id: params.answerId ?? "msg_mock_reasoned_answer", - phase: "final_answer", - text: params.answerText, - }); - return [ - { - type: "response.output_item.added", - item: { - type: "reasoning", - id: params.reasoningId, - summary: [], - }, - }, - { - type: "response.output_item.done", - item: reasoningItem, - }, - { - type: "response.output_item.added", - item: { - type: "message", - id: answerItem.id, - role: "assistant", - phase: "final_answer", - content: [], - status: "in_progress", - }, - }, - { - type: "response.output_text.delta", - item_id: answerItem.id, - output_index: 1, - content_index: 0, - delta: params.answerText, - }, - { - type: "response.output_text.done", - item_id: answerItem.id, - output_index: 1, - content_index: 0, - text: params.answerText, - }, - { - type: "response.output_item.done", - item: answerItem, - }, - { - type: "response.completed", - response: { - id: `resp_${params.reasoningId}`, - status: "completed", - output: [reasoningItem, answerItem], - usage: { input_tokens: 64, output_tokens: 16, total_tokens: 80 }, - }, - }, - ]; -} +import { buildMessagesPayload } from "./mock-anthropic-messages.js"; +import { buildAssistantText } from "./mock-openai-assistant-text.js"; +import { + type ResponsesInputItem, + type StreamEvent, + resolveProviderVariant, + type MockOpenAiRequestSnapshot, + type MockOpenAiRequestSnapshotInput, + type AnthropicMessagesRequest, + TINY_PNG_BASE64, + QA_REASONING_ONLY_RECOVERY_PROMPT_RE, + QA_REASONING_ONLY_SIDE_EFFECT_PROMPT_RE, + QA_THINKING_VISIBILITY_OFF_PROMPT_RE, + QA_THINKING_VISIBILITY_MAX_PROMPT_RE, + QA_EMPTY_RESPONSE_RECOVERY_PROMPT_RE, + QA_EMPTY_RESPONSE_EXHAUSTION_PROMPT_RE, + QA_STREAMING_PROMPT_RE, + QA_FINAL_ONLY_MARKER_STREAMING_PROMPT_RE, + QA_BLOCK_STREAMING_PROMPT_RE, + QA_TOOL_PROGRESS_ERROR_PROMPT_RE, + QA_TOOL_PROGRESS_PROMPT_RE, + QA_GROUP_VISIBLE_REPLY_TOOL_PROMPT_RE, + QA_A2A_MESSAGE_TOOL_MIRROR_PROMPT_RE, + QA_GROUP_MESSAGE_UNAVAILABLE_FALLBACK_PROMPT_RE, + QA_STRANDED_FINAL_RECOVERY_PROMPT_RE, + QA_STRANDED_FINAL_RETRY_PROMPT_RE, + QA_TELEGRAM_CURRENT_SESSION_STATUS_PROMPT_RE, + QA_TELEGRAM_STREAM_SINGLE_MARKER, + QA_TELEGRAM_LONG_FINAL_THREE_CHUNK_PROMPT_RE, + QA_TELEGRAM_LONG_FINAL_PROMPT_RE, + QA_WHATSAPP_LONG_FINAL_PROMPT_RE, + QA_SLACK_CHART_PRESENTATION_PROMPT_RE, + QA_WHATSAPP_AGENT_MESSAGE_ACTION_REACT_PROMPT_RE, + QA_WHATSAPP_AGENT_MESSAGE_ACTION_UPLOAD_PROMPT_RE, + QA_SUBAGENT_DIRECT_FALLBACK_PROMPT_RE, + buildStrandedFinalRecoveryText, + buildStrandedFinalRetryFailureText, + isStrandedFinalRetryFailureRequest, + QA_SUBAGENT_DIRECT_FALLBACK_MARKER, + QA_NATIVE_STOP_DELAY_PROMPT_RE, + QA_NATIVE_STOP_DELAY_MS, + QA_IMAGE_GENERATION_PROMPT_RE, + QA_REASONING_ONLY_RETRY_NEEDLE, + QA_EMPTY_RESPONSE_RETRY_NEEDLE, + QA_SKILL_WORKSHOP_GIF_PROMPT_RE, + QA_SKILL_WORKSHOP_REVIEW_PROMPT_RE, + QA_RELEASE_AUDIT_PROMPT_RE, + QA_TOOL_SEARCH_PROMPT_RE, + QA_TOOL_SEARCH_FAILURE_PROMPT_RE, + QA_MCP_CODE_MODE_PROMPT_RE, + QA_RESTART_CODE_MODE_WAIT_PROMPT_RE, + QA_RESTART_RECOVERY_PROMPT_RE, + QA_MCP_CODE_MODE_API_FILE_PROMPT_RE, + type MockScenarioState, + sourceDiscoveryReadPathForProvider, + subagentHandoffTaskForProvider, + subagentFanoutTaskForProvider, + MOCK_OPENAI_DEBUG_REQUEST_LIMIT, + readBody, + parseJsonObjectBody, + writeOpenAiMalformedJsonError, + transcriptionTextForAudioRequest, + writeSse, + isRemoteCompactionV2Request, + buildRemoteCompactionV2Events, + writeSseWithPreviewPause, + writeAnthropicSse, + countApproxTokens, + extractEmbeddingInputTexts, + buildDeterministicEmbedding, +} from "./mock-openai-contracts.js"; +import { + extractLastMatchingUserText, + extractExactReplyDirective, + extractExactMarkerDirective, + extractWhatsAppLocationMarkerDirective, + extractWhatsAppContactMarkerDirective, + extractWhatsAppStickerMarkerDirective, + shouldUseWhatsAppLocationMarker, + shouldUseWhatsAppContactMarker, + shouldUseWhatsAppStickerMarker, + extractBlockStreamingMarkerDirectives, + hasDeclaredTool, + hasToolDefinition, + isQaToolSearchFixture, + buildExplicitSessionsSpawnArgs, + buildQaA2aMessageToolMirrorSessionsSendArgs, + hasToolErrorOutput, + extractSessionStatusSessionKey, + isHeartbeatPrompt, +} from "./mock-openai-directives.js"; +import { + buildToolCallEvents, + buildReleaseAuditJson, + buildReleaseHandoffMarkdown, + extractPlannedToolName, + extractPlannedToolCallId, + extractPlannedToolArgs, + splitMockStreamingText, + buildQaLongFinalText, + buildAssistantThenToolCallEvents, + buildAssistantEvents, + buildReasoningOnlyEvents, + buildReasoningAndAssistantEvents, +} from "./mock-openai-events.js"; +import { + extractLastUserText, + extractToolOutput, + extractToolOutputStructuredError, + extractToolOutputCallId, + extractLatestToolOutput, + extractAllToolOutputText, + extractUserTextAfterLatestToolOutput, + extractAllUserTexts, + extractSystemInputText, + extractAllInputTexts, + extractInstructionsText, + extractAllRequestTexts, + buildWhatsAppPendingHistoryReply, + buildWhatsAppBroadcastReply, + buildWhatsAppGroupDispatchReply, + buildWhatsAppBatchedReply, + countImageInputs, + extractLatestImageUserTurn, + parseToolOutputJson, +} from "./mock-openai-input.js"; +import { + readTargetFromPrompt, + execCommandFromToolProgressPrompt, + buildToolCallEventsWithArgs, + extractOrbitCode, + extractToolSearchTarget, + buildQaToolSearchArgs, + isActiveMemorySubagentPrompt, + extractSnackPreference, +} from "./mock-openai-tooling.js"; async function buildResponsesPayload( body: Record, @@ -3293,503 +1271,6 @@ async function buildResponsesPayload( return buildAssistantEvents(buildAssistantText(input, body, scenarioState)); } -// --------------------------------------------------------------------------- -// Anthropic /v1/messages adapter -// --------------------------------------------------------------------------- -// -// The QA parity gate needs two comparable scenario runs: one against the -// "candidate" (openai/gpt-5.6-luna) and one against the "baseline" -// (anthropic/claude-opus-4-8). The OpenAI mock above already dispatches all -// the scenario prompt branches we care about. Rather than duplicating that -// machinery, the /v1/messages route below translates Anthropic request -// shapes into the shared ResponsesInputItem[] format, calls the same -// buildResponsesPayload() dispatcher, and then re-serializes the resulting -// events into an Anthropic response. This gives the parity harness a -// baseline lane that exercises the same scenario logic and selected -// provider-specific plans without requiring real Anthropic API keys. -// -// Scope: handles Anthropic Messages requests with text and tool_result -// content blocks, supporting both non-streaming JSON responses and the -// streaming SSE path used by the parity harness. - -function normalizeAnthropicSystemToString( - system: AnthropicMessagesRequest["system"], -): string | undefined { - if (typeof system === "string") { - return system.trim() || undefined; - } - if (Array.isArray(system)) { - const joined = system - .map((block) => (block?.type === "text" ? block.text : "")) - .filter(Boolean) - .join("\n") - .trim(); - return joined || undefined; - } - return undefined; -} - -function stringifyToolResultContent( - content: Extract["content"], -): string { - if (typeof content === "string") { - return content; - } - if (Array.isArray(content)) { - return content - .map((block) => (block?.type === "text" ? block.text : "")) - .filter(Boolean) - .join("\n"); - } - return ""; -} - -function convertAnthropicMessagesToResponsesInput(params: { - system?: AnthropicMessagesRequest["system"]; - messages: AnthropicMessage[]; -}): ResponsesInputItem[] { - const items: ResponsesInputItem[] = []; - const systemText = normalizeAnthropicSystemToString(params.system); - if (systemText) { - items.push({ - role: "system", - content: [{ type: "input_text", text: systemText }], - }); - } - for (const message of params.messages) { - const content = message.content; - if (typeof content === "string") { - items.push({ - role: message.role, - content: [ - message.role === "assistant" - ? { type: "output_text", text: content } - : { type: "input_text", text: content }, - ], - }); - continue; - } - if (!Array.isArray(content)) { - continue; - } - // Buffer each block type so we can push in OpenAI-Responses order instead - // of the order they appear in the Anthropic content array. The parent - // role message must precede any function_call_output items from the same - // turn, otherwise extractToolOutput() (which scans for - // function_call_output AFTER the last user-role index) will not see the - // output and the downstream scenario dispatcher will behave as if no - // tool output was returned. Similarly, assistant tool_use blocks become - // function_call items that must follow the assistant text message they - // narrate. - const textPieces: Array<{ type: "input_text" | "output_text"; text: string }> = []; - const imagePieces: Array<{ type: "input_image"; image_url: string }> = []; - const toolResultItems: ResponsesInputItem[] = []; - const toolUseItems: ResponsesInputItem[] = []; - for (const block of content) { - if (!block || typeof block !== "object") { - continue; - } - if (block.type === "text") { - textPieces.push({ - type: message.role === "assistant" ? "output_text" : "input_text", - text: block.text ?? "", - }); - continue; - } - if (block.type === "image") { - // Mock only needs to count image inputs; a placeholder URL is fine. - imagePieces.push({ type: "input_image", image_url: "anthropic-mock:image" }); - continue; - } - if (block.type === "tool_result") { - const output = stringifyToolResultContent(block.content); - if (output.trim()) { - toolResultItems.push({ - type: "function_call_output", - call_id: block.tool_use_id, - output, - ...(block.is_error === true ? { is_error: true } : {}), - }); - } - continue; - } - if (block.type === "tool_use") { - // Mirror OpenAI's function_call output_item shape so downstream - // prompt extraction still sees "the assistant just emitted a tool - // call". The scenario dispatcher looks for tool_output on the next - // user turn, not the assistant's prior tool_use, so a minimal - // placeholder is enough. - toolUseItems.push({ - type: "function_call", - name: block.name, - arguments: JSON.stringify(block.input ?? {}), - call_id: block.id, - }); - continue; - } - } - if (textPieces.length > 0 || imagePieces.length > 0) { - const combinedContent: Array> = [...textPieces, ...imagePieces]; - items.push({ role: message.role, content: combinedContent }); - } - // Emit tool_use (assistant prior calls) and tool_result (user-side - // returns) AFTER the parent role message so extractLastUserText and - // extractToolOutput walk the array in the order they expect. For a - // tool_result-only user turn with no text/image blocks, the parent - // message is intentionally omitted ā€” the function_call_output itself - // represents the user's "return the tool output" turn. - for (const toolUse of toolUseItems) { - items.push(toolUse); - } - for (const toolResult of toolResultItems) { - items.push(toolResult); - } - } - return items; -} - -type ExtractedAssistantOutput = { - text: string; - toolCalls: Array<{ id: string; name: string; input: Record }>; -}; - -function extractFinalAssistantOutputFromEvents(events: StreamEvent[]): ExtractedAssistantOutput { - const toolCalls: ExtractedAssistantOutput["toolCalls"] = []; - let text = ""; - for (const event of events) { - if (event.type !== "response.output_item.done") { - continue; - } - const item = event.item as { - type?: unknown; - name?: unknown; - call_id?: unknown; - id?: unknown; - arguments?: unknown; - content?: unknown; - }; - if (item.type === "function_call" && typeof item.name === "string") { - let input: Record = {}; - if (typeof item.arguments === "string" && item.arguments.trim()) { - try { - const parsed = JSON.parse(item.arguments) as unknown; - if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { - input = parsed as Record; - } - } catch { - // keep empty input on malformed args ā€” mock dispatcher owns arg shape - } - } - toolCalls.push({ - id: typeof item.call_id === "string" ? item.call_id : `toolu_mock_${toolCalls.length + 1}`, - name: item.name, - input, - }); - continue; - } - if (item.type === "message" && Array.isArray(item.content)) { - for (const piece of item.content as Array<{ type?: unknown; text?: unknown }>) { - if (piece?.type === "output_text" && typeof piece.text === "string") { - text = piece.text; - } - } - } - } - return { text, toolCalls }; -} - -function buildAnthropicMessageResponse(params: { - model: string; - extracted: ExtractedAssistantOutput; -}): Record { - const content: Array> = []; - if (params.extracted.text) { - content.push({ type: "text", text: params.extracted.text }); - } - for (const call of params.extracted.toolCalls) { - content.push({ - type: "tool_use", - id: call.id, - name: call.name, - input: call.input, - }); - } - if (content.length === 0) { - content.push({ type: "text", text: "" }); - } - const stopReason = params.extracted.toolCalls.length > 0 ? "tool_use" : "end_turn"; - const approxInputTokens = 64; - const approxOutputTokens = Math.max( - 16, - countApproxTokens(params.extracted.text) + params.extracted.toolCalls.length * 16, - ); - return { - id: `msg_mock_${Math.floor(Math.random() * 1_000_000).toString(16)}`, - type: "message", - role: "assistant", - model: params.model || "claude-opus-4-8", - content, - stop_reason: stopReason, - stop_sequence: null, - usage: { - input_tokens: approxInputTokens, - output_tokens: approxOutputTokens, - }, - }; -} - -const QA_ANTHROPIC_THINKING_ERROR_TEXT = - "QA replay-safe read completed, but the provider stream failed after signed thinking."; -const QA_ANTHROPIC_THINKING_ERROR_SIGNATURE = "qa_signed_thinking_block_91953"; -const QA_ANTHROPIC_THINKING_ERROR_MESSAGE = "QA injected provider stream failure"; - -function buildAnthropicThinkingErrorResponse(params: { model: string }): Record { - return { - type: "error", - error: { - type: "api_error", - message: QA_ANTHROPIC_THINKING_ERROR_MESSAGE, - }, - model: params.model || "claude-opus-4-8", - }; -} - -function buildAnthropicThinkingErrorStreamEvents(params: { - model: string; -}): AnthropicStreamEvent[] { - const messageId = `msg_mock_${Math.floor(Math.random() * 1_000_000).toString(16)}`; - return [ - { - type: "message_start", - message: { - id: messageId, - type: "message", - role: "assistant", - model: params.model || "claude-opus-4-8", - content: [], - stop_reason: null, - stop_sequence: null, - usage: { - input_tokens: 64, - output_tokens: 0, - }, - }, - }, - { - type: "content_block_start", - index: 0, - content_block: { - type: "thinking", - thinking: "", - signature: "", - }, - }, - { - type: "content_block_delta", - index: 0, - delta: { - type: "thinking_delta", - thinking: QA_ANTHROPIC_THINKING_ERROR_TEXT, - }, - }, - { - type: "content_block_delta", - index: 0, - delta: { - type: "signature_delta", - signature: QA_ANTHROPIC_THINKING_ERROR_SIGNATURE, - }, - }, - { - type: "content_block_stop", - index: 0, - }, - { - type: "message_delta", - delta: {}, - usage: { - input_tokens: 64, - output_tokens: 1120, - }, - }, - { - type: "error", - error: { - type: "api_error", - message: QA_ANTHROPIC_THINKING_ERROR_MESSAGE, - }, - }, - ]; -} - -function buildAnthropicMessageStreamEvents(params: { - model: string; - extracted: ExtractedAssistantOutput; -}): AnthropicStreamEvent[] { - const approxInputTokens = 64; - const approxOutputTokens = Math.max( - 16, - countApproxTokens(params.extracted.text) + params.extracted.toolCalls.length * 16, - ); - const messageId = `msg_mock_${Math.floor(Math.random() * 1_000_000).toString(16)}`; - const events: AnthropicStreamEvent[] = [ - { - type: "message_start", - message: { - id: messageId, - type: "message", - role: "assistant", - model: params.model || "claude-opus-4-8", - content: [], - stop_reason: null, - stop_sequence: null, - usage: { - input_tokens: approxInputTokens, - output_tokens: 0, - }, - }, - }, - ]; - let index = 0; - if (params.extracted.text || params.extracted.toolCalls.length === 0) { - events.push({ - type: "content_block_start", - index, - content_block: { - type: "text", - text: "", - }, - }); - if (params.extracted.text) { - events.push({ - type: "content_block_delta", - index, - delta: { - type: "text_delta", - text: params.extracted.text, - }, - }); - } - events.push({ - type: "content_block_stop", - index, - }); - index += 1; - } - for (const call of params.extracted.toolCalls) { - events.push({ - type: "content_block_start", - index, - content_block: { - type: "tool_use", - id: call.id, - name: call.name, - input: {}, - }, - }); - events.push({ - type: "content_block_delta", - index, - delta: { - type: "input_json_delta", - partial_json: JSON.stringify(call.input ?? {}), - }, - }); - events.push({ - type: "content_block_stop", - index, - }); - index += 1; - } - events.push({ - type: "message_delta", - delta: { - stop_reason: params.extracted.toolCalls.length > 0 ? "tool_use" : "end_turn", - }, - usage: { - input_tokens: approxInputTokens, - output_tokens: approxOutputTokens, - }, - }); - events.push({ - type: "message_stop", - }); - return events; -} - -async function buildMessagesPayload( - body: AnthropicMessagesRequest, - scenarioState: MockScenarioState, -): Promise<{ - events: StreamEvent[]; - input: ResponsesInputItem[]; - extracted: ExtractedAssistantOutput; - responseBody: Record; - streamEvents: AnthropicStreamEvent[]; - model: string; -}> { - const messages = Array.isArray(body.messages) ? body.messages : []; - const input = convertAnthropicMessagesToResponsesInput({ - system: body.system, - messages, - }); - // Treat empty-string model the same as absent. A bare typeof check lets - // `""` leak through to `responseBody.model` and `lastRequest.model`, - // which then confuses parity consumers that assume the mock always - // echoes the real provider label. Normalize once and reuse everywhere. - const normalizedModel = - typeof body.model === "string" && body.model.trim() !== "" ? body.model : "claude-opus-4-8"; - // Dispatch through the same scenario logic the /v1/responses route uses. - // Preserve declared tools so route-specific adapters mirror what the - // real provider request made available to the model. - const dispatchBody: Record = { - input, - model: normalizedModel, - stream: false, - ...(Array.isArray(body.tools) ? { tools: body.tools } : {}), - }; - const allInputText = extractAllRequestTexts(input, dispatchBody); - if (QA_ANTHROPIC_THINKING_ERROR_RECOVERY_PROMPT_RE.test(allInputText)) { - const toolOutput = extractToolOutput(input); - const shouldEmitThinkingError = - toolOutput.length > 0 && scenarioState.anthropicThinkingErrorPhase === 0; - const events = - toolOutput.length === 0 - ? buildToolCallEventsWithArgs("read", { path: "QA_KICKOFF_TASK.md" }) - : shouldEmitThinkingError - ? (() => { - scenarioState.anthropicThinkingErrorPhase = 1; - return buildAssistantEvents(""); - })() - : buildAssistantEvents("ANTHROPIC-THINKING-ERROR-RECOVERED-OK"); - const extracted = extractFinalAssistantOutputFromEvents(events); - const responseBody = shouldEmitThinkingError - ? buildAnthropicThinkingErrorResponse({ model: normalizedModel }) - : buildAnthropicMessageResponse({ - model: normalizedModel, - extracted, - }); - const streamEvents = shouldEmitThinkingError - ? buildAnthropicThinkingErrorStreamEvents({ model: normalizedModel }) - : buildAnthropicMessageStreamEvents({ - model: normalizedModel, - extracted, - }); - return { events, input, extracted, responseBody, streamEvents, model: normalizedModel }; - } - const events = await buildResponsesPayload(dispatchBody, scenarioState); - const extracted = extractFinalAssistantOutputFromEvents(events); - const responseBody = buildAnthropicMessageResponse({ - model: normalizedModel, - extracted, - }); - const streamEvents = buildAnthropicMessageStreamEvents({ - model: normalizedModel, - extracted, - }); - return { events, input, extracted, responseBody, streamEvents, model: normalizedModel }; -} - export async function startQaMockOpenAiServer(params?: { host?: string; port?: number; @@ -4020,7 +1501,7 @@ export async function startQaMockOpenAiServer(params?: { responseBody, streamEvents, model: normalizedModel, - } = await buildMessagesPayload(body, scenarioState); + } = await buildMessagesPayload(body, scenarioState, buildResponsesPayload); // Record the adapted request snapshot so /debug/requests gives the QA // suite the same plannedToolName / allInputText / toolOutput signals // on the Anthropic route that the OpenAI route already exposes. This