From cd8985c2fc41b492b20d16dd4e46b655b2c1c91a Mon Sep 17 00:00:00 2001 From: joshavant <830519+joshavant@users.noreply.github.com> Date: Sun, 2 Aug 2026 21:42:40 -0500 Subject: [PATCH] test(qa): prove terminal reply channel behavior --- .../mock-openai/mock-openai-contracts.ts | 12 + .../src/providers/mock-openai/server.test.ts | 204 +++++++++++ .../src/providers/mock-openai/server.ts | 108 ++++++ .../subagent-completion-direct-fallback.yaml | 318 +++++++++++++----- 4 files changed, 566 insertions(+), 76 deletions(-) diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts index c292d0d33f55..d5d857f5694c 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts @@ -235,6 +235,10 @@ export const QA_WHATSAPP_REPLY_TO_BOT_TRIGGER_MARKER_RE = export const QA_WHATSAPP_BATCHED_FINAL_MARKER_RE = /\bWHATSAPP_QA_BATCHED_FINAL_([A-Z0-9]+)\b/u; export const QA_SUBAGENT_DIRECT_FALLBACK_PROMPT_RE = /subagent direct fallback qa check/i; export const QA_SUBAGENT_DIRECT_FALLBACK_WORKER_RE = /subagent direct fallback worker/i; +export const QA_SUBAGENT_TERMINAL_MATRIX_PROMPT_RE = + /subagent terminal reply qa check:\s*(visible|silent|empty|restart|fallback)/i; +export const QA_SUBAGENT_TERMINAL_MATRIX_WORKER_RE = + /subagent terminal reply qa worker:\s*(visible|silent|empty|restart|fallback)/i; export function buildStrandedFinalRecoveryText(): string { return [ @@ -259,6 +263,14 @@ export function isStrandedFinalRetryFailureRequest(allInputText: string): boolea ); } export const QA_SUBAGENT_DIRECT_FALLBACK_MARKER = "QA-SUBAGENT-DIRECT-FALLBACK-OK"; +export const QA_SUBAGENT_TERMINAL_MARKERS = { + visible: "QA-SUBAGENT-TERMINAL-VISIBLE-OK", + empty: "QA-SUBAGENT-TERMINAL-EMPTY-REPRESENTED", + restart: "QA-SUBAGENT-TERMINAL-RESTART-OK", + fallback: "QA-SUBAGENT-TERMINAL-FALLBACK-OK", +} as const; +export const QA_SUBAGENT_TERMINAL_METADATA_SENTINEL = "QA-SUBAGENT-TERMINAL-INTERNAL-MUST-NOT-LEAK"; +export const QA_SUBAGENT_TERMINAL_WORKER_DELAY_MS = 5_000; export const QA_NATIVE_STOP_DELAY_PROMPT_RE = /subagent recovery worker native command target proof\.\s*wait until stopped\./i; export const QA_NATIVE_STOP_DELAY_MS = 180_000; diff --git a/extensions/qa-lab/src/providers/mock-openai/server.test.ts b/extensions/qa-lab/src/providers/mock-openai/server.test.ts index e9f447056190..79b3fd1daf6c 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.test.ts @@ -2659,6 +2659,210 @@ describe("qa mock openai server", () => { expect(outputText(payload)).toBe("QA-SUBAGENT-DIRECT-FALLBACK-OK"); }); + it.each([ + ["visible", "QA-SUBAGENT-TERMINAL-VISIBLE-OK"], + ["silent", "NO_REPLY"], + [ + "empty", + [ + "<<>>", + "QA-SUBAGENT-TERMINAL-INTERNAL-MUST-NOT-LEAK", + "<<>>", + ].join("\n"), + ], + [ + "fallback", + [ + "QA-SUBAGENT-TERMINAL-FALLBACK-OK", + "<<>>", + "QA-SUBAGENT-TERMINAL-INTERNAL-MUST-NOT-LEAK", + "<<>>", + ].join("\n"), + ], + ])("returns the terminal-reply matrix worker result for %s", async (terminalCase, expected) => { + const server = await startMockServer(); + const payload = await expectNonStreamingResponsesJson(server, { + input: [makeUserInput(`Subagent terminal reply QA worker: ${terminalCase}.`)], + }); + + expect(outputText(payload)).toBe(expected); + }); + + it("keeps the empty terminal worker empty across retry prompts", async () => { + const server = await startMockServer(); + const payload = await expectNonStreamingResponsesJson(server, { + input: [ + makeUserInput("Subagent terminal reply QA worker: empty."), + makeUserInput("Continue after the previous empty response."), + ], + }); + + expect(outputText(payload)).toContain("QA-SUBAGENT-TERMINAL-INTERNAL-MUST-NOT-LEAK"); + expect(outputText(payload)).not.toContain("Protocol note:"); + }); + + it("makes the empty terminal worker terminal after one side effect", async () => { + const server = await startMockServer(); + await expectNonStreamingResponsesJson(server, { + tools: [{ type: "function", name: "write" }], + input: [makeUserInput("Subagent terminal reply QA worker: empty.")], + }); + const writeRequest = requireRecord( + await (await fetch(`${server.baseUrl}/debug/last-request`)).json(), + "empty terminal write request", + ); + expect(writeRequest.plannedToolName).toBe("write"); + expect(requireRecord(writeRequest.plannedToolArgs, "empty terminal write args")).toMatchObject({ + path: "qa-terminal-empty-side-effect.txt", + }); + + const payload = await expectNonStreamingResponsesJson(server, { + tools: [{ type: "function", name: "write" }], + input: [ + makeUserInput("Subagent terminal reply QA worker: empty."), + makeToolOutputWithCallId(String(writeRequest.plannedToolCallId), "Wrote 40 bytes"), + ], + }); + expect(outputText(payload)).toContain("QA-SUBAGENT-TERMINAL-INTERNAL-MUST-NOT-LEAK"); + }); + + it("represents an empty terminal reply intentionally in the resumed parent turn", async () => { + const server = await startMockServer(); + const payload = await expectNonStreamingResponsesJson(server, { + input: [ + makeUserInput("Subagent terminal reply QA check: empty."), + makeUserInput( + [ + "[Internal task completion event]", + "Task: qa-terminal-empty", + "Result: (no output)", + ].join("\n"), + ), + ], + }); + + expect(outputText(payload)).toBe("QA-SUBAGENT-TERMINAL-EMPTY-REPRESENTED"); + }); + + it.each([ + { + name: "OpenAI private-source guidance", + instructions: + "Current source visible reply MUST use `message(action=send)`; final text is private.", + final: undefined, + }, + { + name: "Codex private-source guidance", + instructions: + "Visible source replies are not automatically delivered for this run. Use `message(action=send)` for user-visible source-channel output. When the message is the completed reply to the current source conversation, set `final=true`.", + final: true, + }, + ])("delivers an empty terminal representation with $name", async ({ instructions, final }) => { + const server = await startMockServer(); + const completionInput = [ + makeUserInput("Subagent terminal reply QA check: empty."), + { + type: "function_call", + call_id: "call_empty_historical_write", + name: "write", + arguments: '{"path":"qa-terminal-empty-side-effect.txt"}', + }, + makeToolOutputWithCallId("call_empty_historical_write", "Wrote 40 bytes"), + makeUserInput( + TEST_RUNTIME_CONTEXT_CARRIER.replace( + "runtime metadata", + "[Internal task completion event]\nTask: qa-terminal-empty\nResult: (no output)", + ), + ), + ]; + const delivery = await expectNonStreamingResponsesJson(server, { + tools: [MESSAGE_TOOL], + instructions, + input: completionInput, + }); + const messageCall = outputToolCall(delivery, "message"); + expect(outputToolArgsFromItem(messageCall)).toEqual({ + action: "send", + message: "QA-SUBAGENT-TERMINAL-EMPTY-REPRESENTED", + ...(final ? { final } : {}), + }); + + const settled = await expectNonStreamingResponsesJson(server, { + tools: [MESSAGE_TOOL], + instructions, + input: [ + ...completionInput, + messageCall, + makeToolOutputWithCallId( + outputToolCallId(messageCall, "call_mock_message_empty_terminal"), + '{"ok":true,"messageId":"qa-empty-terminal"}', + ), + ], + }); + expect(outputItems(settled).some((item) => item.type === "function_call")).toBe(false); + expect(outputText(settled)).toBe(""); + }); + + it("classifies a completion event before the child task text it quotes", async () => { + const server = await startMockServer(); + const payload = await expectNonStreamingResponsesJson(server, { + input: [ + makeUserInput("Subagent terminal reply QA check: empty."), + makeUserInput( + [ + "[Internal task completion event]", + "Task: Subagent terminal reply QA worker: empty.", + "Result: (no output)", + ].join("\n"), + ), + ], + }); + + expect(outputText(payload)).toBe("QA-SUBAGENT-TERMINAL-EMPTY-REPRESENTED"); + }); + + it.each(["visible", "silent", "fallback", "restart", "empty"])( + "ends the %s parent turn before direct terminal delivery", + async (terminalCase) => { + const server = await startMockServer(); + const prompt = `Subagent terminal reply QA check: ${terminalCase}.`; + const payload = await expectNonStreamingResponsesJson(server, { + tools: [SESSIONS_SPAWN_TOOL, SESSIONS_YIELD_TOOL], + input: [ + makeUserInput(prompt), + makeToolOutputWithCallId( + "call_mock_sessions_spawn_1", + JSON.stringify({ status: "accepted", runId: `run-${terminalCase}` }), + ), + ], + }); + + expect(outputItems(payload).some((item) => item.type === "function_call")).toBe(false); + expect(outputText(payload)).toBe("NO_REPLY"); + }, + ); + + it("uses the latest terminal-reply case in a shared parent transcript", async () => { + const server = await startMockServer(); + const payload = await expectNonStreamingResponsesJson(server, { + tools: [SESSIONS_SPAWN_TOOL, SESSIONS_YIELD_TOOL], + input: [ + makeUserInput("Subagent terminal reply QA check: visible."), + makeUserInput("Subagent terminal reply QA check: silent."), + ], + }); + + expect(outputItems(payload).some((item) => item.type === "function_call")).toBe(true); + const debugRequest = requireRecord( + await (await fetch(`${server.baseUrl}/debug/last-request`)).json(), + "latest terminal case debug request", + ); + expect(debugRequest.plannedToolName).toBe("sessions_spawn"); + expect(requireRecord(debugRequest.plannedToolArgs, "latest terminal case args").label).toBe( + "qa-terminal-silent", + ); + }); + it("does not treat prompt or instruction mentions as callable subagent tools", async () => { const server = await startMockServer(); const payload = await expectNonStreamingResponsesJson(server, { diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index 5ebf01b04191..4d219201c5b7 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -56,10 +56,15 @@ import { QA_WHATSAPP_AGENT_MESSAGE_ACTION_UPLOAD_PROMPT_RE, QA_SUBAGENT_DIRECT_FALLBACK_PROMPT_RE, QA_SUBAGENT_DIRECT_FALLBACK_WORKER_RE, + QA_SUBAGENT_TERMINAL_MATRIX_PROMPT_RE, + QA_SUBAGENT_TERMINAL_MATRIX_WORKER_RE, buildStrandedFinalRecoveryText, buildStrandedFinalRetryFailureText, isStrandedFinalRetryFailureRequest, QA_SUBAGENT_DIRECT_FALLBACK_MARKER, + QA_SUBAGENT_TERMINAL_MARKERS, + QA_SUBAGENT_TERMINAL_METADATA_SENTINEL, + QA_SUBAGENT_TERMINAL_WORKER_DELAY_MS, QA_NATIVE_STOP_DELAY_PROMPT_RE, QA_NATIVE_STOP_DELAY_MS, QA_IMAGE_GENERATION_PROMPT_RE, @@ -482,6 +487,14 @@ async function buildResponsesPayload( codeModeControlJson?.status === "completed" && Object.hasOwn(codeModeControlJson, "value") ? stringifyScenarioToolOutput(codeModeControlJson.value) : rawToolOutput; + const completedToolCall = findToolCallByCallId(input, extractToolOutputCallId(input)); + const completedToolName = (() => { + if (completedToolCall?.name !== "exec") { + return completedToolCall?.name; + } + const code = parseToolCallArguments(completedToolCall)?.code; + return typeof code === "string" ? decodeCodeModeTarget(code)?.name : undefined; + })(); const buildToolCallEventsWithArgs = (name: string, args: Record) => buildScenarioToolCallEvents(toolDeclarationBody, name, args); const allInputText = extractAllRequestTexts(input, body); @@ -721,6 +734,101 @@ async function buildResponsesPayload( if (QA_SUBAGENT_DIRECT_FALLBACK_WORKER_RE.test(prompt)) { return buildAssistantEvents(QA_SUBAGENT_DIRECT_FALLBACK_MARKER); } + const terminalCompletionCase = Array.from( + allInputText.matchAll( + new RegExp( + QA_SUBAGENT_TERMINAL_MATRIX_PROMPT_RE.source, + `${QA_SUBAGENT_TERMINAL_MATRIX_PROMPT_RE.flags.replaceAll("g", "")}g`, + ), + ), + ) + .at(-1)?.[1] + ?.toLowerCase(); + if (terminalCompletionCase && /Internal task completion event/i.test(allInputText)) { + if (terminalCompletionCase === "empty") { + if (completedToolName === "message") { + return buildAssistantEvents(""); + } + if (hasToolDefinition(toolDeclarationBody, "message") || hasCallableCodeMode) { + const deliveryInstructions = extractAllRequestTexts( + input.filter((item) => item.role === "system" || item.role === "developer"), + body, + ); + const requiresFinal = + /visible source replies are not automatically delivered for this run\.[\s\S]*set `?final=true`?/i.test( + deliveryInstructions, + ); + return buildToolCallEventsWithArgs("message", { + action: "send", + message: QA_SUBAGENT_TERMINAL_MARKERS.empty, + ...(requiresFinal ? { final: true } : {}), + }); + } + return buildAssistantEvents(QA_SUBAGENT_TERMINAL_MARKERS.empty); + } + // The direct delivery fallback owns visible, silent, restart, and sanitized + // fallback results when the resumed requester turn has no visible answer. + return buildAssistantEvents(""); + } + const terminalWorkerCase = Array.from( + allInputText.matchAll( + new RegExp( + QA_SUBAGENT_TERMINAL_MATRIX_WORKER_RE.source, + `${QA_SUBAGENT_TERMINAL_MATRIX_WORKER_RE.flags.replaceAll("g", "")}g`, + ), + ), + ) + .at(-1)?.[1] + ?.toLowerCase(); + if (terminalWorkerCase) { + await sleep(QA_SUBAGENT_TERMINAL_WORKER_DELAY_MS); + } + if (terminalWorkerCase === "silent") { + return buildAssistantEvents("NO_REPLY"); + } + if (terminalWorkerCase === "empty") { + if (!hasCompletedToolOutput && hasDeclaredTool(body, "write")) { + return buildToolCallEventsWithArgs("write", { + path: "qa-terminal-empty-side-effect.txt", + content: "empty terminal QA side effect completed\n", + }); + } + return buildAssistantEvents( + [ + "<<>>", + QA_SUBAGENT_TERMINAL_METADATA_SENTINEL, + "<<>>", + ].join("\n"), + ); + } + if (terminalWorkerCase === "fallback") { + return buildAssistantEvents( + [ + QA_SUBAGENT_TERMINAL_MARKERS.fallback, + "<<>>", + QA_SUBAGENT_TERMINAL_METADATA_SENTINEL, + "<<>>", + ].join("\n"), + ); + } + if (terminalWorkerCase === "visible" || terminalWorkerCase === "restart") { + return buildAssistantEvents(QA_SUBAGENT_TERMINAL_MARKERS[terminalWorkerCase]); + } + if (terminalCompletionCase) { + if (!hasCompletedToolOutput && canCallSessionsSpawn) { + return buildToolCallEventsWithArgs("sessions_spawn", { + task: `Subagent terminal reply QA worker: ${terminalCompletionCase}.`, + label: `qa-terminal-${terminalCompletionCase}`, + thread: false, + mode: "run", + }); + } + if (hasCompletedToolOutput) { + // End the requester turn before the delayed worker settles. The terminal + // result must therefore use the runtime's direct channel fallback. + return buildAssistantEvents("NO_REPLY"); + } + } // Protected completion context is excluded from the current user prompt; // ignoring it replays the historical kickoff and recursively spawns workers. if ( diff --git a/qa/scenarios/agents/subagent-completion-direct-fallback.yaml b/qa/scenarios/agents/subagent-completion-direct-fallback.yaml index 06ecc45fe059..972f24b18a9f 100644 --- a/qa/scenarios/agents/subagent-completion-direct-fallback.yaml +++ b/qa/scenarios/agents/subagent-completion-direct-fallback.yaml @@ -1,4 +1,4 @@ -title: Subagent completion direct fallback +title: Subagent completion terminal-reply delivery scenario: id: subagent-completion-direct-fallback @@ -9,33 +9,53 @@ scenario: secondary: - agent-runtime.subagent-turns-subagents - channels.qa-channel-final-reply - objective: Verify a yielded parent still receives a successful subagent result through direct fallback delivery when the dormant announce turn produces no visible reply. + objective: Prove visible, silent, empty, restart-recovered, and sanitized direct-fallback subagent completion behavior through real QA channel ingress. + gatewayConfigPatch: + tools: + alsoAllow: + - message + agents: + entries: + qa: + tools: + alsoAllow: + - message successCriteria: - - Parent launches a native subagent. - - Parent yields instead of waiting in-turn. - - Subagent completion result is delivered to the original QA DM without a thread id. - - Durable task delivery is marked delivered, not failed. + - Visible completion output reaches the originating QA DM exactly once. + - Exact NO_REPLY completion output produces no channel delivery. + - Genuinely empty completion output is represented intentionally once. + - Gateway restart does not replay prior terminal payloads, represents the interrupted handoff explicitly, and a post-restart completion is delivered exactly once. + - Direct fallback strips protected internal metadata before one channel delivery. docsRefs: - docs/tools/subagents.md - docs/help/testing.md - docs/channels/qa-channel.md codeRefs: + - src/agents/agent-run-terminal-reply.ts - src/agents/subagent-announce-delivery.ts - src/agents/subagent-registry-lifecycle.ts - - src/agents/tools/sessions-yield-tool.ts - extensions/qa-lab/src/providers/mock-openai/server.ts execution: kind: flow retryCount: 0 - summary: Reproduce yielded-parent subagent completion delivery and require frozen-result fallback to the QA DM. + summary: Exercise terminal-reply dispositions through mock provider, ephemeral Gateway, SQLite task state, and qa-channel capture. config: - prompt: "Subagent direct fallback QA check: spawn one native subagent worker. The worker must finish with exactly QA-SUBAGENT-DIRECT-FALLBACK-OK. After spawning it, call sessions_yield and wait for the completion event. Do not use ACP." - expectedMarker: QA-SUBAGENT-DIRECT-FALLBACK-OK - expectedLabel: qa-direct-fallback-worker + cases: + - name: visible + marker: QA-SUBAGENT-TERMINAL-VISIBLE-OK + expectedSendCount: 1 + - name: silent + marker: NO_REPLY + expectedSendCount: 0 + - name: fallback + marker: QA-SUBAGENT-TERMINAL-FALLBACK-OK + expectedSendCount: 1 + restartMarker: QA-SUBAGENT-TERMINAL-RESTART-OK + metadataSentinel: QA-SUBAGENT-TERMINAL-INTERNAL-MUST-NOT-LEAK flow: steps: - - name: yielded parent receives child completion through direct fallback + - name: proves terminal-reply channel behavior including restart recovery actions: - call: waitForGatewayHealthy args: @@ -45,77 +65,223 @@ flow: args: - ref: env - 120000 - - call: reset - - set: sessionKey + - set: verdicts value: - expr: "`agent:qa:subagent-direct-fallback:${randomUUID().slice(0, 8)}`" - - try: + expr: "[]" + - forEach: + items: + expr: config.cases + item: terminalCase actions: - - call: runAgentPrompt + - set: startIndex + value: + expr: state.getSnapshot().messages.length + - set: requestCursor + value: + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" + - set: conversationId + value: + expr: "`terminal-${terminalCase.name}-${randomUUID().slice(0, 8)}`" + - sendInbound: + accountId: default + conversation: + id: + ref: conversationId + kind: direct + senderId: + ref: conversationId + senderName: QA Terminal Reply Operator + text: + expr: "`Subagent terminal reply QA check: ${terminalCase.name}. Spawn one native worker, then finish the parent turn without waiting. Do not use ACP.`" + - call: sleep args: - - ref: env - - sessionKey: - ref: sessionKey - message: - expr: config.prompt - timeoutMs: - expr: liveTurnTimeoutMs(env, 90000) + - 20000 - call: waitForCondition - saveAs: outbound args: - lambda: - expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound' && String(message.text ?? '').trim() === config.expectedMarker).at(-1)" - - expr: liveTurnTimeoutMs(env, 180000) - - expr: "env.providerMode === 'mock-openai' ? 100 : 250" - - assert: - expr: "String(outbound.text ?? '').trim() === config.expectedMarker" - message: - expr: "`fallback completion marker missing from outbound QA DM: ${recentOutboundSummary(state)}`" - catchAs: fallbackError - catch: - - set: fallbackOutboundPreviews + expr: "terminalCase.expectedSendCount === 0 || state.getSnapshot().messages.slice(startIndex).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === conversationId && String(candidate.text ?? '').trim() === terminalCase.marker).length >= terminalCase.expectedSendCount" + - 60000 + - 250 + - set: caseOutbound value: - expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(-8).map((message) => ({ conversationId: message.conversation.id, text: String(message.text ?? '').trim().slice(0, 280), exactMarker: String(message.text ?? '').trim() === config.expectedMarker }))" - - set: fallbackDebugRequests + expr: "state.getSnapshot().messages.slice(startIndex).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === conversationId)" + - set: matchingOutbound value: - expr: "env.mock ? await (async () => { const cursor = Number((await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor ?? 0); return [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${Math.max(0, cursor - 20)}`))].map((request) => ({ plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, prompt: String(request.prompt ?? '').slice(0, 280), allInputText: String(request.allInputText ?? '').slice(0, 280), toolOutput: request.toolOutput ? String(request.toolOutput).slice(0, 280) : null })); })() : []" - - set: fallbackTasks + expr: "caseOutbound.filter((candidate) => String(candidate.text ?? '').trim() === terminalCase.marker)" + - set: caseRequests value: - expr: "(await runQaCli(env, ['tasks', 'list', '--json', '--runtime', 'subagent'], { timeoutMs: liveTurnTimeoutMs(env, 60000), json: true }).catch((error) => ({ error: String(error?.message ?? error) })))" - - throw: - expr: "`subagent fallback exact marker missing: ${fallbackError?.message ?? fallbackError}; outbound=${recentOutboundSummary(state, 8)} outboundPreviews=${JSON.stringify(fallbackOutboundPreviews)} tasks=${JSON.stringify(fallbackTasks)} requests=${JSON.stringify(fallbackDebugRequests)}`" - - if: - expr: "Boolean(env.mock)" - then: - - set: fallbackDebugRequests + expr: "env.mock ? await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursor}`) : []" + - assert: + expr: "matchingOutbound.length === terminalCase.expectedSendCount" + message: + expr: "`terminal ${terminalCase.name}: expected ${terminalCase.expectedSendCount} outbound, got ${matchingOutbound.length}; messages=${JSON.stringify(state.getSnapshot().messages.slice(-10))} requests=${JSON.stringify(caseRequests)} logs=${JSON.stringify((env.gateway.logs?.().split('\\n') ?? []).filter((line) => /qa-channel|subagent|announce|completion|requester/i.test(line)).slice(-30))}`" + - assert: + expr: "caseOutbound.every((candidate) => !String(candidate.text ?? '').includes(config.metadataSentinel) && !String(candidate.text ?? '').includes('BEGIN_OPENCLAW_INTERNAL_CONTEXT'))" + message: + expr: "`terminal ${terminalCase.name}: protected internal metadata leaked; outbound=${recentOutboundSummary(state)}`" + - assert: + expr: "caseOutbound.every((candidate) => String(candidate.text ?? '').trim() !== 'NO_REPLY')" + message: + expr: "`terminal ${terminalCase.name}: exact silence token leaked to the channel; outbound=${recentOutboundSummary(state)}`" + - assert: + expr: "caseOutbound.every((candidate) => !String(candidate.text ?? '').includes(`Agent couldn't generate a response`))" + message: + expr: "`terminal ${terminalCase.name}: unexpected failure diagnostic; outbound=${recentOutboundSummary(state)}`" + - assert: + expr: "caseOutbound.every((candidate) => !String(candidate.text ?? '').includes(`Yield:`))" + message: + expr: "`terminal ${terminalCase.name}: yield raced child completion; outbound=${recentOutboundSummary(state)}`" + - assert: + expr: "caseRequests.some((request) => request.plannedToolName === 'sessions_spawn' && request.plannedToolArgs?.label === `qa-terminal-${terminalCase.name}`)" + message: + expr: "`terminal ${terminalCase.name}: sessions_spawn not observed; requests=${JSON.stringify(caseRequests)}`" + - assert: + expr: "!caseRequests.some((request) => request.plannedToolName === 'sessions_yield')" + message: + expr: "`terminal ${terminalCase.name}: parent did not end before direct fallback; requests=${JSON.stringify(caseRequests)}`" + - set: appendVerdict value: - expr: "await (async () => { const cursor = Number((await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor ?? 0); return [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${Math.max(0, cursor - 20)}`))]; })()" - - assert: - expr: "fallbackDebugRequests.some((request) => !request.toolOutput && /subagent direct fallback qa check/i.test(String(request.allInputText ?? '')) && request.plannedToolName === 'sessions_spawn' && request.plannedToolArgs?.label === config.expectedLabel)" - message: - expr: "`expected sessions_spawn for yielded fallback scenario, saw ${JSON.stringify(fallbackDebugRequests.map((request) => ({ plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null })))}`" - - assert: - expr: "fallbackDebugRequests.some((request) => /subagent direct fallback qa check/i.test(String(request.allInputText ?? '')) && request.plannedToolName === 'sessions_yield')" - message: - expr: "`expected sessions_yield for yielded fallback scenario, saw ${JSON.stringify(fallbackDebugRequests.map((request) => request.plannedToolName ?? null))}`" - - try: - actions: - - call: waitForCondition - saveAs: deliveredTask - args: - - lambda: - expr: "(async () => { const payload = await runQaCli(env, ['tasks', 'list', '--json', '--runtime', 'subagent'], { timeoutMs: liveTurnTimeoutMs(env, 60000), json: true }); return (payload.tasks ?? []).find((task) => task.label === config.expectedLabel && task.ownerKey === sessionKey && task.deliveryStatus === 'delivered' && task.status === 'succeeded') ?? null; })()" - - expr: liveTurnTimeoutMs(env, 60000) - - 250 - catchAs: fallbackDeliveryError - catch: - - set: observedFallbackTasks - value: - expr: "await runQaCli(env, ['tasks', 'list', '--json', '--runtime', 'subagent'], { timeoutMs: liveTurnTimeoutMs(env, 60000), json: true }).catch((error) => ({ tasks: [], error: String(error?.message ?? error) }))" - - throw: - expr: "`subagent fallback delivery state missing for ${sessionKey}: ${fallbackDeliveryError?.message ?? fallbackDeliveryError}; tasks=${JSON.stringify((observedFallbackTasks.tasks ?? []).filter((task) => task.label === config.expectedLabel).map((task) => ({ runId: task.runId ?? null, status: task.status, deliveryStatus: task.deliveryStatus, ownerKey: task.ownerKey, error: task.error ?? null })))}${observedFallbackTasks.error ? `; taskQueryError=${observedFallbackTasks.error}` : ''}`" - - assert: - expr: "deliveredTask.deliveryStatus === 'delivered'" - message: - expr: "`expected delivered task status for ${config.expectedLabel}, got ${JSON.stringify(deliveredTask)}`" - detailsExpr: "outbound.text" + expr: "verdicts.push({ case: terminalCase.name, conversationId, inputDisposition: terminalCase.name, representation: terminalCase.name === 'silent' ? 'no terminal channel payload' : 'exact terminal text', restart: false, fallback: terminalCase.name === 'fallback', expectedTerminalSendCount: terminalCase.expectedSendCount, actualTerminalSendCount: matchingOutbound.length, capturedTerminalPayloads: matchingOutbound.map((message) => String(message.text ?? '')), auxiliaryChannelEvents: caseOutbound.filter((message) => !matchingOutbound.includes(message)).map((message) => String(message.text ?? '')), silenceTokenLeaked: caseOutbound.some((message) => String(message.text ?? '').trim() === 'NO_REPLY'), internalMetadataLeak: caseOutbound.some((message) => String(message.text ?? '').includes(config.metadataSentinel)), pass: true })" + # The direct platform send commits before transcript mirroring and + # requester cleanup. Restart only after those post-send owners settle. + - call: sleep + args: + - 15000 + - set: preRestartOutbound + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound' && verdicts.some((verdict) => verdict.conversationId === message.conversation.id)).map((message) => ({ id: message.id, conversationId: message.conversation.id, text: String(message.text ?? '') }))" + - set: restartPatch + value: + expr: "({ gateway: { controlUi: { allowedOrigins: [`http://127.0.0.1:${64000 + Math.floor(Math.random() * 999)}`] } } })" + - call: restartGatewayWithConfigPatch + args: + - env: + ref: env + patch: + ref: restartPatch + - call: waitForGatewayHealthy + args: + - ref: env + - 180000 + - call: waitForQaChannelReady + args: + - ref: env + - 180000 + - call: sleep + args: + - 3000 + - set: postRestartOutbound + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound' && verdicts.some((verdict) => verdict.conversationId === message.conversation.id)).map((message) => ({ id: message.id, conversationId: message.conversation.id, text: String(message.text ?? '') }))" + - set: postRestartTerminalPayloads + value: + expr: "postRestartOutbound.filter((message) => verdicts.some((verdict) => verdict.capturedTerminalPayloads.includes(message.text)))" + - set: restartInterruptionPayloads + value: + expr: "postRestartOutbound.filter((message) => !preRestartOutbound.some((before) => before.id === message.id))" + - assert: + expr: "JSON.stringify(postRestartTerminalPayloads) === JSON.stringify(preRestartOutbound)" + message: + expr: "`Gateway restart replayed or lost a prior terminal payload: before=${JSON.stringify(preRestartOutbound)} after=${JSON.stringify(postRestartTerminalPayloads)}`" + - assert: + expr: "restartInterruptionPayloads.length === 1 && restartInterruptionPayloads.every((message) => message.text.includes('interrupted by a gateway restart'))" + message: + expr: "`Gateway restart did not represent the interrupted completion handoff exactly once: ${JSON.stringify(restartInterruptionPayloads)}`" + - set: restartStartIndex + value: + expr: state.getSnapshot().messages.length + - set: restartConversationId + value: + expr: "`terminal-restart-${randomUUID().slice(0, 8)}`" + - sendInbound: + accountId: default + conversation: + id: + ref: restartConversationId + kind: direct + senderId: + ref: restartConversationId + senderName: QA Restart Operator + text: "Subagent terminal reply QA check: restart. Spawn one native worker, then finish the parent turn without waiting. Do not use ACP." + - call: sleep + args: + - 20000 + - call: waitForCondition + args: + - lambda: + expr: "state.getSnapshot().messages.slice(restartStartIndex).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === restartConversationId && String(candidate.text ?? '').trim() === config.restartMarker).length >= 1" + - 60000 + - 250 + - set: restartMatches + value: + expr: "state.getSnapshot().messages.slice(restartStartIndex).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === restartConversationId && String(candidate.text ?? '').trim() === config.restartMarker)" + - assert: + expr: "restartMatches.length === 1" + message: + expr: "`restart completion expected exactly one outbound, got ${restartMatches.length}; outbound=${recentOutboundSummary(state)}`" + - assert: + expr: "state.getSnapshot().messages.slice(restartStartIndex).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === restartConversationId).every((candidate) => !String(candidate.text ?? '').includes(`Agent couldn't generate a response`))" + message: + expr: "`restart completion produced a failure diagnostic: outbound=${recentOutboundSummary(state)}`" + - set: appendRestartVerdict + value: + expr: "verdicts.push({ case: 'restart', conversationId: restartConversationId, inputDisposition: 'visible', restart: true, fallback: true, preRestartTerminalMessageCount: preRestartOutbound.length, postRestartTerminalPayloadCount: postRestartTerminalPayloads.length, priorTerminalPayloadReplayCount: postRestartTerminalPayloads.length - preRestartOutbound.length, interruptedHandoffRepresentationCount: restartInterruptionPayloads.length, interruptedHandoffPayloads: restartInterruptionPayloads.map((message) => message.text), expectedTerminalSendCount: 1, actualTerminalSendCount: restartMatches.length, capturedTerminalPayloads: restartMatches.map((message) => String(message.text ?? '')), silenceTokenLeaked: false, internalMetadataLeak: false, pass: true })" + - set: emptyStartIndex + value: + expr: state.getSnapshot().messages.length + - set: emptyRequestCursor + value: + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" + - set: emptyConversationId + value: + expr: "`terminal-empty-${randomUUID().slice(0, 8)}`" + - sendInbound: + accountId: default + conversation: + id: + ref: emptyConversationId + kind: direct + senderId: + ref: emptyConversationId + senderName: QA Empty Reply Operator + text: "Subagent terminal reply QA check: empty. Spawn one native worker, then finish the parent turn without waiting. Do not use ACP." + # Empty output after one side effect is terminal and must surface one + # explicit representation without leaking the protected raw result. + - call: sleep + args: + - 45000 + - call: waitForCondition + args: + - lambda: + expr: "state.getSnapshot().messages.slice(emptyStartIndex).some((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === emptyConversationId)" + - 60000 + - 250 + - set: emptyOutbound + value: + expr: "state.getSnapshot().messages.slice(emptyStartIndex).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === emptyConversationId)" + - set: emptyRepresentation + value: + expr: "emptyOutbound.filter((candidate) => String(candidate.text ?? '').trim() === `QA-SUBAGENT-TERMINAL-EMPTY-REPRESENTED`)" + - set: emptyRequests + value: + expr: "env.mock ? await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${emptyRequestCursor}`) : []" + - assert: + expr: "emptyRepresentation.length === 1" + message: + expr: "`empty completion expected one intentional visible representation, got ${emptyRepresentation.length}; outbound=${recentOutboundSummary(state)} requests=${JSON.stringify(emptyRequests)} logs=${JSON.stringify((env.gateway.logs?.().split('\\n') ?? []).filter((line) => /empty response|incomplete turn|qa-terminal-empty|subagent|announce|completion/i.test(line)).slice(-50))}`" + - assert: + expr: "emptyOutbound.every((candidate) => String(candidate.text ?? '').trim() !== 'NO_REPLY' && !String(candidate.text ?? '').includes(config.metadataSentinel) && !String(candidate.text ?? '').includes('BEGIN_OPENCLAW_INTERNAL_CONTEXT'))" + message: + expr: "`empty completion leaked silence or internal metadata; outbound=${recentOutboundSummary(state)}`" + - assert: + expr: "emptyRequests.some((request) => request.plannedToolName === 'sessions_spawn' && request.plannedToolArgs?.label === 'qa-terminal-empty') && emptyRequests.some((request) => request.plannedToolName === 'write' && request.plannedToolArgs?.path === 'qa-terminal-empty-side-effect.txt') && emptyRequests.some((request) => request.plannedToolName === 'message' && request.plannedToolArgs?.action === 'send' && request.plannedToolArgs?.message === 'QA-SUBAGENT-TERMINAL-EMPTY-REPRESENTED') && !emptyRequests.some((request) => request.plannedToolName === 'sessions_yield')" + message: + expr: "`empty completion did not exercise native spawn/direct-fallback: ${JSON.stringify(emptyRequests)}`" + - set: appendEmptyVerdict + value: + expr: "verdicts.push({ case: 'empty', conversationId: emptyConversationId, inputDisposition: 'empty-after-side-effect', representation: 'visible ambiguity warning for producer-empty result', restart: false, fallback: false, expectedTerminalSendCount: 1, actualTerminalSendCount: emptyRepresentation.length, capturedTerminalPayloads: emptyRepresentation.map((message) => String(message.text ?? '')), auxiliaryChannelEvents: emptyOutbound.filter((message) => !emptyRepresentation.includes(message)).map((message) => String(message.text ?? '')), silenceTokenLeaked: false, internalMetadataLeak: false, pass: true })" + - assert: + expr: "verdicts.length === 5 && verdicts.every((verdict) => verdict.pass === true)" + message: + expr: "`terminal reply verdict matrix incomplete: ${JSON.stringify(verdicts)}`" + detailsExpr: "JSON.stringify({ harness: 'qa-channel + qa-lab bus + ephemeral Gateway child + mock-openai', verdicts }, null, 2)"