diff --git a/extensions/qa-lab/src/manual-lane.runtime.test.ts b/extensions/qa-lab/src/manual-lane.runtime.test.ts index 7d4a77f40b61..ace4af8e8906 100644 --- a/extensions/qa-lab/src/manual-lane.runtime.test.ts +++ b/extensions/qa-lab/src/manual-lane.runtime.test.ts @@ -41,8 +41,10 @@ describe("runQaManualLane", () => { const gatewayStop = vi.fn(); const mockStop = vi.fn(); const labStop = vi.fn(); + let outboundReply: string | null; beforeEach(() => { + outboundReply = "Protocol note: mock reply."; gatewayCall.mockReset(); gatewayStop.mockReset(); mockStop.mockReset(); @@ -78,13 +80,16 @@ describe("runQaManualLane", () => { searchMessages: vi.fn(() => []), waitFor: vi.fn(), getSnapshot: () => ({ - messages: [ - { - direction: "outbound", - conversation: { id: "qa-operator" }, - text: "Protocol note: mock reply.", - }, - ], + messages: + outboundReply === null + ? [] + : [ + { + direction: "outbound", + conversation: { id: "qa-operator" }, + text: outboundReply, + }, + ], }), }, stop: labStop, @@ -147,6 +152,97 @@ describe("runQaManualLane", () => { expect(labStop).toHaveBeenCalledTimes(1); }); + it.each([ + { + label: "timed-out run despite a stale reply", + waited: { status: "timeout", error: "provider stalled" }, + reply: "stale assistant reply", + error: "provider stalled", + }, + { + label: "failed run despite a stale reply", + waited: { status: "error", error: "authentication failed" }, + reply: "stale assistant reply", + error: "authentication failed", + }, + { + label: "successful run without an outbound reply", + waited: { status: "ok" }, + reply: null, + error: "manual lane did not produce a successful reply", + }, + { + label: "successful run with a whitespace-only reply", + waited: { status: "ok" }, + reply: " ", + error: "manual lane did not produce a successful reply", + }, + { + label: "legacy completed run without an outbound reply", + waited: { status: "error", error: "completed" }, + reply: null, + error: "manual lane did not produce a successful reply", + }, + { + label: "legacy completed run with a whitespace-only reply", + waited: { status: "error", error: " completed " }, + reply: " ", + error: "manual lane did not produce a successful reply", + }, + { + label: "timed-out run without an outbound reply", + waited: { status: "timeout", error: "provider stalled" }, + reply: null, + error: "manual lane did not produce a successful reply", + }, + { + label: "failed run with a whitespace-only reply", + waited: { status: "error", error: "authentication failed" }, + reply: " ", + error: "manual lane did not produce a successful reply", + }, + ])("rejects a $label after resource cleanup", async ({ waited, reply, error }) => { + outboundReply = reply; + gatewayCall.mockReset().mockResolvedValueOnce({ runId: "run-1" }).mockResolvedValueOnce(waited); + + await expect( + runQaManualLane({ + repoRoot: "/tmp/openclaw-repo", + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + alternateModel: "mock-openai/gpt-5.6-luna-alt", + message: "check the kickoff file", + replySettleMs: 0, + }), + ).rejects.toThrow(error); + + expect(gatewayStop).toHaveBeenCalledOnce(); + expect(cleanupTransportBeforeGatewayStop).toHaveBeenCalledOnce(); + expect(cleanupTransportAfterGatewayStop).toHaveBeenCalledOnce(); + expect(mockStop).toHaveBeenCalledOnce(); + expect(labStop).toHaveBeenCalledOnce(); + }); + + it.each([ + { status: "ok" }, + { status: "completed" }, + { status: "succeeded" }, + { status: "error", error: " completed " }, + ])("accepts the canonical successful agent wait result %#", async (waited) => { + gatewayCall.mockReset().mockResolvedValueOnce({ runId: "run-1" }).mockResolvedValueOnce(waited); + + await expect( + runQaManualLane({ + repoRoot: "/tmp/openclaw-repo", + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + alternateModel: "mock-openai/gpt-5.6-luna-alt", + message: "check the kickoff file", + replySettleMs: 0, + }), + ).resolves.toMatchObject({ waited, reply: "Protocol note: mock reply." }); + }); + it("skips the mock provider bootstrap for live frontier runs", async () => { const result = await runQaManualLane({ repoRoot: "/tmp/openclaw-repo", diff --git a/extensions/qa-lab/src/manual-lane.runtime.ts b/extensions/qa-lab/src/manual-lane.runtime.ts index 00aa810e78fb..a27514c55448 100644 --- a/extensions/qa-lab/src/manual-lane.runtime.ts +++ b/extensions/qa-lab/src/manual-lane.runtime.ts @@ -174,12 +174,7 @@ export async function runQaManualLane(params: QaManualLaneParams) { candidate.direction === "outbound" && candidate.conversation.id === "qa-operator", )?.text ?? null; - result = { - model: params.primaryModel, - waited, - reply, - watchUrl: lab.baseUrl, - }; + result = { model: params.primaryModel, waited, reply, watchUrl: lab.baseUrl }; } catch (error) { runError = error; } finally { @@ -208,8 +203,14 @@ export async function runQaManualLane(params: QaManualLaneParams) { throw cleanupError; } - if (!result) { - throw new Error("manual lane did not produce a result"); + if ( + !result?.reply?.trim() || + (result.waited.status === "error" + ? result.waited.error?.trim().toLowerCase() !== "completed" + : !["ok", "completed", "succeeded"].includes(result.waited.status ?? "")) + ) { + const providerError = result?.reply?.trim() && result.waited.error; + throw new Error(providerError || "manual lane did not produce a successful reply"); } return result; }