From c27fb940cfba63397fbcb22e5672e311a00cb3e1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 4 Aug 2026 06:43:40 +0800 Subject: [PATCH] test(qa): cover image generation task lifecycle (#118915) --- extensions/qa-lab/api.ts | 6 + .../media/image-generation-lifecycle.yaml | 26 ++ .../image-generation-lifecycle.e2e.test.ts | 332 ++++++++++++++++++ 3 files changed, 364 insertions(+) create mode 100644 qa/scenarios/media/image-generation-lifecycle.yaml create mode 100644 test/e2e/qa-lab/media/image-generation-lifecycle.e2e.test.ts diff --git a/extensions/qa-lab/api.ts b/extensions/qa-lab/api.ts index a1c682f81a0b..bb7bcf0bf7cc 100644 --- a/extensions/qa-lab/api.ts +++ b/extensions/qa-lab/api.ts @@ -49,6 +49,7 @@ export { readQaAgentIdentityMarkdown, } from "./src/qa-agent-bootstrap.js"; export { seedQaAgentWorkspace } from "./src/qa-agent-workspace.js"; +export { createQaChannelTransport } from "./src/qa-channel-transport.js"; export { buildQaGatewayConfig, DEFAULT_QA_CONTROL_UI_ALLOWED_ORIGINS, @@ -57,6 +58,11 @@ export { QA_BASE_RUNTIME_PLUGIN_IDS, type QaThinkingLevel, } from "./src/qa-gateway-config.js"; +export { + TINY_PNG_BASE64, + type MockOpenAiRequestSnapshot, +} from "./src/providers/mock-openai/mock-openai-contracts.js"; +export { startQaMockOpenAiServer } from "./src/providers/mock-openai/server.js"; export { renderQaMarkdownReport, type QaReportCheck, type QaReportScenario } from "./src/report.js"; export { type QaScenarioDefinition, diff --git a/qa/scenarios/media/image-generation-lifecycle.yaml b/qa/scenarios/media/image-generation-lifecycle.yaml new file mode 100644 index 000000000000..504038fc10cc --- /dev/null +++ b/qa/scenarios/media/image-generation-lifecycle.yaml @@ -0,0 +1,26 @@ +title: Image generation task lifecycle + +scenario: + id: image-generation-lifecycle + surface: image-generation + coverage: + primary: + - media.generated-image-task-lifecycle + objective: Verify one real QA-channel image request owns a stable asynchronous task across running, completion, cleanup, and recent-success duplicate guards. + successCriteria: + - The first image_generate request creates one running image-generation task with a stable task ID. + - An exact duplicate while the task is running returns that task ID and does not make a second provider call. + - The task completes through the generated-media wake path and delivers exactly one PNG outcome to QA-channel. + - After completion cleanup, an exact duplicate resolves to the same recently succeeded task without another provider call. + docsRefs: + - docs/tools/image-generation.md + - docs/channels/qa-channel.md + codeRefs: + - src/agents/tools/image-generate-tool.ts + - src/agents/tools/media-generate-background-shared.ts + - src/agents/image-generation-task-status.ts + - test/e2e/qa-lab/media/image-generation-lifecycle.e2e.test.ts + execution: + kind: vitest + path: test/e2e/qa-lab/media/image-generation-lifecycle.e2e.test.ts + summary: Hold one deterministic image provider call while QA-channel verifies running and recently completed duplicate guards around one task and one delivery. diff --git a/test/e2e/qa-lab/media/image-generation-lifecycle.e2e.test.ts b/test/e2e/qa-lab/media/image-generation-lifecycle.e2e.test.ts new file mode 100644 index 000000000000..7edc2912aa74 --- /dev/null +++ b/test/e2e/qa-lab/media/image-generation-lifecycle.e2e.test.ts @@ -0,0 +1,332 @@ +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + createQaBusState, + createQaChannelTransport, + startQaBusServer, + startQaGatewayChild, + startQaMockOpenAiServer, + TINY_PNG_BASE64, + type MockOpenAiRequestSnapshot, +} from "../../../../extensions/qa-lab/api.js"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../../.."); +const MODEL_REF = "mock-openai/gpt-5.6-luna"; +const IMAGE_MODEL_REF = "openai/gpt-image-1"; +const REQUEST_TEXT = + "Image generation check IMAGE_TASK_LIFECYCLE: generate the QA lighthouse image."; +const CONVERSATION = { id: "image-generation-lifecycle", kind: "direct" as const }; + +type GatewayTask = { + id: string; + kind?: string; + sourceId?: string; + status: string; + progressSummary?: string; + endedAt?: string | number; +}; + +async function readRequestBody(request: IncomingMessage) { + const chunks: Buffer[] = []; + for await (const chunk of request) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks).toString("utf8"); +} + +function writeJson(response: ServerResponse, status: number, body: unknown) { + response.writeHead(status, { "content-type": "application/json" }); + response.end(JSON.stringify(body)); +} + +async function startControlledImageProvider() { + const requests: Record[] = []; + const pendingResponses = new Set(); + let released = false; + + const complete = (response: ServerResponse) => { + pendingResponses.delete(response); + writeJson(response, 200, { + data: [ + { + b64_json: TINY_PNG_BASE64, + revised_prompt: "A QA lighthouse with a tiny protocol droid silhouette.", + }, + ], + }); + }; + const server = createServer(async (request, response) => { + if (request.method !== "POST" || request.url !== "/v1/images/generations") { + writeJson(response, 404, { error: "not found" }); + return; + } + try { + const body = JSON.parse(await readRequestBody(request)) as Record; + requests.push(body); + if (released) { + complete(response); + return; + } + pendingResponses.add(response); + } catch (error) { + writeJson(response, 400, { + error: error instanceof Error ? error.message : String(error), + }); + } + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address() as AddressInfo; + const release = () => { + released = true; + for (const response of [...pendingResponses]) { + complete(response); + } + }; + + return { + baseUrl: `http://127.0.0.1:${address.port}`, + requests, + release, + async stop() { + release(); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + }, + }; +} + +function configureImageProvider( + config: OpenClawConfig, + imageProviderBaseUrl: string, +): OpenClawConfig { + const openAiProvider = config.models?.providers?.openai; + if (!openAiProvider) { + throw new Error("openai image provider is missing from QA gateway config"); + } + return { + ...config, + agents: { + ...config.agents, + defaults: { + ...config.agents?.defaults, + mediaModels: { + ...config.agents?.defaults?.mediaModels, + image: { primary: IMAGE_MODEL_REF }, + }, + }, + }, + models: { + ...config.models, + providers: { + ...config.models?.providers, + openai: { + ...openAiProvider, + baseUrl: `${imageProviderBaseUrl}/v1`, + request: { + ...openAiProvider.request, + allowPrivateNetwork: true, + }, + }, + }, + }, + }; +} + +async function readMockRequests(baseUrl: string): Promise { + const response = await fetch(`${baseUrl}/debug/requests`); + if (!response.ok) { + throw new Error(`mock request log failed with HTTP ${response.status}`); + } + return (await response.json()) as MockOpenAiRequestSnapshot[]; +} + +async function waitForToolOutput(baseUrl: string, needle: string) { + let matched: MockOpenAiRequestSnapshot | undefined; + await vi.waitFor( + async () => { + matched = (await readMockRequests(baseUrl)).find( + (request) => + request.allInputText.includes("IMAGE_TASK_LIFECYCLE") && + request.toolOutput.includes(needle), + ); + expect(matched).toBeDefined(); + }, + { interval: 50, timeout: 30_000 }, + ); + return matched as MockOpenAiRequestSnapshot; +} + +async function readImageTasks( + gateway: Awaited>, +): Promise { + const payload = (await gateway.call("tasks.list", { limit: 100 })) as { + tasks?: GatewayTask[]; + }; + return (payload.tasks ?? []).filter( + (task) => task.kind === "image_generation" && task.sourceId === "image_generate:openai", + ); +} + +describe("image generation task lifecycle through QA-channel", () => { + const cleanups: Array<() => Promise> = []; + + afterEach(async () => { + const errors: unknown[] = []; + for (const cleanup of cleanups.splice(0).toReversed()) { + try { + await cleanup(); + } catch (error) { + errors.push(error); + } + } + if (errors.length === 1) { + throw errors[0]; + } + if (errors.length > 1) { + throw new AggregateError(errors, "image generation lifecycle cleanup failed"); + } + }); + + it("deduplicates running and recently completed requests around one completion", async () => { + const state = createQaBusState(); + const transport = createQaChannelTransport(state); + const bus = await startQaBusServer({ state }); + cleanups.push(() => bus.stop()); + + const mock = await startQaMockOpenAiServer(); + cleanups.push(() => mock.stop()); + + const imageProvider = await startControlledImageProvider(); + cleanups.push(() => imageProvider.stop()); + + const gateway = await startQaGatewayChild({ + repoRoot: REPO_ROOT, + useRepoCli: true, + providerBaseUrl: `${mock.baseUrl}/v1`, + providerMode: "mock-openai", + primaryModel: MODEL_REF, + alternateModel: MODEL_REF, + transport, + transportBaseUrl: bus.baseUrl, + controlUiEnabled: false, + mutateConfig: (config) => configureImageProvider(config, imageProvider.baseUrl), + }); + cleanups.push(() => gateway.stop()); + await transport.waitReady({ gateway }); + + const sendExactRequest = () => + transport.sendInbound({ + accountId: "default", + conversation: CONVERSATION, + senderId: CONVERSATION.id, + text: REQUEST_TEXT, + }); + + await sendExactRequest(); + await vi.waitFor(() => expect(imageProvider.requests).toHaveLength(1), { + interval: 50, + timeout: 30_000, + }); + + let runningTasks: GatewayTask[] = []; + await vi.waitFor( + async () => { + runningTasks = await readImageTasks(gateway); + expect(runningTasks).toHaveLength(1); + expect(runningTasks[0]).toMatchObject({ + id: expect.any(String), + status: "running", + progressSummary: "Generating image", + }); + }, + { interval: 50, timeout: 30_000 }, + ); + const taskId = runningTasks[0]?.id; + expect(taskId).toEqual(expect.any(String)); + + await sendExactRequest(); + const runningDuplicate = await waitForToolOutput(mock.baseUrl, "is already running"); + expect(runningDuplicate.toolOutput).toContain(taskId); + expect(imageProvider.requests).toHaveLength(1); + expect(await readImageTasks(gateway)).toEqual([ + expect.objectContaining({ id: taskId, status: "running" }), + ]); + + imageProvider.release(); + await vi.waitFor( + () => { + const completions = state + .getSnapshot() + .messages.filter( + (message) => + message.direction === "outbound" && + message.conversation.id === CONVERSATION.id && + message.attachments !== undefined && + message.attachments.some( + (attachment) => + attachment.kind === "image" && + attachment.mimeType === "image/png" && + attachment.contentBase64 === TINY_PNG_BASE64, + ), + ); + expect(completions).toHaveLength(1); + }, + { interval: 50, timeout: 90_000 }, + ); + + let completedTasks: GatewayTask[] = []; + await vi.waitFor( + async () => { + completedTasks = await readImageTasks(gateway); + expect(completedTasks).toEqual([ + expect.objectContaining({ + id: taskId, + status: "completed", + progressSummary: "Generated 1 image", + endedAt: expect.anything(), + }), + ]); + }, + { interval: 50, timeout: 30_000 }, + ); + + await sendExactRequest(); + const completedDuplicate = await waitForToolOutput(mock.baseUrl, "recently succeeded"); + expect(completedDuplicate.toolOutput).toContain(taskId); + + const plannedCalls = (await readMockRequests(mock.baseUrl)).filter( + (request) => + request.allInputText.includes("IMAGE_TASK_LIFECYCLE") && + request.plannedToolName === "image_generate", + ); + expect(plannedCalls).toHaveLength(3); + expect(plannedCalls.map((request) => request.plannedToolArgs)).toEqual([ + plannedCalls[0]?.plannedToolArgs, + plannedCalls[0]?.plannedToolArgs, + plannedCalls[0]?.plannedToolArgs, + ]); + expect(imageProvider.requests).toHaveLength(1); + expect(await readImageTasks(gateway)).toEqual([ + expect.objectContaining({ id: taskId, status: "completed" }), + ]); + + await new Promise((resolve) => setTimeout(resolve, 500)); + const completionOutcomes = state + .getSnapshot() + .messages.filter( + (message) => + message.direction === "outbound" && + message.conversation.id === CONVERSATION.id && + message.attachments !== undefined && + message.attachments.some((attachment) => attachment.kind === "image"), + ); + expect(completionOutcomes).toHaveLength(1); + }, 180_000); +});