mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(qa): restore current scenario source and documentation references (#113639)
Co-authored-by: Peter Steinberger <steipete@golden-gate.local>
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
001319f1ad
commit
3890a5efb6
@@ -4,6 +4,7 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveQaParityPackScenarioIds } from "./agentic-parity.js";
|
||||
import { resolveQaRepoPath } from "./repo-path.js";
|
||||
import {
|
||||
listQaScenarioYamlPaths,
|
||||
readQaBootstrapScenarioCatalog,
|
||||
@@ -91,6 +92,24 @@ describe("qa scenario catalog", () => {
|
||||
expect(recall.coverage?.primary).toContain(`${memory}.memory-recall`);
|
||||
});
|
||||
|
||||
it("keeps scenario documentation and code references backed by the repository", () => {
|
||||
for (const scenario of readQaScenarioPack().scenarios) {
|
||||
const referenceGroups = [
|
||||
["docsRefs", scenario.docsRefs],
|
||||
["codeRefs", scenario.codeRefs],
|
||||
] as const;
|
||||
|
||||
for (const [kind, references] of referenceGroups) {
|
||||
for (const reference of references ?? []) {
|
||||
const resolvedReference =
|
||||
resolveQaRepoPath(import.meta.dirname, reference, "file") ??
|
||||
resolveQaRepoPath(import.meta.dirname, reference, "directory");
|
||||
expect(resolvedReference, `${scenario.id} ${kind} ${reference} exists`).not.toBeNull();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("exposes bootstrap data from the YAML pack", () => {
|
||||
const catalog = readQaBootstrapScenarioCatalog();
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ scenario:
|
||||
codeRefs:
|
||||
- src/agents/system-prompt.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
- extensions/qa-lab/src/mock-openai-server.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify the agent reads repo instructions first, then completes the first bounded followthrough task without stalling.
|
||||
|
||||
@@ -12,7 +12,7 @@ scenario:
|
||||
- The model returns the requested exact marker through the reply path.
|
||||
- Exactly one outbound message contains the marker.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/reply-payload.ts
|
||||
- src/auto-reply/reply-payload.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
profiles: { "telegram:adapter": 13, "telegram:all": 12 }
|
||||
|
||||
@@ -12,7 +12,7 @@ scenario:
|
||||
- The compact tools command reaches native command dispatch.
|
||||
- The joined reply includes exec and the verbose-mode hint.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-tools.ts
|
||||
- src/auto-reply/reply/commands-info.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
profiles:
|
||||
|
||||
@@ -18,7 +18,7 @@ scenario:
|
||||
docsRefs:
|
||||
- docs/channels/whatsapp.md
|
||||
codeRefs:
|
||||
- extensions/whatsapp/src/monitor.ts
|
||||
- extensions/whatsapp/src/inbound/access-control.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: whatsapp
|
||||
|
||||
@@ -18,7 +18,7 @@ scenario:
|
||||
docsRefs:
|
||||
- docs/channels/whatsapp.md
|
||||
codeRefs:
|
||||
- extensions/whatsapp/src/monitor.ts
|
||||
- extensions/whatsapp/src/inbound/access-control.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: whatsapp
|
||||
|
||||
@@ -18,7 +18,7 @@ scenario:
|
||||
docsRefs:
|
||||
- docs/channels/whatsapp.md
|
||||
codeRefs:
|
||||
- extensions/whatsapp/src/monitor.ts
|
||||
- extensions/whatsapp/src/inbound/access-control.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: whatsapp
|
||||
|
||||
@@ -18,7 +18,7 @@ scenario:
|
||||
docsRefs:
|
||||
- docs/channels/whatsapp.md
|
||||
codeRefs:
|
||||
- extensions/whatsapp/src/monitor.ts
|
||||
- extensions/whatsapp/src/inbound/access-control.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: whatsapp
|
||||
|
||||
@@ -21,7 +21,7 @@ scenario:
|
||||
docsRefs:
|
||||
- docs/channels/whatsapp.md
|
||||
codeRefs:
|
||||
- extensions/whatsapp/src/monitor.ts
|
||||
- extensions/whatsapp/src/inbound/access-control.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: whatsapp
|
||||
|
||||
@@ -12,7 +12,7 @@ scenario:
|
||||
- The compact tools command reaches native command dispatch.
|
||||
- The joined reply includes exec and the verbose-mode hint.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-tools.ts
|
||||
- src/auto-reply/reply/commands-info.ts
|
||||
- extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts
|
||||
execution:
|
||||
profiles: { "whatsapp:mock-default": 34 }
|
||||
|
||||
@@ -20,7 +20,7 @@ scenario:
|
||||
codeRefs:
|
||||
- src/agents/tools/image-generate-tool.ts
|
||||
- src/gateway/chat-attachments.ts
|
||||
- extensions/qa-lab/src/mock-openai-server.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify a generated image is saved as media, reattached on the next turn, and described correctly through the vision path.
|
||||
|
||||
@@ -20,7 +20,7 @@ scenario:
|
||||
codeRefs:
|
||||
- src/gateway/server-methods/agent.ts
|
||||
- extensions/qa-lab/src/suite.ts
|
||||
- extensions/qa-lab/src/mock-openai-server.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify an attached image reaches the agent model and the agent can describe what it sees.
|
||||
|
||||
@@ -19,7 +19,7 @@ scenario:
|
||||
- docs/providers/openai.md
|
||||
codeRefs:
|
||||
- src/agents/tools/image-generate-tool.ts
|
||||
- extensions/qa-lab/src/mock-openai-server.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify image_generate appears when configured and returns a real saved media artifact.
|
||||
|
||||
@@ -38,7 +38,7 @@ scenario:
|
||||
- extensions/active-memory/index.ts
|
||||
- extensions/active-memory/doctor-contract-api.ts
|
||||
- extensions/qa-lab/src/suite.ts
|
||||
- extensions/qa-lab/src/mock-openai-server.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify Active Memory stays off when session-toggled off, runs memory search/get when enabled, and helps a live model answer with the recalled preference in the first visible reply.
|
||||
|
||||
@@ -19,7 +19,7 @@ scenario:
|
||||
- docs/concepts/model-failover.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/suite.ts
|
||||
- extensions/qa-lab/src/mock-openai-server.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Verify switching models preserves session context and tool use instead of dropping into plain-text only behavior.
|
||||
|
||||
@@ -18,8 +18,8 @@ scenario:
|
||||
- docs/plugins/manifest.md
|
||||
codeRefs:
|
||||
- scripts/stage-bundled-plugin-runtime.mjs
|
||||
- src/agents/skills/workspace.ts
|
||||
- src/agents/skills/plugin-skills.ts
|
||||
- src/skills/loading/workspace.ts
|
||||
- src/skills/loading/plugin-skills.ts
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Force the packaged dist-runtime plugin tree and verify an enabled bundled plugin skill survives discovery.
|
||||
|
||||
@@ -19,7 +19,7 @@ scenario:
|
||||
- docs/gateway/configuration.md
|
||||
- docs/plugins/manifest.md
|
||||
codeRefs:
|
||||
- src/agents/skills-status.ts
|
||||
- src/skills/discovery/status.ts
|
||||
- src/gateway/server-methods/config.ts
|
||||
- extensions/qa-lab/src/suite-runtime-agent-tools.ts
|
||||
execution:
|
||||
|
||||
@@ -17,7 +17,7 @@ scenario:
|
||||
- docs/tools/skills.md
|
||||
- docs/gateway/configuration.md
|
||||
codeRefs:
|
||||
- src/agents/skills-status.ts
|
||||
- src/skills/discovery/status.ts
|
||||
- extensions/qa-lab/src/suite.ts
|
||||
execution:
|
||||
kind: flow
|
||||
|
||||
@@ -17,7 +17,7 @@ scenario:
|
||||
- docs/tools/skills.md
|
||||
- docs/gateway/protocol.md
|
||||
codeRefs:
|
||||
- src/agents/skills-status.ts
|
||||
- src/skills/discovery/status.ts
|
||||
- extensions/qa-lab/src/suite.ts
|
||||
execution:
|
||||
kind: flow
|
||||
|
||||
@@ -19,7 +19,7 @@ scenario:
|
||||
- docs/channels/qa-channel.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/suite.ts
|
||||
- extensions/qa-lab/src/mock-openai-server.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
execution:
|
||||
kind: flow
|
||||
|
||||
@@ -19,7 +19,7 @@ scenario:
|
||||
- docs/help/testing.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/suite.ts
|
||||
- extensions/qa-lab/src/mock-openai-server.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- src/agents/embedded-agent-subscribe.ts
|
||||
- src/agents/embedded-agent-subscribe.handlers.lifecycle.ts
|
||||
execution:
|
||||
|
||||
@@ -27,7 +27,7 @@ scenario:
|
||||
- docs/concepts/qa-e2e-automation.md
|
||||
codeRefs:
|
||||
- extensions/diagnostics-prometheus/src/service.ts
|
||||
- src/diagnostics/internal-diagnostics.ts
|
||||
- src/infra/diagnostic-events.ts
|
||||
- extensions/qa-lab/src/suite.ts
|
||||
execution:
|
||||
kind: flow
|
||||
|
||||
@@ -17,7 +17,7 @@ scenario:
|
||||
docsRefs:
|
||||
- docs/help/testing.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/mock-openai-server.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
execution:
|
||||
kind: flow
|
||||
|
||||
@@ -16,7 +16,7 @@ scenario:
|
||||
docsRefs:
|
||||
- docs/help/testing.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/mock-openai-server.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
execution:
|
||||
kind: flow
|
||||
|
||||
@@ -26,7 +26,7 @@ scenario:
|
||||
- The Gateway child and qa-channel recover after every replacement without a user follow-up.
|
||||
- Restart-safe policy remains active, the unsafe probe stays unavailable, and the completed audit is delivered exactly once.
|
||||
docsRefs:
|
||||
- docs/reference/code-mode.md
|
||||
- docs/tools/code-mode.md
|
||||
- docs/channels/qa-channel.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/suite-runtime-agent-session.ts
|
||||
|
||||
@@ -17,7 +17,7 @@ scenario:
|
||||
docsRefs:
|
||||
- docs/help/testing.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/mock-openai-server.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
execution:
|
||||
kind: flow
|
||||
|
||||
@@ -17,7 +17,7 @@ scenario:
|
||||
docsRefs:
|
||||
- docs/help/testing.md
|
||||
codeRefs:
|
||||
- extensions/qa-lab/src/mock-openai-server.ts
|
||||
- extensions/qa-lab/src/providers/mock-openai/server.ts
|
||||
- src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
execution:
|
||||
kind: flow
|
||||
|
||||
@@ -14,7 +14,7 @@ scenario:
|
||||
docsRefs:
|
||||
- docs/tools/skills.md
|
||||
codeRefs:
|
||||
- src/agents/skills-clawhub.ts
|
||||
- src/skills/runtime/tool-dispatch.ts
|
||||
- extensions/qa-lab/src/runtime-tool-fixture.ts
|
||||
execution:
|
||||
kind: flow
|
||||
|
||||
@@ -21,7 +21,7 @@ scenario:
|
||||
codeRefs:
|
||||
- src/cron/service.ts
|
||||
- src/cron/service/timer.ts
|
||||
- src/cron/run-log.ts
|
||||
- src/cron/task-run-history.ts
|
||||
- extensions/qa-lab/src/cron-run-wait.ts
|
||||
- extensions/qa-lab/src/suite-runtime-transport.ts
|
||||
execution:
|
||||
|
||||
@@ -20,7 +20,7 @@ scenario:
|
||||
- docs/channels/qa-channel.md
|
||||
codeRefs:
|
||||
- src/cron/service.ts
|
||||
- src/cron/run-log.ts
|
||||
- src/cron/task-run-history.ts
|
||||
- extensions/qa-lab/src/cron-run-wait.ts
|
||||
- extensions/qa-lab/src/suite-runtime-transport.ts
|
||||
execution:
|
||||
|
||||
@@ -24,7 +24,7 @@ scenario:
|
||||
- extensions/qa-lab/src/scenario-runtime-api.ts
|
||||
- extensions/qa-lab/src/suite.ts
|
||||
- extensions/qa-lab/src/web-runtime.ts
|
||||
- ui/src/ui/views/chat.ts
|
||||
- ui/src/pages/chat/chat-view.ts
|
||||
gatewayRuntime:
|
||||
forwardHostHome: true
|
||||
execution:
|
||||
|
||||
Reference in New Issue
Block a user