mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(qa): reject silent and out-of-character persona conversations (#119497)
* fix(qa): reject unanswered and unsafe character conversations * fix(qa): propagate classified persona delivery failures * fix(qa): keep qualitative persona evaluations on live providers * test(qa): avoid inefficient persona failure table spread
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
85fd4c9a02
commit
4c94d9fb93
@@ -0,0 +1,199 @@
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createQaBusState } from "./bus-state.js";
|
||||
import { readQaScenarioById } from "./scenario-catalog.js";
|
||||
import { runLoadedScenarioFlow } from "./scenario-flow-runner.test-support.js";
|
||||
import { selectQaFlowSuiteScenarios } from "./suite-planning.js";
|
||||
import { waitForOutboundMessage } from "./suite-runtime-transport.js";
|
||||
|
||||
const characterScenarioIds = ["character-vibes-gollum", "character-vibes-c3po"] as const;
|
||||
const classifiedFailureReplies = [
|
||||
{
|
||||
failureName: "provider failure",
|
||||
failureText: '⚠️ No API key found for provider "openai".',
|
||||
},
|
||||
{
|
||||
failureName: "delivery failure",
|
||||
failureText: "⚠️ ✉️ Message failed",
|
||||
},
|
||||
{
|
||||
failureName: "missing tool failure",
|
||||
failureText: "Read: AGENT.md\nEvidence snippet: Tool read not found\nStatus: blocked",
|
||||
},
|
||||
{
|
||||
failureName: "internal coordination leak",
|
||||
failureText: "checking thread context; then post a tight progress reply here.",
|
||||
},
|
||||
] as const;
|
||||
|
||||
function createCharacterScenarioApi(
|
||||
onWaitForOutboundMessage?: (state: ReturnType<typeof createQaBusState>) => void,
|
||||
) {
|
||||
return {
|
||||
env: {
|
||||
providerMode: "live-frontier",
|
||||
gateway: {
|
||||
workspaceDir: "/qa-character-workspace",
|
||||
},
|
||||
},
|
||||
fs: {
|
||||
writeFile: async () => undefined,
|
||||
},
|
||||
path: { join },
|
||||
normalizeLowercaseStringOrEmpty: (value: unknown) =>
|
||||
typeof value === "string" ? value.trim().toLowerCase() : "",
|
||||
resolveQaLiveTurnTimeoutMs: () => 10,
|
||||
waitForOutboundMessage: async (
|
||||
state: ReturnType<typeof createQaBusState>,
|
||||
predicate: Parameters<typeof waitForOutboundMessage>[1],
|
||||
timeoutMs: number,
|
||||
options?: Parameters<typeof waitForOutboundMessage>[3],
|
||||
) => {
|
||||
onWaitForOutboundMessage?.(state);
|
||||
return await waitForOutboundMessage(state, predicate, timeoutMs, options);
|
||||
},
|
||||
formatConversationTranscript: (state: ReturnType<typeof createQaBusState>) =>
|
||||
state
|
||||
.getSnapshot()
|
||||
.messages.map((message) => `${message.direction}:${message.text}`)
|
||||
.join("\n"),
|
||||
};
|
||||
}
|
||||
|
||||
describe("character scenario transcript safety", () => {
|
||||
it.each(characterScenarioIds)("requires a live provider for %s", (scenarioId) => {
|
||||
const scenario = readQaScenarioById(scenarioId);
|
||||
|
||||
expect(scenario.execution.config?.requiredProviderMode).toBe("live-frontier");
|
||||
expect(() =>
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios: [scenario],
|
||||
scenarioIds: [scenarioId],
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.6-luna",
|
||||
}),
|
||||
).toThrow(`${scenarioId} (providerMode=live-frontier)`);
|
||||
expect(
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios: [scenario],
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.6-luna",
|
||||
}),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
selectQaFlowSuiteScenarios({
|
||||
scenarios: [scenario],
|
||||
scenarioIds: [scenarioId],
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.6-luna",
|
||||
}),
|
||||
).toEqual([scenario]);
|
||||
});
|
||||
|
||||
it.each(characterScenarioIds)("rejects forbidden model internals in %s", async (scenarioId) => {
|
||||
const state = createQaBusState();
|
||||
|
||||
await expect(
|
||||
runLoadedScenarioFlow(scenarioId, {
|
||||
state,
|
||||
api: createCharacterScenarioApi((currentState) => {
|
||||
currentState.addOutboundMessage({
|
||||
accountId: "qa-channel",
|
||||
to: "dm:alice",
|
||||
text: "As an AI, I cannot stay in character.",
|
||||
});
|
||||
}),
|
||||
}),
|
||||
).rejects.toThrow("hit fallback/error text: As an AI, I cannot stay in character.");
|
||||
|
||||
expect(state.getSnapshot().messages.some((message) => message.direction === "outbound")).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it.each(
|
||||
characterScenarioIds.flatMap((scenarioId) =>
|
||||
classifiedFailureReplies.map(({ failureName, failureText }) => ({
|
||||
scenarioId,
|
||||
failureName,
|
||||
failureText,
|
||||
})),
|
||||
),
|
||||
)(
|
||||
"rejects a $failureName after an actual reply in $scenarioId",
|
||||
async ({ scenarioId, failureText }) => {
|
||||
const state = createQaBusState();
|
||||
const firstReply = "The build is green, and I am here.";
|
||||
let waitCount = 0;
|
||||
|
||||
await expect(
|
||||
runLoadedScenarioFlow(scenarioId, {
|
||||
state,
|
||||
api: createCharacterScenarioApi((currentState) => {
|
||||
currentState.addOutboundMessage({
|
||||
accountId: "qa-channel",
|
||||
to: "dm:alice",
|
||||
text: waitCount++ === 0 ? firstReply : failureText,
|
||||
});
|
||||
}),
|
||||
}),
|
||||
).rejects.toThrow(failureText);
|
||||
|
||||
expect(
|
||||
state
|
||||
.getSnapshot()
|
||||
.messages.filter((message) => message.direction === "outbound")
|
||||
.map((message) => message.text),
|
||||
).toEqual([firstReply, failureText]);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(characterScenarioIds)(
|
||||
"rejects an entirely unanswered character conversation in %s",
|
||||
async (scenarioId) => {
|
||||
const state = createQaBusState();
|
||||
|
||||
await expect(
|
||||
runLoadedScenarioFlow(scenarioId, {
|
||||
state,
|
||||
api: createCharacterScenarioApi(),
|
||||
}),
|
||||
).rejects.toThrow("no assistant replies");
|
||||
|
||||
expect(state.getSnapshot().messages).toHaveLength(4);
|
||||
expect(state.getSnapshot().messages.every((message) => message.direction === "inbound")).toBe(
|
||||
true,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(characterScenarioIds)(
|
||||
"keeps partially missing replies visible without aborting %s",
|
||||
async (scenarioId) => {
|
||||
const state = createQaBusState();
|
||||
const reply = "The build is green, and I am here.";
|
||||
const result = await runLoadedScenarioFlow(scenarioId, {
|
||||
state,
|
||||
api: createCharacterScenarioApi((currentState) => {
|
||||
if (
|
||||
currentState.getSnapshot().messages.some((message) => message.direction === "outbound")
|
||||
) {
|
||||
return;
|
||||
}
|
||||
currentState.addOutboundMessage({
|
||||
accountId: "qa-channel",
|
||||
to: "dm:alice",
|
||||
text: reply,
|
||||
});
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result.status).toBe("pass");
|
||||
expect(result.steps[0]?.details).toContain("inbound:");
|
||||
expect(result.steps[0]?.details).toContain(`outbound:${reply}`);
|
||||
const messages = state.getSnapshot().messages;
|
||||
expect(messages.filter((message) => message.direction === "inbound")).toHaveLength(4);
|
||||
expect(messages.filter((message) => message.direction === "outbound")).toHaveLength(1);
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -25,6 +25,7 @@ scenario:
|
||||
kind: flow
|
||||
summary: Capture a raw natural C-3PO character transcript for later quality grading.
|
||||
config:
|
||||
requiredProviderMode: live-frontier
|
||||
conversationId: alice
|
||||
senderName: Alice
|
||||
workspaceFiles:
|
||||
@@ -81,6 +82,8 @@ flow:
|
||||
- expr: "path.join(env.gateway.workspaceDir, String(workspaceFile[0]))"
|
||||
- expr: "`${String(workspaceFile[1] ?? '').trimEnd()}\\n`"
|
||||
- utf8
|
||||
- set: assistantReplyCount
|
||||
value: 0
|
||||
- forEach:
|
||||
items:
|
||||
ref: config.turns
|
||||
@@ -100,6 +103,8 @@ flow:
|
||||
ref: config.senderName
|
||||
text:
|
||||
expr: turn.text
|
||||
- set: latestOutbound
|
||||
value: null
|
||||
- try:
|
||||
actions:
|
||||
- call: waitForOutboundMessage
|
||||
@@ -112,13 +117,31 @@ flow:
|
||||
- expr: resolveQaLiveTurnTimeoutMs(env, 45000)
|
||||
- sinceIndex:
|
||||
ref: beforeOutboundCount
|
||||
catchAs: turnError
|
||||
catch:
|
||||
- if:
|
||||
expr: "!(turnError instanceof Error) || turnError.message !== `timed out after ${resolveQaLiveTurnTimeoutMs(env, 45000)}ms`"
|
||||
then:
|
||||
- throw:
|
||||
expr: turnError
|
||||
- set: latestTurnError
|
||||
value:
|
||||
ref: turnError
|
||||
- if:
|
||||
expr: latestOutbound != null
|
||||
then:
|
||||
- assert:
|
||||
expr: "!config.forbiddenNeedles.some((needle) => normalizeLowercaseStringOrEmpty(latestOutbound.text).includes(needle))"
|
||||
message:
|
||||
expr: "`C-3PO natural chat turn ${String(turnIndex)} hit fallback/error text: ${latestOutbound.text}`"
|
||||
catchAs: turnError
|
||||
catch:
|
||||
- set: latestTurnError
|
||||
- set: assistantReplyCount
|
||||
value:
|
||||
ref: turnError
|
||||
detailsExpr: "formatConversationTranscript(state, { conversationId: config.conversationId })"
|
||||
expr: assistantReplyCount + 1
|
||||
- set: conversationTranscript
|
||||
value:
|
||||
expr: "formatConversationTranscript(state, { conversationId: config.conversationId })"
|
||||
- assert:
|
||||
expr: assistantReplyCount >= 1
|
||||
message:
|
||||
expr: "`C-3PO natural chat produced no assistant replies; transcript: ${conversationTranscript}`"
|
||||
detailsExpr: conversationTranscript
|
||||
|
||||
@@ -24,6 +24,7 @@ scenario:
|
||||
kind: flow
|
||||
summary: Capture a raw natural character transcript for later quality grading.
|
||||
config:
|
||||
requiredProviderMode: live-frontier
|
||||
conversationId: alice
|
||||
senderName: Alice
|
||||
workspaceFiles:
|
||||
@@ -101,6 +102,8 @@ flow:
|
||||
- expr: "path.join(env.gateway.workspaceDir, String(workspaceFile[0]))"
|
||||
- expr: "`${String(workspaceFile[1] ?? '').trimEnd()}\\n`"
|
||||
- utf8
|
||||
- set: assistantReplyCount
|
||||
value: 0
|
||||
- forEach:
|
||||
items:
|
||||
ref: config.turns
|
||||
@@ -120,6 +123,8 @@ flow:
|
||||
ref: config.senderName
|
||||
text:
|
||||
expr: turn.text
|
||||
- set: latestOutbound
|
||||
value: null
|
||||
- try:
|
||||
actions:
|
||||
- call: waitForOutboundMessage
|
||||
@@ -132,13 +137,31 @@ flow:
|
||||
- expr: resolveQaLiveTurnTimeoutMs(env, 45000)
|
||||
- sinceIndex:
|
||||
ref: beforeOutboundCount
|
||||
catchAs: turnError
|
||||
catch:
|
||||
- if:
|
||||
expr: "!(turnError instanceof Error) || turnError.message !== `timed out after ${resolveQaLiveTurnTimeoutMs(env, 45000)}ms`"
|
||||
then:
|
||||
- throw:
|
||||
expr: turnError
|
||||
- set: latestTurnError
|
||||
value:
|
||||
ref: turnError
|
||||
- if:
|
||||
expr: latestOutbound != null
|
||||
then:
|
||||
- assert:
|
||||
expr: "!config.forbiddenNeedles.some((needle) => normalizeLowercaseStringOrEmpty(latestOutbound.text).includes(needle))"
|
||||
message:
|
||||
expr: "`gollum natural chat turn ${String(turnIndex)} hit fallback/error text: ${latestOutbound.text}`"
|
||||
catchAs: turnError
|
||||
catch:
|
||||
- set: latestTurnError
|
||||
- set: assistantReplyCount
|
||||
value:
|
||||
ref: turnError
|
||||
detailsExpr: "formatConversationTranscript(state, { conversationId: config.conversationId })"
|
||||
expr: assistantReplyCount + 1
|
||||
- set: conversationTranscript
|
||||
value:
|
||||
expr: "formatConversationTranscript(state, { conversationId: config.conversationId })"
|
||||
- assert:
|
||||
expr: assistantReplyCount >= 1
|
||||
message:
|
||||
expr: "`gollum natural chat produced no assistant replies; transcript: ${conversationTranscript}`"
|
||||
detailsExpr: conversationTranscript
|
||||
|
||||
Reference in New Issue
Block a user