ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-27 21:07:01 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(qa): use read failure for Matrix mention progress

Browse Source
This commit is contained in:
Vincent Koc
2026-05-27 15:59:30 +02:00
parent 5bf1f168d4
commit bf5fef857a
3 changed files with 9 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts
+1
View File
@@ -1227,6 +1227,7 @@ export async function runToolProgressMentionSafetyScenario(context: MatrixQaScen
expectedPreviewKind: "message",
finalText: buildMatrixQaToken("MATRIX_QA_TOOL_PROGRESS_MENTION_SAFE"),
label: "tool progress mention safety",
allowTopLevelFinalWithProgress: true,
mentionSafety: true,
progressPattern: /@room|@alice:matrix-qa\.test|!room:matrix-qa\.test/i,
triggerBodyBuilder: buildMatrixToolProgressMentionSafetyPrompt,
extensions/qa-matrix/src/runners/contract/scenario-runtime-shared.ts
+3 -4
View File
@@ -121,12 +121,11 @@ export function buildMatrixToolProgressErrorPrompt(sutUserId: string, text: stri
}
export function buildMatrixToolProgressMentionSafetyPrompt(sutUserId: string, text: string) {
const command = `rg -n '${MATRIX_QA_TOOL_PROGRESS_MENTION_FILENAME}' . ; sleep 2`;
return [
`${sutUserId} Tool progress QA check: call the exec tool exactly once with this exact command before answering: \`${command}\`.`,
`The QA harness must observe that exec tool call in a Matrix tool-progress preview.`,
`${sutUserId} Tool progress QA check: read the missing workspace file \`${MATRIX_QA_TOOL_PROGRESS_MENTION_FILENAME}\` before answering.`,
`The QA harness must observe that failed read in a Matrix tool-progress preview.`,
`Do not guess or send any marker before the tool result returns.`,
`After that exec command completes or fails, reply exactly \`${text}\`.`,
`After that read fails, reply exactly \`${text}\`.`,
].join(" ");
}
extensions/qa-matrix/src/runners/contract/scenarios.test.ts
+5 -8
View File
@@ -3879,7 +3879,7 @@ describe("matrix live qa scenarios", () => {
event: matrixQaMessageEvent({
kind: "message",
eventId: previewEventId,
body: "Working...\n- `tool: exec`",
body: "Working...\n- `tool: read`",
}),
since: "driver-sync-preview",
},
@@ -3888,9 +3888,9 @@ describe("matrix live qa scenarios", () => {
kind: "message",
eventId: "$tool-progress-mention-edit",
body:
'Working...\n- `search "matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt" in . -> run sleep 2`',
"Working...\n- `read matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt failed`",
formattedBody:
'Working...<br><ul><li><code>search "matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt" in . -&gt; run sleep 2</code></li></ul>',
"Working...<br><ul><li><code>read matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt failed</code></li></ul>",
mentions: {},
relatesTo: {
relType: "m.replace",
@@ -3933,12 +3933,9 @@ describe("matrix live qa scenarios", () => {
expect(artifacts.reply?.eventId).toBe("$tool-progress-mention-final");
const prompt = mockMessageBody(sendTextMessage, "sendTextMessage");
expect(prompt).toContain(
"call the exec tool exactly once with this exact command before answering",
"read the missing workspace file `matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt` before answering",
);
expect(prompt).toContain(
"`rg -n 'matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt' . ; sleep 2`",
);
expect(prompt).toContain("The QA harness must observe that exec tool call");
expect(prompt).toContain("The QA harness must observe that failed read");
});
it("preserves separate finalized block events when Matrix block streaming is enabled", async () => {