From d76f71cee7d2e466eda501706ab274dd59eeab0c Mon Sep 17 00:00:00 2001 From: Alix-007 Date: Wed, 12 Aug 2026 03:02:22 +0800 Subject: [PATCH] fix(auto-reply): omit heartbeat inbound metadata (#122075) * fix(auto-reply): omit heartbeat inbound metadata Prevent synthetic heartbeat routes from appearing as user-role Conversation info while preserving delivery routing. Fixes #97067. Reported by @xxtyyq. * test(auto-reply): use complete heartbeat model state * test(qa): prove heartbeat metadata boundary * test(auto-reply): deduplicate heartbeat metadata coverage Co-authored-by: Alix-007 Punchcard-Session: frost-cedar-harbor-2n --------- Co-authored-by: Vincent Koc --- .../heartbeat-inbound-metadata.yaml | 140 ++++++++++++++++++ src/auto-reply/reply/get-reply-run-context.ts | 14 +- .../reply/get-reply-run.media-only.test.ts | 76 ++++++---- 3 files changed, 193 insertions(+), 37 deletions(-) create mode 100644 qa/scenarios/scheduling/heartbeat-inbound-metadata.yaml diff --git a/qa/scenarios/scheduling/heartbeat-inbound-metadata.yaml b/qa/scenarios/scheduling/heartbeat-inbound-metadata.yaml new file mode 100644 index 000000000000..5577e9d250b6 --- /dev/null +++ b/qa/scenarios/scheduling/heartbeat-inbound-metadata.yaml @@ -0,0 +1,140 @@ +title: Heartbeat inbound metadata boundary + +scenario: + id: heartbeat-inbound-metadata + surface: cron + category: automation.heartbeat + coverage: + secondary: + - automation.wake-and-cooldown-handling + - channels.qa-channel-final-reply + regressionRefs: + - openclaw/openclaw#97067 + gatewayConfigPatch: + agents: + defaults: + heartbeat: + every: 5m + target: qa-channel + to: dm:qa-heartbeat-metadata-route + directPolicy: allow + prompt: "Configured heartbeat metadata boundary proof. Reply exactly: QA-HEARTBEAT-METADATA-OK" + objective: Verify a configured heartbeat keeps delivery routing while omitting user-role inbound metadata, and ordinary channel ingress still retains that metadata. + successCriteria: + - A wake RPC executes the configured heartbeat through a real Gateway and mock provider boundary. + - The heartbeat provider request contains neither the Conversation info block nor the raw delivery target. + - The heartbeat reply is delivered to the configured qa-channel DM target. + - A normal qa-channel inbound turn retains both the Conversation info block and its raw route id. + docsRefs: + - docs/gateway/heartbeat.md + - docs/channels/qa-channel.md + codeRefs: + - src/infra/heartbeat-runner-execution.ts + - src/auto-reply/reply/get-reply-run-context.ts + - src/auto-reply/reply/inbound-meta.ts + - extensions/qa-channel/src/inbound.ts + execution: + kind: flow + suiteIsolation: isolated + isolationReason: Configures heartbeat delivery and forces an immediate wake on the real Gateway. + channel: qa-channel + retryCount: 0 + summary: Prove heartbeat routing stays operational without leaking inbound metadata into the model prompt. + config: + requiredProviderMode: mock-openai + heartbeatPromptNeedle: Configured heartbeat metadata boundary proof + heartbeatRoute: dm:qa-heartbeat-metadata-route + heartbeatConversationId: qa-heartbeat-metadata-route + heartbeatMarker: QA-HEARTBEAT-METADATA-OK + normalPromptNeedle: Normal inbound metadata control + normalRoute: dm:qa-normal-metadata-route + normalConversationId: qa-normal-metadata-route + normalMarker: QA-NORMAL-METADATA-OK + +flow: + steps: + - name: omits inbound metadata from a configured heartbeat without losing delivery + actions: + - assert: + expr: "env.providerMode === config.requiredProviderMode" + message: heartbeat metadata proof requires mock-openai + - call: waitForGatewayHealthy + args: + - ref: env + - 60000 + - call: waitForQaChannelReady + args: + - ref: env + - 60000 + - call: reset + - set: heartbeatRequestCursor + value: + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor" + - call: env.gateway.call + saveAs: wakeResponse + args: + - wake + - mode: now + text: Run the configured heartbeat metadata boundary proof now. + agentId: qa + - timeoutMs: 30000 + - assert: + expr: "wakeResponse?.ok === true" + message: + expr: "`expected wake RPC success, got ${JSON.stringify(wakeResponse)}`" + - call: waitForCondition + saveAs: heartbeatRequest + args: + - lambda: + expr: "(async () => (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${heartbeatRequestCursor}`)).find((request) => String(request.allInputText ?? '').includes(config.heartbeatPromptNeedle)))()" + - expr: "liveTurnTimeoutMs(env, 60000)" + - 100 + - assert: + expr: "!String(heartbeatRequest.allInputText ?? '').includes('Conversation info:') && !String(heartbeatRequest.allInputText ?? '').includes(config.heartbeatRoute)" + message: + expr: "`heartbeat leaked user-role routing metadata: ${JSON.stringify({ prompt: heartbeatRequest.prompt, allInputText: heartbeatRequest.allInputText })}`" + - call: waitForOutboundMessage + saveAs: heartbeatOutbound + args: + - ref: state + - lambda: + params: [candidate] + expr: "candidate.direction === 'outbound' && candidate.conversation.kind === 'direct' && candidate.conversation.id === config.heartbeatConversationId && String(candidate.text ?? '').trim() === config.heartbeatMarker" + - expr: "liveTurnTimeoutMs(env, 60000)" + detailsExpr: "JSON.stringify({ verdict: 'PASS', gateway: 'ephemeral-child', provider: env.providerMode, heartbeatConversationInfoPresent: String(heartbeatRequest.allInputText ?? '').includes('Conversation info:'), heartbeatRawRoutePresent: String(heartbeatRequest.allInputText ?? '').includes(config.heartbeatRoute), deliveredTo: `${heartbeatOutbound.conversation.kind}:${heartbeatOutbound.conversation.id}`, deliveredText: heartbeatOutbound.text })" + + - name: retains inbound metadata for an ordinary channel turn + actions: + - set: normalRequestCursor + value: + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor" + - sendInbound: + conversation: + id: + expr: config.normalConversationId + kind: direct + title: QA Normal Metadata Route + senderId: qa-normal-sender + senderName: QA Normal Sender + text: + expr: "`${config.normalPromptNeedle}. Reply exactly: ${config.normalMarker}`" + - call: waitForCondition + saveAs: normalRequest + args: + - lambda: + expr: "(async () => (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${normalRequestCursor}`)).find((request) => String(request.allInputText ?? '').includes(config.normalPromptNeedle)))()" + - expr: "liveTurnTimeoutMs(env, 60000)" + - 100 + - assert: + expr: "String(normalRequest.allInputText ?? '').includes('Conversation info:') && String(normalRequest.allInputText ?? '').includes(config.normalRoute)" + message: + expr: "`ordinary inbound lost routing metadata: ${JSON.stringify({ prompt: normalRequest.prompt, allInputText: normalRequest.allInputText })}`" + - call: waitForOutboundMessage + saveAs: normalOutbound + args: + - ref: state + - lambda: + params: [candidate] + expr: "candidate.direction === 'outbound' && candidate.conversation.kind === 'direct' && candidate.conversation.id === config.normalConversationId && String(candidate.text ?? '').trim() === config.normalMarker" + - expr: "liveTurnTimeoutMs(env, 60000)" + detailsExpr: "JSON.stringify({ verdict: 'PASS', normalConversationInfoPresent: String(normalRequest.allInputText ?? '').includes('Conversation info:'), normalRawRoutePresent: String(normalRequest.allInputText ?? '').includes(config.normalRoute), deliveredTo: `${normalOutbound.conversation.kind}:${normalOutbound.conversation.id}`, deliveredText: normalOutbound.text })" diff --git a/src/auto-reply/reply/get-reply-run-context.ts b/src/auto-reply/reply/get-reply-run-context.ts index 85ebb73487ef..36346eda876b 100644 --- a/src/auto-reply/reply/get-reply-run-context.ts +++ b/src/auto-reply/reply/get-reply-run-context.ts @@ -343,11 +343,15 @@ export async function prepareReplyRunContext(params: RunPreparedReplyParams) { ? undefined : (sessionStore?.[sessionKey] ?? sessionEntryHandle?.getCurrent() ?? sessionEntry); let activeGoalContext = formatActiveGoalContext(inboundContextSessionEntry); - let inboundUserContext = buildInboundUserContextPrefix( - inboundUserContextSessionCtx, - envelopeOptions, - inboundContextSessionEntry, - ); + // Heartbeats are synthetic system turns: delivery facts still drive routing and + // formatting, but must not be presented to the model as user-role inbound context. + let inboundUserContext = isHeartbeat + ? "" + : buildInboundUserContextPrefix( + inboundUserContextSessionCtx, + envelopeOptions, + inboundContextSessionEntry, + ); const refreshInboundContextAfterAdmissionWait = async () => { if (isHeartbeat) { return; diff --git a/src/auto-reply/reply/get-reply-run.media-only.test.ts b/src/auto-reply/reply/get-reply-run.media-only.test.ts index 4b2cd142b1e9..96c19fcb83c1 100644 --- a/src/auto-reply/reply/get-reply-run.media-only.test.ts +++ b/src/auto-reply/reply/get-reply-run.media-only.test.ts @@ -2937,35 +2937,51 @@ describe("runPreparedReply media-only handling", () => { expect(call?.followupRun.run.sourceReplyDeliveryMode).toBe("message_tool_only"); }); - it("keeps heartbeat prompts out of visible transcript prompt", async () => { - const heartbeatPrompt = "Read HEARTBEAT.md and run any due maintenance."; + it.each(["heartbeat", "cron-event", "exec-event"] as const)( + "keeps %s heartbeat metadata out of the model prompt", + async (provider) => { + const heartbeatPrompt = "Read HEARTBEAT.md and run any due maintenance."; + const syntheticConversationInfo = + 'Conversation info:\n```json\n{"chat_id":"discord:channel-123"}\n```'; + vi.mocked(buildInboundUserContextPrefix).mockReturnValueOnce(syntheticConversationInfo); - await runPrepared({ - opts: { isHeartbeat: true }, - ctx: { - Body: heartbeatPrompt, - RawBody: heartbeatPrompt, - CommandBody: heartbeatPrompt, - ...createProviderSurface("heartbeat"), - ChatType: "direct", - }, - sessionCtx: { - Body: heartbeatPrompt, - BodyStripped: heartbeatPrompt, - ...createProviderSurface("heartbeat"), - ChatType: "direct", - }, - }); + await runPrepared({ + opts: { isHeartbeat: true }, + ctx: { + Body: heartbeatPrompt, + RawBody: heartbeatPrompt, + CommandBody: heartbeatPrompt, + ...createProviderSurface(provider), + ChatType: "direct", + OriginatingChannel: "discord", + OriginatingTo: "discord:channel-123", + }, + sessionCtx: { + Body: heartbeatPrompt, + BodyStripped: heartbeatPrompt, + ...createProviderSurface(provider), + ChatType: "direct", + OriginatingChannel: "discord", + OriginatingTo: "discord:channel-123", + }, + }); - const call = requireLastRunReplyAgentCall(); - expect(call?.commandBody).toContain(heartbeatPrompt); - expect(call?.followupRun.prompt).toContain(heartbeatPrompt); - expect(call?.transcriptCommandBody).toBe("[OpenClaw heartbeat poll]"); - expect(call?.followupRun.transcriptPrompt).toBe("[OpenClaw heartbeat poll]"); - expect(call?.followupRun.userTurnTranscriptRecorder?.message).toMatchObject({ - provenance: { kind: "internal_system", sourceTool: "heartbeat" }, - }); - }); + const call = requireLastRunReplyAgentCall(); + expect(call?.commandBody).toContain(heartbeatPrompt); + expect(call?.followupRun.prompt).toContain(heartbeatPrompt); + expect(call?.followupRun.prompt).not.toContain(syntheticConversationInfo); + expect(buildInboundUserContextPrefix).not.toHaveBeenCalled(); + expect(call?.sessionCtx).toMatchObject({ + OriginatingChannel: "discord", + OriginatingTo: "discord:channel-123", + }); + expect(call?.transcriptCommandBody).toBe("[OpenClaw heartbeat poll]"); + expect(call?.followupRun.transcriptPrompt).toBe("[OpenClaw heartbeat poll]"); + expect(call?.followupRun.userTurnTranscriptRecorder?.message).toMatchObject({ + provenance: { kind: "internal_system", sourceTool: "heartbeat" }, + }); + }, + ); it("keeps active goal context out of background heartbeat turns", async () => { const sessionEntry: SessionEntry = { @@ -2990,11 +3006,7 @@ describe("runPreparedReply media-only handling", () => { sessionStore: { "session-key": sessionEntry }, }); - expect(buildInboundUserContextPrefix).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - undefined, - ); + expect(buildInboundUserContextPrefix).not.toHaveBeenCalled(); }); it("uses persisted Discord chat metadata for system-event CLI static prompt identity", async () => {