fix(auto-reply): omit heartbeat inbound metadata (#122075)

* fix(auto-reply): omit heartbeat inbound metadata

Prevent synthetic heartbeat routes from appearing as user-role Conversation info while preserving delivery routing. Fixes #97067. Reported by @xxtyyq.

* test(auto-reply): use complete heartbeat model state

* test(qa): prove heartbeat metadata boundary

* test(auto-reply): deduplicate heartbeat metadata coverage

Co-authored-by: Alix-007 <li.long15@xydigit.com>
Punchcard-Session: frost-cedar-harbor-2n

---------

Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
This commit is contained in:
Alix-007
2026-08-12 03:02:22 +08:00
committed by GitHub
parent 79701dfc84
commit d76f71cee7
3 changed files with 193 additions and 37 deletions
@@ -0,0 +1,140 @@
title: Heartbeat inbound metadata boundary
scenario:
id: heartbeat-inbound-metadata
surface: cron
category: automation.heartbeat
coverage:
secondary:
- automation.wake-and-cooldown-handling
- channels.qa-channel-final-reply
regressionRefs:
- openclaw/openclaw#97067
gatewayConfigPatch:
agents:
defaults:
heartbeat:
every: 5m
target: qa-channel
to: dm:qa-heartbeat-metadata-route
directPolicy: allow
prompt: "Configured heartbeat metadata boundary proof. Reply exactly: QA-HEARTBEAT-METADATA-OK"
objective: Verify a configured heartbeat keeps delivery routing while omitting user-role inbound metadata, and ordinary channel ingress still retains that metadata.
successCriteria:
- A wake RPC executes the configured heartbeat through a real Gateway and mock provider boundary.
- The heartbeat provider request contains neither the Conversation info block nor the raw delivery target.
- The heartbeat reply is delivered to the configured qa-channel DM target.
- A normal qa-channel inbound turn retains both the Conversation info block and its raw route id.
docsRefs:
- docs/gateway/heartbeat.md
- docs/channels/qa-channel.md
codeRefs:
- src/infra/heartbeat-runner-execution.ts
- src/auto-reply/reply/get-reply-run-context.ts
- src/auto-reply/reply/inbound-meta.ts
- extensions/qa-channel/src/inbound.ts
execution:
kind: flow
suiteIsolation: isolated
isolationReason: Configures heartbeat delivery and forces an immediate wake on the real Gateway.
channel: qa-channel
retryCount: 0
summary: Prove heartbeat routing stays operational without leaking inbound metadata into the model prompt.
config:
requiredProviderMode: mock-openai
heartbeatPromptNeedle: Configured heartbeat metadata boundary proof
heartbeatRoute: dm:qa-heartbeat-metadata-route
heartbeatConversationId: qa-heartbeat-metadata-route
heartbeatMarker: QA-HEARTBEAT-METADATA-OK
normalPromptNeedle: Normal inbound metadata control
normalRoute: dm:qa-normal-metadata-route
normalConversationId: qa-normal-metadata-route
normalMarker: QA-NORMAL-METADATA-OK
flow:
steps:
- name: omits inbound metadata from a configured heartbeat without losing delivery
actions:
- assert:
expr: "env.providerMode === config.requiredProviderMode"
message: heartbeat metadata proof requires mock-openai
- call: waitForGatewayHealthy
args:
- ref: env
- 60000
- call: waitForQaChannelReady
args:
- ref: env
- 60000
- call: reset
- set: heartbeatRequestCursor
value:
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor"
- call: env.gateway.call
saveAs: wakeResponse
args:
- wake
- mode: now
text: Run the configured heartbeat metadata boundary proof now.
agentId: qa
- timeoutMs: 30000
- assert:
expr: "wakeResponse?.ok === true"
message:
expr: "`expected wake RPC success, got ${JSON.stringify(wakeResponse)}`"
- call: waitForCondition
saveAs: heartbeatRequest
args:
- lambda:
expr: "(async () => (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${heartbeatRequestCursor}`)).find((request) => String(request.allInputText ?? '').includes(config.heartbeatPromptNeedle)))()"
- expr: "liveTurnTimeoutMs(env, 60000)"
- 100
- assert:
expr: "!String(heartbeatRequest.allInputText ?? '').includes('Conversation info:') && !String(heartbeatRequest.allInputText ?? '').includes(config.heartbeatRoute)"
message:
expr: "`heartbeat leaked user-role routing metadata: ${JSON.stringify({ prompt: heartbeatRequest.prompt, allInputText: heartbeatRequest.allInputText })}`"
- call: waitForOutboundMessage
saveAs: heartbeatOutbound
args:
- ref: state
- lambda:
params: [candidate]
expr: "candidate.direction === 'outbound' && candidate.conversation.kind === 'direct' && candidate.conversation.id === config.heartbeatConversationId && String(candidate.text ?? '').trim() === config.heartbeatMarker"
- expr: "liveTurnTimeoutMs(env, 60000)"
detailsExpr: "JSON.stringify({ verdict: 'PASS', gateway: 'ephemeral-child', provider: env.providerMode, heartbeatConversationInfoPresent: String(heartbeatRequest.allInputText ?? '').includes('Conversation info:'), heartbeatRawRoutePresent: String(heartbeatRequest.allInputText ?? '').includes(config.heartbeatRoute), deliveredTo: `${heartbeatOutbound.conversation.kind}:${heartbeatOutbound.conversation.id}`, deliveredText: heartbeatOutbound.text })"
- name: retains inbound metadata for an ordinary channel turn
actions:
- set: normalRequestCursor
value:
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor"
- sendInbound:
conversation:
id:
expr: config.normalConversationId
kind: direct
title: QA Normal Metadata Route
senderId: qa-normal-sender
senderName: QA Normal Sender
text:
expr: "`${config.normalPromptNeedle}. Reply exactly: ${config.normalMarker}`"
- call: waitForCondition
saveAs: normalRequest
args:
- lambda:
expr: "(async () => (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${normalRequestCursor}`)).find((request) => String(request.allInputText ?? '').includes(config.normalPromptNeedle)))()"
- expr: "liveTurnTimeoutMs(env, 60000)"
- 100
- assert:
expr: "String(normalRequest.allInputText ?? '').includes('Conversation info:') && String(normalRequest.allInputText ?? '').includes(config.normalRoute)"
message:
expr: "`ordinary inbound lost routing metadata: ${JSON.stringify({ prompt: normalRequest.prompt, allInputText: normalRequest.allInputText })}`"
- call: waitForOutboundMessage
saveAs: normalOutbound
args:
- ref: state
- lambda:
params: [candidate]
expr: "candidate.direction === 'outbound' && candidate.conversation.kind === 'direct' && candidate.conversation.id === config.normalConversationId && String(candidate.text ?? '').trim() === config.normalMarker"
- expr: "liveTurnTimeoutMs(env, 60000)"
detailsExpr: "JSON.stringify({ verdict: 'PASS', normalConversationInfoPresent: String(normalRequest.allInputText ?? '').includes('Conversation info:'), normalRawRoutePresent: String(normalRequest.allInputText ?? '').includes(config.normalRoute), deliveredTo: `${normalOutbound.conversation.kind}:${normalOutbound.conversation.id}`, deliveredText: normalOutbound.text })"
@@ -343,11 +343,15 @@ export async function prepareReplyRunContext(params: RunPreparedReplyParams) {
? undefined
: (sessionStore?.[sessionKey] ?? sessionEntryHandle?.getCurrent() ?? sessionEntry);
let activeGoalContext = formatActiveGoalContext(inboundContextSessionEntry);
let inboundUserContext = buildInboundUserContextPrefix(
inboundUserContextSessionCtx,
envelopeOptions,
inboundContextSessionEntry,
);
// Heartbeats are synthetic system turns: delivery facts still drive routing and
// formatting, but must not be presented to the model as user-role inbound context.
let inboundUserContext = isHeartbeat
? ""
: buildInboundUserContextPrefix(
inboundUserContextSessionCtx,
envelopeOptions,
inboundContextSessionEntry,
);
const refreshInboundContextAfterAdmissionWait = async () => {
if (isHeartbeat) {
return;
@@ -2937,35 +2937,51 @@ describe("runPreparedReply media-only handling", () => {
expect(call?.followupRun.run.sourceReplyDeliveryMode).toBe("message_tool_only");
});
it("keeps heartbeat prompts out of visible transcript prompt", async () => {
const heartbeatPrompt = "Read HEARTBEAT.md and run any due maintenance.";
it.each(["heartbeat", "cron-event", "exec-event"] as const)(
"keeps %s heartbeat metadata out of the model prompt",
async (provider) => {
const heartbeatPrompt = "Read HEARTBEAT.md and run any due maintenance.";
const syntheticConversationInfo =
'Conversation info:\n```json\n{"chat_id":"discord:channel-123"}\n```';
vi.mocked(buildInboundUserContextPrefix).mockReturnValueOnce(syntheticConversationInfo);
await runPrepared({
opts: { isHeartbeat: true },
ctx: {
Body: heartbeatPrompt,
RawBody: heartbeatPrompt,
CommandBody: heartbeatPrompt,
...createProviderSurface("heartbeat"),
ChatType: "direct",
},
sessionCtx: {
Body: heartbeatPrompt,
BodyStripped: heartbeatPrompt,
...createProviderSurface("heartbeat"),
ChatType: "direct",
},
});
await runPrepared({
opts: { isHeartbeat: true },
ctx: {
Body: heartbeatPrompt,
RawBody: heartbeatPrompt,
CommandBody: heartbeatPrompt,
...createProviderSurface(provider),
ChatType: "direct",
OriginatingChannel: "discord",
OriginatingTo: "discord:channel-123",
},
sessionCtx: {
Body: heartbeatPrompt,
BodyStripped: heartbeatPrompt,
...createProviderSurface(provider),
ChatType: "direct",
OriginatingChannel: "discord",
OriginatingTo: "discord:channel-123",
},
});
const call = requireLastRunReplyAgentCall();
expect(call?.commandBody).toContain(heartbeatPrompt);
expect(call?.followupRun.prompt).toContain(heartbeatPrompt);
expect(call?.transcriptCommandBody).toBe("[OpenClaw heartbeat poll]");
expect(call?.followupRun.transcriptPrompt).toBe("[OpenClaw heartbeat poll]");
expect(call?.followupRun.userTurnTranscriptRecorder?.message).toMatchObject({
provenance: { kind: "internal_system", sourceTool: "heartbeat" },
});
});
const call = requireLastRunReplyAgentCall();
expect(call?.commandBody).toContain(heartbeatPrompt);
expect(call?.followupRun.prompt).toContain(heartbeatPrompt);
expect(call?.followupRun.prompt).not.toContain(syntheticConversationInfo);
expect(buildInboundUserContextPrefix).not.toHaveBeenCalled();
expect(call?.sessionCtx).toMatchObject({
OriginatingChannel: "discord",
OriginatingTo: "discord:channel-123",
});
expect(call?.transcriptCommandBody).toBe("[OpenClaw heartbeat poll]");
expect(call?.followupRun.transcriptPrompt).toBe("[OpenClaw heartbeat poll]");
expect(call?.followupRun.userTurnTranscriptRecorder?.message).toMatchObject({
provenance: { kind: "internal_system", sourceTool: "heartbeat" },
});
},
);
it("keeps active goal context out of background heartbeat turns", async () => {
const sessionEntry: SessionEntry = {
@@ -2990,11 +3006,7 @@ describe("runPreparedReply media-only handling", () => {
sessionStore: { "session-key": sessionEntry },
});
expect(buildInboundUserContextPrefix).toHaveBeenCalledWith(
expect.anything(),
expect.anything(),
undefined,
);
expect(buildInboundUserContextPrefix).not.toHaveBeenCalled();
});
it("uses persisted Discord chat metadata for system-event CLI static prompt identity", async () => {