mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(auto-reply): omit heartbeat inbound metadata (#122075)
* fix(auto-reply): omit heartbeat inbound metadata Prevent synthetic heartbeat routes from appearing as user-role Conversation info while preserving delivery routing. Fixes #97067. Reported by @xxtyyq. * test(auto-reply): use complete heartbeat model state * test(qa): prove heartbeat metadata boundary * test(auto-reply): deduplicate heartbeat metadata coverage Co-authored-by: Alix-007 <li.long15@xydigit.com> Punchcard-Session: frost-cedar-harbor-2n --------- Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
This commit is contained in:
@@ -0,0 +1,140 @@
|
||||
title: Heartbeat inbound metadata boundary
|
||||
|
||||
scenario:
|
||||
id: heartbeat-inbound-metadata
|
||||
surface: cron
|
||||
category: automation.heartbeat
|
||||
coverage:
|
||||
secondary:
|
||||
- automation.wake-and-cooldown-handling
|
||||
- channels.qa-channel-final-reply
|
||||
regressionRefs:
|
||||
- openclaw/openclaw#97067
|
||||
gatewayConfigPatch:
|
||||
agents:
|
||||
defaults:
|
||||
heartbeat:
|
||||
every: 5m
|
||||
target: qa-channel
|
||||
to: dm:qa-heartbeat-metadata-route
|
||||
directPolicy: allow
|
||||
prompt: "Configured heartbeat metadata boundary proof. Reply exactly: QA-HEARTBEAT-METADATA-OK"
|
||||
objective: Verify a configured heartbeat keeps delivery routing while omitting user-role inbound metadata, and ordinary channel ingress still retains that metadata.
|
||||
successCriteria:
|
||||
- A wake RPC executes the configured heartbeat through a real Gateway and mock provider boundary.
|
||||
- The heartbeat provider request contains neither the Conversation info block nor the raw delivery target.
|
||||
- The heartbeat reply is delivered to the configured qa-channel DM target.
|
||||
- A normal qa-channel inbound turn retains both the Conversation info block and its raw route id.
|
||||
docsRefs:
|
||||
- docs/gateway/heartbeat.md
|
||||
- docs/channels/qa-channel.md
|
||||
codeRefs:
|
||||
- src/infra/heartbeat-runner-execution.ts
|
||||
- src/auto-reply/reply/get-reply-run-context.ts
|
||||
- src/auto-reply/reply/inbound-meta.ts
|
||||
- extensions/qa-channel/src/inbound.ts
|
||||
execution:
|
||||
kind: flow
|
||||
suiteIsolation: isolated
|
||||
isolationReason: Configures heartbeat delivery and forces an immediate wake on the real Gateway.
|
||||
channel: qa-channel
|
||||
retryCount: 0
|
||||
summary: Prove heartbeat routing stays operational without leaking inbound metadata into the model prompt.
|
||||
config:
|
||||
requiredProviderMode: mock-openai
|
||||
heartbeatPromptNeedle: Configured heartbeat metadata boundary proof
|
||||
heartbeatRoute: dm:qa-heartbeat-metadata-route
|
||||
heartbeatConversationId: qa-heartbeat-metadata-route
|
||||
heartbeatMarker: QA-HEARTBEAT-METADATA-OK
|
||||
normalPromptNeedle: Normal inbound metadata control
|
||||
normalRoute: dm:qa-normal-metadata-route
|
||||
normalConversationId: qa-normal-metadata-route
|
||||
normalMarker: QA-NORMAL-METADATA-OK
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: omits inbound metadata from a configured heartbeat without losing delivery
|
||||
actions:
|
||||
- assert:
|
||||
expr: "env.providerMode === config.requiredProviderMode"
|
||||
message: heartbeat metadata proof requires mock-openai
|
||||
- call: waitForGatewayHealthy
|
||||
args:
|
||||
- ref: env
|
||||
- 60000
|
||||
- call: waitForQaChannelReady
|
||||
args:
|
||||
- ref: env
|
||||
- 60000
|
||||
- call: reset
|
||||
- set: heartbeatRequestCursor
|
||||
value:
|
||||
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor"
|
||||
- call: env.gateway.call
|
||||
saveAs: wakeResponse
|
||||
args:
|
||||
- wake
|
||||
- mode: now
|
||||
text: Run the configured heartbeat metadata boundary proof now.
|
||||
agentId: qa
|
||||
- timeoutMs: 30000
|
||||
- assert:
|
||||
expr: "wakeResponse?.ok === true"
|
||||
message:
|
||||
expr: "`expected wake RPC success, got ${JSON.stringify(wakeResponse)}`"
|
||||
- call: waitForCondition
|
||||
saveAs: heartbeatRequest
|
||||
args:
|
||||
- lambda:
|
||||
expr: "(async () => (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${heartbeatRequestCursor}`)).find((request) => String(request.allInputText ?? '').includes(config.heartbeatPromptNeedle)))()"
|
||||
- expr: "liveTurnTimeoutMs(env, 60000)"
|
||||
- 100
|
||||
- assert:
|
||||
expr: "!String(heartbeatRequest.allInputText ?? '').includes('Conversation info:') && !String(heartbeatRequest.allInputText ?? '').includes(config.heartbeatRoute)"
|
||||
message:
|
||||
expr: "`heartbeat leaked user-role routing metadata: ${JSON.stringify({ prompt: heartbeatRequest.prompt, allInputText: heartbeatRequest.allInputText })}`"
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: heartbeatOutbound
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.direction === 'outbound' && candidate.conversation.kind === 'direct' && candidate.conversation.id === config.heartbeatConversationId && String(candidate.text ?? '').trim() === config.heartbeatMarker"
|
||||
- expr: "liveTurnTimeoutMs(env, 60000)"
|
||||
detailsExpr: "JSON.stringify({ verdict: 'PASS', gateway: 'ephemeral-child', provider: env.providerMode, heartbeatConversationInfoPresent: String(heartbeatRequest.allInputText ?? '').includes('Conversation info:'), heartbeatRawRoutePresent: String(heartbeatRequest.allInputText ?? '').includes(config.heartbeatRoute), deliveredTo: `${heartbeatOutbound.conversation.kind}:${heartbeatOutbound.conversation.id}`, deliveredText: heartbeatOutbound.text })"
|
||||
|
||||
- name: retains inbound metadata for an ordinary channel turn
|
||||
actions:
|
||||
- set: normalRequestCursor
|
||||
value:
|
||||
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor"
|
||||
- sendInbound:
|
||||
conversation:
|
||||
id:
|
||||
expr: config.normalConversationId
|
||||
kind: direct
|
||||
title: QA Normal Metadata Route
|
||||
senderId: qa-normal-sender
|
||||
senderName: QA Normal Sender
|
||||
text:
|
||||
expr: "`${config.normalPromptNeedle}. Reply exactly: ${config.normalMarker}`"
|
||||
- call: waitForCondition
|
||||
saveAs: normalRequest
|
||||
args:
|
||||
- lambda:
|
||||
expr: "(async () => (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${normalRequestCursor}`)).find((request) => String(request.allInputText ?? '').includes(config.normalPromptNeedle)))()"
|
||||
- expr: "liveTurnTimeoutMs(env, 60000)"
|
||||
- 100
|
||||
- assert:
|
||||
expr: "String(normalRequest.allInputText ?? '').includes('Conversation info:') && String(normalRequest.allInputText ?? '').includes(config.normalRoute)"
|
||||
message:
|
||||
expr: "`ordinary inbound lost routing metadata: ${JSON.stringify({ prompt: normalRequest.prompt, allInputText: normalRequest.allInputText })}`"
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: normalOutbound
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.direction === 'outbound' && candidate.conversation.kind === 'direct' && candidate.conversation.id === config.normalConversationId && String(candidate.text ?? '').trim() === config.normalMarker"
|
||||
- expr: "liveTurnTimeoutMs(env, 60000)"
|
||||
detailsExpr: "JSON.stringify({ verdict: 'PASS', normalConversationInfoPresent: String(normalRequest.allInputText ?? '').includes('Conversation info:'), normalRawRoutePresent: String(normalRequest.allInputText ?? '').includes(config.normalRoute), deliveredTo: `${normalOutbound.conversation.kind}:${normalOutbound.conversation.id}`, deliveredText: normalOutbound.text })"
|
||||
@@ -343,11 +343,15 @@ export async function prepareReplyRunContext(params: RunPreparedReplyParams) {
|
||||
? undefined
|
||||
: (sessionStore?.[sessionKey] ?? sessionEntryHandle?.getCurrent() ?? sessionEntry);
|
||||
let activeGoalContext = formatActiveGoalContext(inboundContextSessionEntry);
|
||||
let inboundUserContext = buildInboundUserContextPrefix(
|
||||
inboundUserContextSessionCtx,
|
||||
envelopeOptions,
|
||||
inboundContextSessionEntry,
|
||||
);
|
||||
// Heartbeats are synthetic system turns: delivery facts still drive routing and
|
||||
// formatting, but must not be presented to the model as user-role inbound context.
|
||||
let inboundUserContext = isHeartbeat
|
||||
? ""
|
||||
: buildInboundUserContextPrefix(
|
||||
inboundUserContextSessionCtx,
|
||||
envelopeOptions,
|
||||
inboundContextSessionEntry,
|
||||
);
|
||||
const refreshInboundContextAfterAdmissionWait = async () => {
|
||||
if (isHeartbeat) {
|
||||
return;
|
||||
|
||||
@@ -2937,35 +2937,51 @@ describe("runPreparedReply media-only handling", () => {
|
||||
expect(call?.followupRun.run.sourceReplyDeliveryMode).toBe("message_tool_only");
|
||||
});
|
||||
|
||||
it("keeps heartbeat prompts out of visible transcript prompt", async () => {
|
||||
const heartbeatPrompt = "Read HEARTBEAT.md and run any due maintenance.";
|
||||
it.each(["heartbeat", "cron-event", "exec-event"] as const)(
|
||||
"keeps %s heartbeat metadata out of the model prompt",
|
||||
async (provider) => {
|
||||
const heartbeatPrompt = "Read HEARTBEAT.md and run any due maintenance.";
|
||||
const syntheticConversationInfo =
|
||||
'Conversation info:\n```json\n{"chat_id":"discord:channel-123"}\n```';
|
||||
vi.mocked(buildInboundUserContextPrefix).mockReturnValueOnce(syntheticConversationInfo);
|
||||
|
||||
await runPrepared({
|
||||
opts: { isHeartbeat: true },
|
||||
ctx: {
|
||||
Body: heartbeatPrompt,
|
||||
RawBody: heartbeatPrompt,
|
||||
CommandBody: heartbeatPrompt,
|
||||
...createProviderSurface("heartbeat"),
|
||||
ChatType: "direct",
|
||||
},
|
||||
sessionCtx: {
|
||||
Body: heartbeatPrompt,
|
||||
BodyStripped: heartbeatPrompt,
|
||||
...createProviderSurface("heartbeat"),
|
||||
ChatType: "direct",
|
||||
},
|
||||
});
|
||||
await runPrepared({
|
||||
opts: { isHeartbeat: true },
|
||||
ctx: {
|
||||
Body: heartbeatPrompt,
|
||||
RawBody: heartbeatPrompt,
|
||||
CommandBody: heartbeatPrompt,
|
||||
...createProviderSurface(provider),
|
||||
ChatType: "direct",
|
||||
OriginatingChannel: "discord",
|
||||
OriginatingTo: "discord:channel-123",
|
||||
},
|
||||
sessionCtx: {
|
||||
Body: heartbeatPrompt,
|
||||
BodyStripped: heartbeatPrompt,
|
||||
...createProviderSurface(provider),
|
||||
ChatType: "direct",
|
||||
OriginatingChannel: "discord",
|
||||
OriginatingTo: "discord:channel-123",
|
||||
},
|
||||
});
|
||||
|
||||
const call = requireLastRunReplyAgentCall();
|
||||
expect(call?.commandBody).toContain(heartbeatPrompt);
|
||||
expect(call?.followupRun.prompt).toContain(heartbeatPrompt);
|
||||
expect(call?.transcriptCommandBody).toBe("[OpenClaw heartbeat poll]");
|
||||
expect(call?.followupRun.transcriptPrompt).toBe("[OpenClaw heartbeat poll]");
|
||||
expect(call?.followupRun.userTurnTranscriptRecorder?.message).toMatchObject({
|
||||
provenance: { kind: "internal_system", sourceTool: "heartbeat" },
|
||||
});
|
||||
});
|
||||
const call = requireLastRunReplyAgentCall();
|
||||
expect(call?.commandBody).toContain(heartbeatPrompt);
|
||||
expect(call?.followupRun.prompt).toContain(heartbeatPrompt);
|
||||
expect(call?.followupRun.prompt).not.toContain(syntheticConversationInfo);
|
||||
expect(buildInboundUserContextPrefix).not.toHaveBeenCalled();
|
||||
expect(call?.sessionCtx).toMatchObject({
|
||||
OriginatingChannel: "discord",
|
||||
OriginatingTo: "discord:channel-123",
|
||||
});
|
||||
expect(call?.transcriptCommandBody).toBe("[OpenClaw heartbeat poll]");
|
||||
expect(call?.followupRun.transcriptPrompt).toBe("[OpenClaw heartbeat poll]");
|
||||
expect(call?.followupRun.userTurnTranscriptRecorder?.message).toMatchObject({
|
||||
provenance: { kind: "internal_system", sourceTool: "heartbeat" },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps active goal context out of background heartbeat turns", async () => {
|
||||
const sessionEntry: SessionEntry = {
|
||||
@@ -2990,11 +3006,7 @@ describe("runPreparedReply media-only handling", () => {
|
||||
sessionStore: { "session-key": sessionEntry },
|
||||
});
|
||||
|
||||
expect(buildInboundUserContextPrefix).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
undefined,
|
||||
);
|
||||
expect(buildInboundUserContextPrefix).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses persisted Discord chat metadata for system-event CLI static prompt identity", async () => {
|
||||
|
||||
Reference in New Issue
Block a user