From 3706ebed331fc647a5b5f5d800cd5ccd0800e4a6 Mon Sep 17 00:00:00 2001 From: Ayaan Zaidi Date: Fri, 3 Jul 2026 17:20:52 -0700 Subject: [PATCH] fix(agents): keep mention state and group intro out of session-stable prompt hash --- src/agents/cli-runner/prepare.test.ts | 92 ++++++++++++ src/agents/prompt-composition.test.ts | 6 +- .../reply/get-reply-run.media-only.test.ts | 133 +++++++++++++++++- src/auto-reply/reply/get-reply-run.ts | 4 +- src/auto-reply/reply/groups.test.ts | 14 +- src/auto-reply/reply/groups.ts | 6 - src/auto-reply/reply/inbound-meta.test.ts | 30 ++++ src/auto-reply/reply/inbound-meta.ts | 5 + .../reply/strip-inbound-meta.test.ts | 14 ++ .../agents/prompt-composition-scenarios.ts | 10 +- 10 files changed, 291 insertions(+), 23 deletions(-) diff --git a/src/agents/cli-runner/prepare.test.ts b/src/agents/cli-runner/prepare.test.ts index d28bd3c49fe7..8ad05ff4728e 100644 --- a/src/agents/cli-runner/prepare.test.ts +++ b/src/agents/cli-runner/prepare.test.ts @@ -6,6 +6,7 @@ import path from "node:path"; import { CURRENT_SESSION_VERSION } from "openclaw/plugin-sdk/agent-sessions"; import { Type } from "typebox"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { buildGroupChatContext, buildGroupIntro } from "../../auto-reply/reply/groups.js"; import type { ChannelPlugin } from "../../channels/plugins/types.plugin.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { registerLegacyContextEngine } from "../../context-engine/legacy.registration.js"; @@ -2117,6 +2118,97 @@ describe("shouldSkipLocalCliCredentialEpoch", () => { }, ); + it("reuses CLI session bindings across explicit mention toggles with stable group prompt facts", async () => { + const { dir, sessionFile } = createSessionFile(); + try { + const baseGroupCtx = { + ChatType: "group", + Provider: "telegram", + BotUsername: "SirPinchALotBot", + } as const; + const mentionedStaticPrompt = [ + buildGroupChatContext({ + sessionCtx: { + ...baseGroupCtx, + ExplicitlyMentionedBot: true, + }, + sourceReplyDeliveryMode: "automatic", + silentReplyPolicy: "allow", + silentToken: "NO_REPLY", + }), + buildGroupIntro({ + defaultActivation: "mention", + }), + ].join("\n\n"); + const unmentionedStaticPrompt = [ + buildGroupChatContext({ + sessionCtx: { + ...baseGroupCtx, + ExplicitlyMentionedBot: false, + }, + sourceReplyDeliveryMode: "automatic", + silentReplyPolicy: "allow", + silentToken: "NO_REPLY", + }), + buildGroupIntro({ + defaultActivation: "mention", + }), + ].join("\n\n"); + expect(unmentionedStaticPrompt).toBe(mentionedStaticPrompt); + + const first = await prepareCliRunContext({ + sessionId: "session-test", + sessionKey: "agent:main:telegram:group:chat123", + sessionFile, + workspaceDir: dir, + prompt: "first ask", + provider: "test-cli", + model: "test-model", + timeoutMs: 1_000, + runId: "run-test-mention-binding-a", + extraSystemPrompt: [ + "The incoming message explicitly mentions your channel identity @SirPinchALotBot.", + mentionedStaticPrompt, + ].join("\n\n"), + sourceReplyDeliveryMode: "automatic", + cliSessionBindingFacts: { + extraSystemPromptStatic: mentionedStaticPrompt, + sourceReplyDeliveryMode: "automatic", + }, + config: createCliBackendConfig(), + }); + const second = await prepareCliRunContext({ + sessionId: "session-test", + sessionKey: "agent:main:telegram:group:chat123", + sessionFile, + workspaceDir: dir, + prompt: "second ask", + provider: "test-cli", + model: "test-model", + timeoutMs: 1_000, + runId: "run-test-mention-binding-b", + extraSystemPrompt: unmentionedStaticPrompt, + sourceReplyDeliveryMode: "automatic", + cliSessionBindingFacts: { + extraSystemPromptStatic: unmentionedStaticPrompt, + sourceReplyDeliveryMode: "automatic", + }, + cliSessionBinding: { + sessionId: "cli-session", + extraSystemPromptHash: first.extraSystemPromptHash, + messageToolPolicyHash: first.messageToolPolicyHash, + cwdHash: hashCliSessionText(dir), + }, + config: createCliBackendConfig(), + }); + + expect(second.extraSystemPromptHash).toBe(first.extraSystemPromptHash); + expect(second.reusableCliSession).toEqual({ sessionId: "cli-session" }); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + it("reuses CLI session bindings across owner sender flips with stable prompt tool scope", async () => { const { dir, sessionFile } = createSessionFile(); try { diff --git a/src/agents/prompt-composition.test.ts b/src/agents/prompt-composition.test.ts index aeb1715ebf59..4706971986f0 100644 --- a/src/agents/prompt-composition.test.ts +++ b/src/agents/prompt-composition.test.ts @@ -76,7 +76,7 @@ describe("prompt composition invariants", () => { expect(always.bodyPrompt).toContain("[Bootstrap truncation warning]"); }); - it("keeps the group auto-reply prompt dynamic only across the first-turn intro boundary", () => { + it("keeps the group auto-reply prompt stable across the first-turn intro boundary", () => { const groupScenario = getScenario(fixture, "auto-reply-group"); const first = getTurn(groupScenario, "t1"); const steady = getTurn(groupScenario, "t2"); @@ -89,10 +89,10 @@ describe("prompt composition invariants", () => { expect(first.systemPrompt).not.toContain("## Silent Replies"); expect(steady.systemPrompt).toContain("You are in a Slack group chat."); expect(steady.systemPrompt).toContain("prefer delegating bounded side investigations early"); + expect(steady.systemPrompt).toContain("Activation: trigger-only"); expect(steady.systemPrompt).toContain('reply with exactly "NO_REPLY"'); expect(steady.systemPrompt).not.toContain("## Silent Replies"); - expect(steady.systemPrompt).not.toContain("Activation: trigger-only"); - expect(first.systemPrompt).not.toBe(steady.systemPrompt); + expect(first.systemPrompt).toBe(steady.systemPrompt); expect(steady.systemPrompt).toBe(eventTurn.systemPrompt); }); diff --git a/src/auto-reply/reply/get-reply-run.media-only.test.ts b/src/auto-reply/reply/get-reply-run.media-only.test.ts index eb19dfea8bfe..e0eef908304c 100644 --- a/src/auto-reply/reply/get-reply-run.media-only.test.ts +++ b/src/auto-reply/reply/get-reply-run.media-only.test.ts @@ -143,6 +143,7 @@ let drainFormattedSystemEvents: typeof import("./session-system-events.js").drai let applySessionHints: typeof import("./body.js").applySessionHints; let resolveTypingMode: typeof import("./typing-mode.js").resolveTypingMode; let buildDirectChatContext: typeof import("./groups.js").buildDirectChatContext; +let buildGroupIntro: typeof import("./groups.js").buildGroupIntro; let buildGroupChatContext: typeof import("./groups.js").buildGroupChatContext; let buildInboundUserContextPrefix: typeof import("./inbound-meta.js").buildInboundUserContextPrefix; let resolveInboundUserContextPromptJoiner: typeof import("./inbound-meta.js").resolveInboundUserContextPromptJoiner; @@ -292,7 +293,8 @@ describe("runPreparedReply media-only handling", () => { ({ drainFormattedSystemEvents } = await import("./session-system-events.js")); ({ applySessionHints } = await import("./body.js")); ({ resolveTypingMode } = await import("./typing-mode.js")); - ({ buildDirectChatContext, buildGroupChatContext } = await import("./groups.js")); + ({ buildDirectChatContext, buildGroupIntro, buildGroupChatContext } = + await import("./groups.js")); ({ buildInboundUserContextPrefix, resolveInboundUserContextPromptJoiner } = await import("./inbound-meta.js")); ({ testing: replyRunTesting, getActiveReplyRunCount } = @@ -305,6 +307,7 @@ describe("runPreparedReply media-only handling", () => { updateAmbientTranscriptWatermarkMock.mockClear(); vi.clearAllMocks(); vi.mocked(buildDirectChatContext).mockReturnValue(""); + vi.mocked(buildGroupIntro).mockReturnValue(""); vi.mocked(buildGroupChatContext).mockReturnValue(""); vi.mocked(buildInboundUserContextPrefix).mockReturnValue(""); vi.mocked(resolveInboundUserContextPromptJoiner).mockReturnValue(undefined); @@ -2792,6 +2795,134 @@ describe("runPreparedReply media-only handling", () => { expect(secondRun.cliSessionBindingFacts).toEqual(firstRun.cliSessionBindingFacts); }); + it("keeps explicit mention state in user context and out of CLI binding facts", async () => { + vi.mocked(buildGroupChatContext).mockReturnValue("group:telegram:group:automatic"); + + await runPreparedReply( + baseParams({ + opts: { + sourceReplyDeliveryMode: "automatic", + sessionPromptSourceReplyDeliveryMode: "automatic", + }, + isNewSession: false, + systemSent: true, + ctx: { + Body: "@SirPinchALotBot check this", + RawBody: "@SirPinchALotBot check this", + CommandBody: "@SirPinchALotBot check this", + Provider: "telegram", + Surface: "telegram", + ChatType: "group", + BotUsername: "SirPinchALotBot", + ExplicitlyMentionedBot: true, + }, + sessionCtx: { + Body: "@SirPinchALotBot check this", + BodyStripped: "@SirPinchALotBot check this", + Provider: "telegram", + Surface: "telegram", + ChatType: "group", + BotUsername: "SirPinchALotBot", + ExplicitlyMentionedBot: true, + }, + }), + ); + + const run = requireRunReplyAgentCall(0).followupRun.run; + const inboundCtx = requireMockCallArg( + vi.mocked(buildInboundUserContextPrefix), + "inbound user context", + ) as { ExplicitlyMentionedBot?: boolean; BotUsername?: string }; + expect(inboundCtx.ExplicitlyMentionedBot).toBe(true); + expect(inboundCtx.BotUsername).toBe("SirPinchALotBot"); + expect(run.extraSystemPromptStatic).toBe("group:telegram:group:automatic"); + expect(run.cliSessionBindingFacts).toEqual({ + extraSystemPromptStatic: "group:telegram:group:automatic", + sourceReplyDeliveryMode: "automatic", + }); + }); + + it("keeps group intro in the session-stable CLI prompt after turn one", async () => { + vi.mocked(buildGroupChatContext).mockReturnValue("group:telegram:group:automatic"); + vi.mocked(buildGroupIntro).mockReturnValue("intro:mention"); + const sessionEntry: SessionEntry = { + sessionId: "session-telegram-group", + updatedAt: 1, + systemSent: true, + chatType: "group", + channel: "telegram", + lastChannel: "telegram", + lastTo: "-100123", + origin: { + provider: "telegram", + surface: "telegram", + chatType: "group", + to: "-100123", + }, + }; + + await runPreparedReply( + baseParams({ + opts: { + sourceReplyDeliveryMode: "automatic", + sessionPromptSourceReplyDeliveryMode: "automatic", + }, + isNewSession: true, + systemSent: false, + sessionEntry, + ctx: { + Body: "@bot first", + RawBody: "@bot first", + CommandBody: "@bot first", + Provider: "telegram", + Surface: "telegram", + ChatType: "group", + }, + sessionCtx: { + Body: "@bot first", + BodyStripped: "@bot first", + Provider: "telegram", + Surface: "telegram", + ChatType: "group", + }, + }), + ); + await runPreparedReply( + baseParams({ + opts: { + sourceReplyDeliveryMode: "automatic", + sessionPromptSourceReplyDeliveryMode: "automatic", + }, + isNewSession: false, + systemSent: true, + sessionEntry, + ctx: { + Body: "second", + RawBody: "second", + CommandBody: "second", + Provider: "telegram", + Surface: "telegram", + ChatType: "group", + }, + sessionCtx: { + Body: "second", + BodyStripped: "second", + Provider: "telegram", + Surface: "telegram", + ChatType: "group", + }, + }), + ); + + const firstRun = requireRunReplyAgentCall(0).followupRun.run; + const secondRun = requireRunReplyAgentCall(1).followupRun.run; + expect(firstRun.extraSystemPromptStatic).toBe( + "group:telegram:group:automatic\n\nintro:mention", + ); + expect(secondRun.extraSystemPromptStatic).toBe(firstRun.extraSystemPromptStatic); + expect(secondRun.cliSessionBindingFacts).toEqual(firstRun.cliSessionBindingFacts); + }); + it.each([ ["/new", "new"], ["/reset", "reset"], diff --git a/src/auto-reply/reply/get-reply-run.ts b/src/auto-reply/reply/get-reply-run.ts index d9127847c82d..8055ed3be45e 100644 --- a/src/auto-reply/reply/get-reply-run.ts +++ b/src/auto-reply/reply/get-reply-run.ts @@ -631,8 +631,8 @@ export async function runPreparedReply( silentToken: SILENT_REPLY_TOKEN, }) : ""; - // Behavioral intro (activation mode, lurking, etc.) only on first turn / activation needed - const groupIntro = shouldInjectGroupIntro + // Claude CLI fixes the system prompt at session creation; group intro must stay session-stable. + const groupIntro = isGroupChat ? buildGroupIntro({ sessionEntry, defaultActivation, diff --git a/src/auto-reply/reply/groups.test.ts b/src/auto-reply/reply/groups.test.ts index d279ff9cec80..651e1b130a81 100644 --- a/src/auto-reply/reply/groups.test.ts +++ b/src/auto-reply/reply/groups.test.ts @@ -144,8 +144,8 @@ describe("group runtime loading", () => { expect(disallowed).not.toContain("Never say that you are staying quiet"); }); - it("binds an explicitly mentioned channel handle to the current assistant identity", () => { - const context = groups.buildGroupChatContext({ + it("keeps per-message mention state out of stable group context", () => { + const mentioned = groups.buildGroupChatContext({ sessionCtx: { ChatType: "group", Provider: "telegram", @@ -156,19 +156,19 @@ describe("group runtime loading", () => { silentReplyPolicy: "allow", }); - expect(context).toContain("explicitly mentions your channel identity @SirPinchALotBot"); - expect(context).toContain("Treat that mention as addressed to you"); - const notExplicit = groups.buildGroupChatContext({ sessionCtx: { ChatType: "group", Provider: "telegram", - BotUsername: "kesslerAIBot", + BotUsername: "SirPinchALotBot", + ExplicitlyMentionedBot: false, }, silentToken: "NO_REPLY", silentReplyPolicy: "allow", }); - expect(notExplicit).not.toContain("channel identity @kesslerAIBot"); + + expect(mentioned).toBe(notExplicit); + expect(mentioned).not.toContain("channel identity @SirPinchALotBot"); }); it("uses channel wording when the authoritative chat type is channel", () => { diff --git a/src/auto-reply/reply/groups.ts b/src/auto-reply/reply/groups.ts index 7e60082445f4..dbdfab9fe911 100644 --- a/src/auto-reply/reply/groups.ts +++ b/src/auto-reply/reply/groups.ts @@ -241,17 +241,11 @@ export function buildGroupChatContext(params: { const providerLabel = resolveProviderLabel(params.sessionCtx.Provider); const provider = normalizeOptionalLowercaseString(params.sessionCtx.Provider); const messageToolOnly = params.sourceReplyDeliveryMode === "message_tool_only"; - const botUsername = normalizeOptionalString(params.sessionCtx.BotUsername); const sharedChatNoun = resolveSharedChatNoun(params.sessionCtx.ChatType); const destinationLabel = sharedChatNoun === "channel" ? "this channel" : "this group chat"; const lines: string[] = []; lines.push(`You are in a ${providerLabel} ${sharedChatNoun}.`); - if (params.sessionCtx.ExplicitlyMentionedBot === true && botUsername) { - lines.push( - `The incoming message explicitly mentions your channel identity @${botUsername}. Treat that mention as addressed to you, even if your persona name differs.`, - ); - } if (messageToolOnly) { lines.push( `Normal final replies are private and are not automatically sent to ${destinationLabel}. To post visible output here, use the message tool with action=send; the target defaults to ${destinationLabel}.`, diff --git a/src/auto-reply/reply/inbound-meta.test.ts b/src/auto-reply/reply/inbound-meta.test.ts index f67ff5af06ab..03d584641cab 100644 --- a/src/auto-reply/reply/inbound-meta.test.ts +++ b/src/auto-reply/reply/inbound-meta.test.ts @@ -160,6 +160,23 @@ describe("buildInboundMetaSystemPrompt", () => { expect(payload["flags"]).toBeUndefined(); }); + it("keeps explicit bot mentions out of the system metadata", () => { + const prompt = buildInboundMetaSystemPrompt({ + OriginatingTo: "telegram:-1001249586642", + OriginatingChannel: "telegram", + Provider: "telegram", + Surface: "telegram", + ChatType: "group", + BotUsername: "SirPinchALotBot", + ExplicitlyMentionedBot: true, + } as TemplateContext); + + const payload = parseInboundMetaPayload(prompt); + expect(payload["flags"]).toBeUndefined(); + expect(prompt).not.toContain("SirPinchALotBot"); + expect(prompt).not.toContain("explicitly mentions your channel identity"); + }); + it("omits sender_id when blank", () => { const prompt = buildInboundMetaSystemPrompt({ MessageSid: "458", @@ -714,6 +731,19 @@ describe("buildInboundUserContextPrefix", () => { expect(conversationInfo["history_count"]).toBe(1); }); + it("carries explicit bot mentions in current-turn user context", () => { + const text = buildInboundUserContextPrefix({ + ChatType: "group", + BotUsername: "SirPinchALotBot", + ExplicitlyMentionedBot: true, + } as TemplateContext); + + const conversationInfo = parseConversationInfoPayload(text); + expect(conversationInfo["explicitly_mentioned_bot"]).toBe(true); + expect(text).toContain("explicitly mentions your channel identity @SirPinchALotBot"); + expect(text).toContain("Treat that mention as addressed to you"); + }); + it("trims sender_id in conversation info", () => { const text = buildInboundUserContextPrefix({ ChatType: "group", diff --git a/src/auto-reply/reply/inbound-meta.ts b/src/auto-reply/reply/inbound-meta.ts index ba40a35728b6..c7556a284021 100644 --- a/src/auto-reply/reply/inbound-meta.ts +++ b/src/auto-reply/reply/inbound-meta.ts @@ -575,6 +575,7 @@ export function buildInboundUserContextPrefix( e164: normalizePromptMetadataString(ctx.SenderE164), is_bot: typeof ctx.SenderIsBot === "boolean" ? ctx.SenderIsBot : undefined, }; + const botUsername = normalizePromptMetadataString(ctx.BotUsername); // Keep volatile conversation/message identifiers in the user-role block so the system // prompt stays byte-stable across task-scoped sessions and reply turns. @@ -605,6 +606,10 @@ export function buildInboundUserContextPrefix( topic_name: normalizePromptMetadataString(ctx.TopicName) ?? undefined, is_forum: ctx.IsForum === true ? true : undefined, ...buildConversationMentionMetadataPayload(ctx, isDirect), + explicit_bot_mention_note: + ctx.ExplicitlyMentionedBot === true && botUsername + ? `The incoming message explicitly mentions your channel identity @${botUsername}. Treat that mention as addressed to you, even if your persona name differs.` + : undefined, has_reply_context: replyChainPayload.length > 0 || sanitizePromptBody(ctx.ReplyToBody) ? true : undefined, has_forwarded_context: normalizePromptMetadataString(ctx.ForwardedFrom) ? true : undefined, diff --git a/src/auto-reply/reply/strip-inbound-meta.test.ts b/src/auto-reply/reply/strip-inbound-meta.test.ts index 958ce30a0ce4..7c2d8692e338 100644 --- a/src/auto-reply/reply/strip-inbound-meta.test.ts +++ b/src/auto-reply/reply/strip-inbound-meta.test.ts @@ -72,6 +72,20 @@ describe("stripInboundMetadata", () => { expect(stripInboundMetadata(input)).toBe("What is the weather today?"); }); + it("strips explicit bot mention notes with conversation info", () => { + const input = `Conversation info (untrusted metadata): +\`\`\`json +{ + "explicitly_mentioned_bot": true, + "explicit_bot_mention_note": "The incoming message explicitly mentions your channel identity @SirPinchALotBot. Treat that mention as addressed to you, even if your persona name differs." +} +\`\`\` + +Actual user message`; + + expect(stripInboundMetadata(input)).toBe("Actual user message"); + }); + it("strips multiple chained metadata blocks", () => { const input = `${CONV_BLOCK}\n\n${SENDER_BLOCK}\n\nCan you help me?`; expect(stripInboundMetadata(input)).toBe("Can you help me?"); diff --git a/test/helpers/agents/prompt-composition-scenarios.ts b/test/helpers/agents/prompt-composition-scenarios.ts index 1c45468fe0de..427e139ced3c 100644 --- a/test/helpers/agents/prompt-composition-scenarios.ts +++ b/test/helpers/agents/prompt-composition-scenarios.ts @@ -347,11 +347,11 @@ function createGroupScenario(workspaceDir: string): PromptScenario { return { scenario: "auto-reply-group", focus: "Group chat bootstrap, steady state, and runtime event turns", - expectedStableSystemAfterTurnIds: ["t3"], + expectedStableSystemAfterTurnIds: ["t2", "t3"], turns: [ { id: "t1", - label: "First group turn with one-time intro", + label: "First group turn with session-stable intro", systemPrompt: buildAutoReplySystemPrompt({ workspaceDir, sessionCtx: { @@ -372,7 +372,7 @@ function createGroupScenario(workspaceDir: string): PromptScenario { }, body: "Can you investigate this issue?", }), - notes: ["Expected first-turn bootstrap churn", "Not steady-state"], + notes: ["Group intro belongs to the session-stable system prompt"], }, { id: "t2", @@ -389,6 +389,7 @@ function createGroupScenario(workspaceDir: string): PromptScenario { ], }, includeGroupChatContext: true, + includeGroupIntro: true, }), bodyPrompt: buildAutoReplyBody({ ctx: { @@ -402,7 +403,7 @@ function createGroupScenario(workspaceDir: string): PromptScenario { }, body: "Give a short update.", }), - notes: ["One-time intro gone", "Should settle afterward"], + notes: ["Group intro remains stable after turn one"], }, { id: "t3", @@ -419,6 +420,7 @@ function createGroupScenario(workspaceDir: string): PromptScenario { ], }, includeGroupChatContext: true, + includeGroupIntro: true, }), bodyPrompt: buildAutoReplyBody({ ctx: {