diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 252cb4cc847e..dae656695177 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -601,7 +601,6 @@ src/auto-reply/command-control.test.ts src/auto-reply/commands-registry.shared.ts src/auto-reply/inbound.test.ts src/auto-reply/reply/abort.test.ts -src/auto-reply/reply/agent-runner-execution.test.ts src/auto-reply/reply/agent-runner-memory.test.ts src/auto-reply/reply/agent-runner-memory.ts src/auto-reply/reply/agent-runner-payloads.test.ts diff --git a/src/auto-reply/reply/agent-runner-auto-fallback.test.ts b/src/auto-reply/reply/agent-runner-auto-fallback.test.ts new file mode 100644 index 000000000000..07f04d4a5a2f --- /dev/null +++ b/src/auto-reply/reply/agent-runner-auto-fallback.test.ts @@ -0,0 +1,427 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { SessionEntry } from "../../config/sessions.js"; +import type { FollowupRun } from "./queue.js"; + +const state = vi.hoisted(() => ({ + updateSessionEntryMock: vi.fn(), +})); + +vi.mock("../../config/sessions/session-accessor.js", () => ({ + updateSessionEntry: (...args: unknown[]) => state.updateSessionEntryMock(...args), +})); + +import { clearRecoveredAutoFallbackPrimaryProbeSelection } from "./agent-runner-auto-fallback.js"; + +describe("clearRecoveredAutoFallbackPrimaryProbeSelection", () => { + beforeEach(() => { + state.updateSessionEntryMock.mockReset(); + }); + + it("refreshes the local selection when the persisted comparison rejects the probe", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + }; + const staleAutoEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: probe.fallbackProvider, + modelOverride: probe.fallbackModel, + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: probe.provider, + modelOverrideFallbackOriginModel: probe.model, + }; + const newerUserEntry: SessionEntry = { + sessionId: "newer-session", + updatedAt: 2, + providerOverride: "openai", + modelOverride: "gpt-5.5", + modelOverrideSource: "user", + }; + const activeSessionStore = { main: staleAutoEntry }; + state.updateSessionEntryMock.mockImplementationOnce( + async (_scope: unknown, update: (entry: SessionEntry) => unknown) => { + expect(await update(newerUserEntry)).toBeNull(); + return null; + }, + ); + + await clearRecoveredAutoFallbackPrimaryProbeSelection({ + run: { + provider: probe.provider, + model: probe.model, + autoFallbackPrimaryProbe: probe, + } as FollowupRun["run"], + provider: probe.provider, + model: probe.model, + sessionKey: "main", + activeSessionStore, + getActiveSessionEntry: () => staleAutoEntry, + storePath: "/tmp/sessions.sqlite", + }); + + expect(activeSessionStore.main).toBe(newerUserEntry); + expect(activeSessionStore.main).toMatchObject({ + sessionId: "newer-session", + providerOverride: "openai", + modelOverride: "gpt-5.5", + modelOverrideSource: "user", + }); + }); + + it("preserves an identically reselected persisted fallback generation", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + }; + const staleAutoEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: probe.fallbackProvider, + modelOverride: probe.fallbackModel, + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: probe.provider, + modelOverrideFallbackOriginModel: probe.model, + }; + const newerAutoEntry: SessionEntry = { + ...staleAutoEntry, + updatedAt: 2, + }; + const activeSessionStore = { main: staleAutoEntry }; + state.updateSessionEntryMock.mockImplementationOnce( + async (_scope: unknown, update: (entry: SessionEntry) => unknown) => { + expect(await update(newerAutoEntry)).toBeNull(); + return null; + }, + ); + + await clearRecoveredAutoFallbackPrimaryProbeSelection({ + run: { + provider: probe.provider, + model: probe.model, + autoFallbackPrimaryProbe: probe, + } as FollowupRun["run"], + provider: probe.provider, + model: probe.model, + sessionKey: "main", + activeSessionStore, + getActiveSessionEntry: () => staleAutoEntry, + storePath: "/tmp/sessions.sqlite", + }); + + expect(activeSessionStore.main).toBe(newerAutoEntry); + expect(activeSessionStore.main).toMatchObject({ + providerOverride: probe.fallbackProvider, + modelOverride: probe.fallbackModel, + modelOverrideSource: "auto", + updatedAt: 2, + }); + }); + + it("keeps a newer local selection installed while persistence is pending", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + }; + const staleAutoEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: probe.fallbackProvider, + modelOverride: probe.fallbackModel, + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: probe.provider, + modelOverrideFallbackOriginModel: probe.model, + }; + const newerUserEntry: SessionEntry = { + sessionId: "session", + updatedAt: 2, + providerOverride: "openai", + modelOverride: "gpt-5.5", + modelOverrideSource: "user", + }; + const activeSessionStore = { main: staleAutoEntry }; + state.updateSessionEntryMock.mockImplementationOnce( + async (_scope: unknown, update: (entry: SessionEntry) => unknown) => { + const persistedEntry = { ...staleAutoEntry }; + const patch = await update(persistedEntry); + activeSessionStore.main = newerUserEntry; + return { ...persistedEntry, ...(patch as Partial) }; + }, + ); + + await clearRecoveredAutoFallbackPrimaryProbeSelection({ + run: { + provider: probe.provider, + model: probe.model, + autoFallbackPrimaryProbe: probe, + } as FollowupRun["run"], + provider: probe.provider, + model: probe.model, + sessionKey: "main", + activeSessionStore, + getActiveSessionEntry: () => staleAutoEntry, + storePath: "/tmp/sessions.sqlite", + }); + + expect(activeSessionStore.main).toBe(newerUserEntry); + }); + + it("preserves an in-place auth selection while applying persisted probe cleanup", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + }; + const staleAutoEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: probe.fallbackProvider, + modelOverride: probe.fallbackModel, + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: probe.provider, + modelOverrideFallbackOriginModel: probe.model, + authProfileOverride: "openai:fallback", + authProfileOverrideSource: "auto", + }; + const activeSessionStore = { main: staleAutoEntry }; + state.updateSessionEntryMock.mockImplementationOnce( + async (_scope: unknown, update: (entry: SessionEntry) => unknown) => { + const persistedEntry = { ...staleAutoEntry }; + const patch = await update(persistedEntry); + Object.assign(staleAutoEntry, { + authProfileOverrideSource: "user", + updatedAt: 3, + }); + return { ...persistedEntry, ...(patch as Partial) }; + }, + ); + + await clearRecoveredAutoFallbackPrimaryProbeSelection({ + run: { + provider: probe.provider, + model: probe.model, + autoFallbackPrimaryProbe: probe, + } as FollowupRun["run"], + provider: probe.provider, + model: probe.model, + sessionKey: "main", + activeSessionStore, + getActiveSessionEntry: () => staleAutoEntry, + storePath: "/tmp/sessions.sqlite", + }); + + expect(activeSessionStore.main).not.toBe(staleAutoEntry); + expect(activeSessionStore.main).toMatchObject({ + authProfileOverride: "openai:fallback", + authProfileOverrideSource: "user", + providerOverride: probe.fallbackProvider, + modelOverride: probe.fallbackModel, + }); + expect(activeSessionStore.main.updatedAt).toBeGreaterThanOrEqual(3); + }); + + it("preserves a value-identical same-session cache replacement", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + }; + const staleAutoEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: probe.fallbackProvider, + modelOverride: probe.fallbackModel, + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: probe.provider, + modelOverrideFallbackOriginModel: probe.model, + authProfileOverride: "openai:fallback", + authProfileOverrideSource: "auto", + }; + const replacementEntry: SessionEntry = { + ...staleAutoEntry, + updatedAt: 3, + }; + const activeSessionStore = { main: staleAutoEntry }; + state.updateSessionEntryMock.mockImplementationOnce( + async (_scope: unknown, update: (entry: SessionEntry) => unknown) => { + const persistedEntry = { ...staleAutoEntry }; + const patch = await update(persistedEntry); + activeSessionStore.main = replacementEntry; + return { ...persistedEntry, ...(patch as Partial) }; + }, + ); + + await clearRecoveredAutoFallbackPrimaryProbeSelection({ + run: { + provider: probe.provider, + model: probe.model, + autoFallbackPrimaryProbe: probe, + } as FollowupRun["run"], + provider: probe.provider, + model: probe.model, + sessionKey: "main", + activeSessionStore, + getActiveSessionEntry: () => staleAutoEntry, + storePath: "/tmp/sessions.sqlite", + }); + + expect(activeSessionStore.main).toBe(replacementEntry); + expect(activeSessionStore.main).toMatchObject({ + authProfileOverride: "openai:fallback", + authProfileOverrideSource: "auto", + providerOverride: probe.fallbackProvider, + modelOverride: probe.fallbackModel, + updatedAt: 3, + }); + }); + + it("preserves a same-session cache replacement when persistence has no row", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + }; + const staleAutoEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: probe.fallbackProvider, + modelOverride: probe.fallbackModel, + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: probe.provider, + modelOverrideFallbackOriginModel: probe.model, + }; + const replacementEntry: SessionEntry = { + ...staleAutoEntry, + updatedAt: 2, + }; + const activeSessionStore = { main: staleAutoEntry }; + state.updateSessionEntryMock.mockImplementationOnce(async () => { + activeSessionStore.main = replacementEntry; + return undefined; + }); + + await clearRecoveredAutoFallbackPrimaryProbeSelection({ + run: { + provider: probe.provider, + model: probe.model, + autoFallbackPrimaryProbe: probe, + } as FollowupRun["run"], + provider: probe.provider, + model: probe.model, + sessionKey: "main", + activeSessionStore, + getActiveSessionEntry: () => staleAutoEntry, + storePath: "/tmp/sessions.sqlite", + }); + + expect(activeSessionStore.main).toBe(replacementEntry); + }); + + it("preserves an in-place cache update when persistence has no row", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + }; + const staleAutoEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: probe.fallbackProvider, + modelOverride: probe.fallbackModel, + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: probe.provider, + modelOverrideFallbackOriginModel: probe.model, + authProfileOverride: "openai:fallback", + authProfileOverrideSource: "auto", + }; + const activeSessionStore = { main: staleAutoEntry }; + state.updateSessionEntryMock.mockImplementationOnce(async () => { + Object.assign(staleAutoEntry, { + authProfileOverrideSource: "user", + updatedAt: 2, + }); + return undefined; + }); + + await clearRecoveredAutoFallbackPrimaryProbeSelection({ + run: { + provider: probe.provider, + model: probe.model, + autoFallbackPrimaryProbe: probe, + } as FollowupRun["run"], + provider: probe.provider, + model: probe.model, + sessionKey: "main", + activeSessionStore, + getActiveSessionEntry: () => staleAutoEntry, + storePath: "/tmp/sessions.sqlite", + }); + + expect(activeSessionStore.main).toBe(staleAutoEntry); + expect(activeSessionStore.main).toMatchObject({ + authProfileOverride: "openai:fallback", + authProfileOverrideSource: "user", + updatedAt: 2, + }); + }); + + it("preserves an in-place nested cache update", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + }; + const staleAutoEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: probe.fallbackProvider, + modelOverride: probe.fallbackModel, + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: probe.provider, + modelOverrideFallbackOriginModel: probe.model, + cliSessionBindings: { + codex: { sessionId: "old-cli-session" }, + }, + }; + const activeSessionStore = { main: staleAutoEntry }; + state.updateSessionEntryMock.mockImplementationOnce( + async (_scope: unknown, update: (entry: SessionEntry) => unknown) => { + const persistedEntry = structuredClone(staleAutoEntry); + const patch = await update(persistedEntry); + staleAutoEntry.cliSessionBindings!.codex!.sessionId = "new-cli-session"; + staleAutoEntry.updatedAt = 2; + return { ...persistedEntry, ...(patch as Partial) }; + }, + ); + + await clearRecoveredAutoFallbackPrimaryProbeSelection({ + run: { + provider: probe.provider, + model: probe.model, + autoFallbackPrimaryProbe: probe, + } as FollowupRun["run"], + provider: probe.provider, + model: probe.model, + sessionKey: "main", + activeSessionStore, + getActiveSessionEntry: () => staleAutoEntry, + storePath: "/tmp/sessions.sqlite", + }); + + expect(activeSessionStore.main).not.toBe(staleAutoEntry); + expect(activeSessionStore.main.cliSessionBindings?.codex?.sessionId).toBe("new-cli-session"); + expect(activeSessionStore.main.updatedAt).toBeGreaterThanOrEqual(2); + expect(activeSessionStore.main.modelOverride).toBeUndefined(); + }); +}); diff --git a/src/auto-reply/reply/agent-runner-auto-fallback.ts b/src/auto-reply/reply/agent-runner-auto-fallback.ts index 3b33de28fa02..9dee80993cba 100644 --- a/src/auto-reply/reply/agent-runner-auto-fallback.ts +++ b/src/auto-reply/reply/agent-runner-auto-fallback.ts @@ -1,12 +1,30 @@ +import { isDeepStrictEqual } from "node:util"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { + clearAutoFallbackPrimaryProbeSelection, + entryMatchesAutoFallbackPrimaryProbe, hasSessionAutoModelFallbackProvenance, resolveAutoFallbackPrimaryProbe, } from "../../agents/agent-scope.js"; import { resolvePersistedOverrideModelRef } from "../../agents/model-selection.js"; import type { SessionEntry } from "../../config/sessions.js"; +import { updateSessionEntry } from "../../config/sessions/session-accessor.js"; +import { mergeSessionSnapshotChanges } from "../../config/sessions/session-snapshot-merge.js"; +import { shouldPreserveUserFacingSessionStateForInputProvenance } from "../../sessions/input-provenance.js"; import type { FollowupRun } from "./queue.js"; +function sessionEntryMatchesSnapshot(entry: SessionEntry, snapshot: SessionEntry): boolean { + return isDeepStrictEqual(entry, snapshot); +} + +function sessionEntryOnlyUpdatedAtChanged(entry: SessionEntry, snapshot: SessionEntry): boolean { + if (entry.updatedAt === snapshot.updatedAt) { + return false; + } + const entryWithoutUpdatedAt = { ...entry, updatedAt: snapshot.updatedAt }; + return isDeepStrictEqual(entryWithoutUpdatedAt, snapshot); +} + /** Decides whether to retry after rechecking auto-fallback primary probe state. */ export function resolveRunAfterAutoFallbackPrimaryProbeRecheck(params: { run: FollowupRun["run"]; @@ -73,3 +91,105 @@ export function resolveRunAfterAutoFallbackPrimaryProbeRecheck(params: { autoFallbackPrimaryProbe: refreshedProbe, }; } + +/** Clears a recovered primary probe without overwriting a newer session selection. */ +export async function clearRecoveredAutoFallbackPrimaryProbeSelection(params: { + run: FollowupRun["run"]; + provider: string; + model: string; + sessionKey?: string; + activeSessionStore?: Record; + getActiveSessionEntry: () => SessionEntry | undefined; + storePath?: string; +}): Promise { + if (shouldPreserveUserFacingSessionStateForInputProvenance(params.run.inputProvenance)) { + return; + } + const probe = params.run.autoFallbackPrimaryProbe; + if (!probe || params.provider !== probe.provider || params.model !== probe.model) { + return; + } + if (!params.sessionKey || !params.activeSessionStore) { + return; + } + const cachedSessionEntry = params.activeSessionStore[params.sessionKey]; + const activeSessionEntry = cachedSessionEntry ?? params.getActiveSessionEntry(); + if (!activeSessionEntry || !entryMatchesAutoFallbackPrimaryProbe(activeSessionEntry, probe)) { + return; + } + const activeSessionEntryBeforeUpdate = structuredClone(activeSessionEntry); + if (!params.storePath) { + clearAutoFallbackPrimaryProbeSelection(activeSessionEntry); + params.activeSessionStore[params.sessionKey] = activeSessionEntry; + return; + } + let comparedEntry: SessionEntry | undefined; + const updatedEntry = await updateSessionEntry( + { storePath: params.storePath, sessionKey: params.sessionKey }, + (persistedEntry) => { + comparedEntry = persistedEntry; + if ( + persistedEntry.sessionId !== activeSessionEntryBeforeUpdate.sessionId || + persistedEntry.updatedAt !== activeSessionEntryBeforeUpdate.updatedAt || + !entryMatchesAutoFallbackPrimaryProbe(persistedEntry, probe) + ) { + return null; + } + const shouldClearAuthProfile = + persistedEntry.authProfileOverrideSource === "auto" || + (persistedEntry.authProfileOverrideSource === undefined && + persistedEntry.authProfileOverrideCompactionCount !== undefined); + clearAutoFallbackPrimaryProbeSelection(persistedEntry); + return { + providerOverride: undefined, + modelOverride: undefined, + modelOverrideSource: undefined, + modelOverrideFallbackOriginProvider: undefined, + modelOverrideFallbackOriginModel: undefined, + ...(shouldClearAuthProfile + ? { + authProfileOverride: undefined, + authProfileOverrideSource: undefined, + authProfileOverrideCompactionCount: undefined, + } + : {}), + fallbackNoticeSelectedModel: undefined, + fallbackNoticeActiveModel: undefined, + fallbackNoticeReason: undefined, + updatedAt: persistedEntry.updatedAt, + }; + }, + ); + // The persisted comparison owns selection freshness. Publish its updated + // result, or refresh the cache from the entry that rejected this probe. + const authoritativeEntry = updatedEntry ?? comparedEntry; + const currentCachedEntry = params.activeSessionStore[params.sessionKey]; + // Object replacement is a new cache generation even when values match. + // Preserve it to avoid clearing an identically reselected fallback. + if (currentCachedEntry !== cachedSessionEntry) { + return; + } + const currentEntry = currentCachedEntry ?? (cachedSessionEntry ? undefined : activeSessionEntry); + if (!currentEntry) { + return; + } + if (authoritativeEntry) { + if (sessionEntryMatchesSnapshot(currentEntry, activeSessionEntryBeforeUpdate)) { + params.activeSessionStore[params.sessionKey] = authoritativeEntry; + return; + } + if ( + currentEntry.sessionId !== activeSessionEntryBeforeUpdate.sessionId || + sessionEntryOnlyUpdatedAtChanged(currentEntry, activeSessionEntryBeforeUpdate) + ) { + return; + } + params.activeSessionStore[params.sessionKey] = mergeSessionSnapshotChanges({ + initial: activeSessionEntryBeforeUpdate, + next: authoritativeEntry, + current: currentEntry, + }); + } else if (sessionEntryMatchesSnapshot(currentEntry, activeSessionEntryBeforeUpdate)) { + delete params.activeSessionStore[params.sessionKey]; + } +} diff --git a/src/auto-reply/reply/agent-runner-context-recovery.test.ts b/src/auto-reply/reply/agent-runner-context-recovery.test.ts new file mode 100644 index 000000000000..3e5ca490383d --- /dev/null +++ b/src/auto-reply/reply/agent-runner-context-recovery.test.ts @@ -0,0 +1,687 @@ +import { describe, expect, it } from "vitest"; +import type { ModelDefinitionConfig } from "../../config/types.models.js"; +import { + buildContextOverflowRecoveryText, + computeContextAwareReserveTokensFloor, +} from "./agent-runner-context-recovery.js"; + +function makeTestModel(id: string, contextTokens: number): ModelDefinitionConfig { + return { + id, + name: id, + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: contextTokens, + contextTokens, + maxTokens: 4096, + }; +} + +describe("computeContextAwareReserveTokensFloor", () => { + it("returns 100000 for 1M context windows", () => { + expect(computeContextAwareReserveTokensFloor(1_000_000)).toBe(100_000); + }); + + it("returns 50000 for 200k context windows", () => { + expect(computeContextAwareReserveTokensFloor(200_000)).toBe(50_000); + }); + + it("returns 35000 for 100k context windows", () => { + expect(computeContextAwareReserveTokensFloor(100_000)).toBe(35_000); + }); + + it("returns 20000 for context windows below 100k", () => { + expect(computeContextAwareReserveTokensFloor(99_999)).toBe(20_000); + expect(computeContextAwareReserveTokensFloor(32_768)).toBe(20_000); + expect(computeContextAwareReserveTokensFloor(50_000)).toBe(20_000); + }); + + it("returns 20000 for undefined context window", () => { + expect(computeContextAwareReserveTokensFloor(undefined)).toBe(20_000); + }); + + it("returns 20000 for non-positive context window", () => { + expect(computeContextAwareReserveTokensFloor(0)).toBe(20_000); + expect(computeContextAwareReserveTokensFloor(-1)).toBe(20_000); + }); + + it("returns correct tiers at exact boundaries", () => { + expect(computeContextAwareReserveTokensFloor(100_000)).toBe(35_000); + expect(computeContextAwareReserveTokensFloor(200_000)).toBe(50_000); + expect(computeContextAwareReserveTokensFloor(1_000_000)).toBe(100_000); + expect(computeContextAwareReserveTokensFloor(99_999)).toBe(20_000); + expect(computeContextAwareReserveTokensFloor(199_999)).toBe(35_000); + expect(computeContextAwareReserveTokensFloor(999_999)).toBe(50_000); + }); +}); + +describe("buildContextOverflowRecoveryText", () => { + it("keeps the generic compaction-buffer hint without heartbeat model evidence", () => { + const text = buildContextOverflowRecoveryText({ + cfg: {}, + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("20000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("suggests 100000 reserveTokensFloor for 1M context models", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("qwen3.6-plus", 1_000_000)], + }, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("100000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("suggests 50000 reserveTokensFloor for 200k context models", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("gpt-5.5-200k", 200_000)], + }, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "gpt-5.5-200k", + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("50000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("suggests 35000 reserveTokensFloor for 100k context models", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("gpt-5.5", 100_000)], + }, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "gpt-5.5", + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("35000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("suggests 20000 reserveTokensFloor for small context windows", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + ollama: { + baseUrl: "http://ollama.test", + models: [makeTestModel("qwen3.5-9b-32k:latest", 32_768)], + }, + }, + }, + }, + primaryProvider: "ollama", + primaryModel: "qwen3.5-9b-32k:latest", + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("20000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("uses session contextTokens as fallback when model metadata is unavailable", () => { + const text = buildContextOverflowRecoveryText({ + cfg: {}, + primaryProvider: "openrouter", + primaryModel: "unknown-model", + activeSessionEntry: { + sessionId: "session", + updatedAt: 1, + modelProvider: "openrouter", + model: "unknown-model", + contextTokens: 200_000, + }, + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("50000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("prefers model metadata over session contextTokens", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("qwen3.6-plus", 1_000_000)], + }, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + activeSessionEntry: { + sessionId: "session", + updatedAt: 1, + modelProvider: "openrouter", + model: "qwen3.6-plus", + contextTokens: 32_768, + }, + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("100000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("keeps the preserved-session copy with the existing overflow hint", () => { + const text = buildContextOverflowRecoveryText({ + preserveSessionMapping: true, + cfg: {}, + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + }); + + expect(text).toContain("kept this conversation mapped to the current session"); + expect(text).toContain("reserveTokensFloor"); + expect(text).not.toContain("reset our conversation"); + }); + + it("falls back to session entry model when runtimeProvider is not provided", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + ollama: { + baseUrl: "http://ollama.test", + models: [makeTestModel("qwen3.5-9b-32k:latest", 32_768)], + }, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "unknown-model", + activeSessionEntry: { + sessionId: "session", + updatedAt: 1, + modelProvider: "ollama", + model: "qwen3.5-9b-32k:latest", + contextTokens: 200_000, + }, + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("20000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("prefers session entry model context over session contextTokens numeric value", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + ollama: { + baseUrl: "http://ollama.test", + models: [makeTestModel("qwen3.5-9b-32k:latest", 32_768)], + }, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "unknown-model", + activeSessionEntry: { + sessionId: "session", + updatedAt: 1, + modelProvider: "ollama", + model: "qwen3.5-9b-32k:latest", + contextTokens: 1_000_000, + }, + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("20000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("uses session contextTokens before primary metadata for uncataloged runtime models", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("qwen3.6-plus", 1_000_000)], + }, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + activeSessionEntry: { + sessionId: "session", + updatedAt: 1, + modelProvider: "custom", + model: "uncataloged-32k", + contextTokens: 32_768, + }, + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("20000"); + expect(text).not.toContain("100000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("does not use primary metadata for explicit uncataloged runtime models", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("qwen3.6-plus", 1_000_000)], + }, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + runtimeProvider: "custom", + runtimeModel: "uncataloged-32k", + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("20000"); + expect(text).not.toContain("100000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("does not use stale session contextTokens for explicit uncataloged runtime models", () => { + const text = buildContextOverflowRecoveryText({ + cfg: {}, + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + runtimeProvider: "custom", + runtimeModel: "uncataloged-32k", + activeSessionEntry: { + sessionId: "session", + updatedAt: 1, + modelProvider: "openrouter", + model: "qwen3.6-plus", + contextTokens: 1_000_000, + }, + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("20000"); + expect(text).not.toContain("100000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("caps reserveTokensFloor hint by agent.defaults.contextTokens", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("qwen3.6-plus", 1_000_000)], + }, + }, + }, + agents: { + defaults: { + contextTokens: 100_000, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("35000"); + expect(text).not.toContain("100000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("caps reserveTokensFloor hint by per-agent contextTokens over defaults", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("qwen3.6-plus", 1_000_000)], + }, + }, + }, + agents: { + defaults: { + contextTokens: 200_000, + }, + list: [ + { + id: "capped-agent", + contextTokens: 32_768, + }, + ], + }, + }, + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + agentId: "capped-agent", + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("20000"); + expect(text).not.toContain("50000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("caps the session contextTokens fallback by agent contextTokens", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + agents: { + defaults: { + contextTokens: 200_000, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "unknown-model", + activeSessionEntry: { + sessionId: "session", + updatedAt: 1, + modelProvider: "openrouter", + model: "unknown-model", + contextTokens: 32_768, + }, + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("20000"); + expect(text).not.toContain("50000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("uses runtime model over primary model when both are available", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("qwen3.6-plus", 1_000_000)], + }, + ollama: { + baseUrl: "http://ollama.test", + models: [makeTestModel("qwen3.5-9b-32k:latest", 32_768)], + }, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + runtimeProvider: "ollama", + runtimeModel: "qwen3.5-9b-32k:latest", + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("20000"); + expect(text).not.toContain("100000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("uses runtime model with 200k context when primary is 1M", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("qwen3.6-plus", 1_000_000)], + }, + openai: { + baseUrl: "https://openai.test", + models: [makeTestModel("gpt-5.5-200k", 200_000)], + }, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + runtimeProvider: "openai", + runtimeModel: "gpt-5.5-200k", + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("50000"); + expect(text).not.toContain("100000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("does not use stale heartbeat bleed hints for different explicit runtime refs", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + agents: { + defaults: { + heartbeat: { model: "ollama/qwen3.5-9b-32k:latest" }, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + runtimeProvider: "custom", + runtimeModel: "uncataloged-32k", + activeSessionEntry: { + sessionId: "session", + updatedAt: 1, + modelProvider: "ollama", + model: "qwen3.5-9b-32k:latest", + contextTokens: 32_768, + }, + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).toContain("20000"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("points to heartbeat model bleed when the last runtime model matches configured heartbeat.model", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("qwen3.6-plus", 1_000_000)], + }, + ollama: { + baseUrl: "http://ollama.test", + models: [makeTestModel("qwen3.5-9b-32k:latest", 32_768)], + }, + }, + }, + agents: { + defaults: { + heartbeat: { model: "ollama/qwen3.5-9b-32k:latest" }, + }, + }, + }, + agentId: "agent", + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + activeSessionEntry: { + sessionId: "session", + updatedAt: 1, + modelProvider: "ollama", + model: "qwen3.5-9b-32k:latest", + contextTokens: 32_768, + }, + }); + + expect(text).toContain("ollama/qwen3.5-9b-32k:latest (32k context)"); + expect(text).toContain("openrouter/qwen3.6-plus"); + expect(text).toContain("heartbeat model bleed"); + expect(text).toContain("heartbeat.isolatedSession"); + expect(text).not.toContain("reserveTokensFloor"); + }); + + it("uses the stored session context window as the uncataloged runtime model fallback", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("qwen3.6-plus", 1_000_000)], + }, + }, + }, + agents: { + defaults: { + contextTokens: 100_000, + heartbeat: { model: "ollama/custom-32k" }, + }, + }, + }, + agentId: "agent", + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + activeSessionEntry: { + sessionId: "session", + updatedAt: 1, + modelProvider: "ollama", + model: "custom-32k", + contextTokens: 32_768, + }, + }); + + expect(text).toContain("ollama/custom-32k (32k context)"); + expect(text).not.toContain("ollama/custom-32k (98k context)"); + expect(text).toContain("heartbeat model bleed"); + }); + + it("does not blame heartbeat when the stored session fallback matches the capped primary window", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("qwen3.6-plus", 1_000_000)], + }, + }, + }, + agents: { + defaults: { + contextTokens: 100_000, + heartbeat: { model: "ollama/custom-large" }, + }, + }, + }, + agentId: "agent", + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + activeSessionEntry: { + sessionId: "session", + updatedAt: 1, + modelProvider: "ollama", + model: "custom-large", + contextTokens: 200_000, + }, + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("does not blame heartbeat when the same agent cap constrains both cataloged models", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("qwen3.6-plus", 1_000_000)], + }, + ollama: { + baseUrl: "http://ollama.test", + models: [makeTestModel("custom-large", 1_000_000)], + }, + }, + }, + agents: { + defaults: { + contextTokens: 100_000, + heartbeat: { model: "ollama/custom-large" }, + }, + }, + }, + agentId: "agent", + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + activeSessionEntry: { + sessionId: "session", + updatedAt: 1, + modelProvider: "ollama", + model: "custom-large", + contextTokens: 1_000_000, + }, + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).not.toContain("heartbeat model bleed"); + }); + + it("does not blame heartbeat when the smaller runtime model is not the configured heartbeat model", () => { + const text = buildContextOverflowRecoveryText({ + cfg: { + agents: { + defaults: { + heartbeat: { model: "ollama/qwen3.5-9b-32k:latest" }, + }, + }, + }, + primaryProvider: "openrouter", + primaryModel: "qwen3.6-plus", + activeSessionEntry: { + sessionId: "session", + updatedAt: 1, + modelProvider: "anthropic", + model: "claude-haiku-4-5", + contextTokens: 32_768, + }, + }); + + expect(text).toContain("reserveTokensFloor"); + expect(text).not.toContain("heartbeat model bleed"); + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-auth-failures.test.ts b/src/auto-reply/reply/agent-runner-execution-auth-failures.test.ts new file mode 100644 index 000000000000..d61c7ce6546f --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-auth-failures.test.ts @@ -0,0 +1,439 @@ +import { describe, expect, it } from "vitest"; +import { OAuthRefreshFailureError } from "../../agents/auth-profiles/oauth-refresh-failure.js"; +import { FailoverError } from "../../agents/failover-error.js"; +import { MissingProviderAuthError } from "../../agents/model-auth.js"; +import type { TemplateContext } from "../templating.js"; +import { + setupAgentRunnerExecutionTestState, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, + createMinimalRunAgentTurnParams, +} from "./agent-runner-execution.test-support.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: authentication failures", () => { + it("surfaces gateway reauth guidance for known OAuth refresh failures", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error( + "OAuth token refresh failed for openai: refresh_token_reused. Please try again or re-authenticate.", + ), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + "⚠️ Model login expired on the gateway for openai. Send `/login codex` from a private chat or Web UI session to pair a new Codex login, or re-auth with `openclaw models auth login --provider openai` in a terminal, then try again.", + ); + } + }); + + it("surfaces gateway reauth guidance from typed OAuth refresh failures", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new OAuthRefreshFailureError({ + provider: "openai", + profileId: "openai:user@example.com", + message: "invalid_grant", + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + "⚠️ Model login expired on the gateway for openai. Send `/login codex` from a private chat or Web UI session to pair a new Codex login, or re-auth with `openclaw models auth login --provider openai --profile-id 'openai:user@example.com'` in a terminal, then try again.", + ); + } + }); + + it("preserves OAuth profile guidance through failover wrappers", async () => { + const refreshError = new OAuthRefreshFailureError({ + provider: "openai", + profileId: "openai:user@example.com", + message: "invalid_grant", + }); + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new FailoverError("OpenAI OAuth failed", { + reason: "auth", + provider: "openai", + model: "gpt-5.5", + profileId: "openai:user@example.com", + authProfileFailure: { allInCooldown: false }, + status: 401, + cause: refreshError, + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toContain("--profile-id 'openai:user@example.com'"); + } + }); + + it("preserves OAuth profile guidance through fallback summaries", async () => { + const refreshError = new OAuthRefreshFailureError({ + provider: "openai", + profileId: "openai:user@example.com", + message: "invalid_grant", + }); + const failoverError = new FailoverError("OpenAI OAuth failed", { + reason: "auth", + provider: "openai", + model: "gpt-5.5", + profileId: "openai:user@example.com", + authProfileFailure: { allInCooldown: false }, + status: 401, + cause: refreshError, + }); + const summaryError = new Error("All models failed", { cause: failoverError }); + summaryError.name = "FallbackSummaryError"; + Object.assign(summaryError, { + attempts: [ + { + provider: "openai", + model: "gpt-5.5", + error: "OpenAI OAuth failed", + reason: "auth", + }, + ], + soonestCooldownExpiry: null, + }); + state.runEmbeddedAgentMock.mockRejectedValueOnce(summaryError); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toContain("--profile-id 'openai:user@example.com'"); + } + }); + + it("omits OAuth profile ids from group reauth guidance", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new OAuthRefreshFailureError({ + provider: "openai", + profileId: "openai:user@example.com", + message: "invalid_grant", + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + ChatType: "group", + } as unknown as TemplateContext, + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toContain( + "openclaw models auth login --provider openai` in a terminal", + ); + expect(result.payload.text).not.toContain("user@example.com"); + } + }); + + it("keeps non-OpenAI OAuth refresh failures on provider-specific terminal guidance", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new OAuthRefreshFailureError({ + provider: "anthropic", + message: "invalid_grant", + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + "⚠️ Model login expired on the gateway for anthropic. Re-auth with `openclaw models auth login --provider anthropic` in a terminal, then try again.", + ); + expect(result.payload.text).not.toContain("/login codex"); + } + }); + + it("surfaces claude-cli re-auth hint over generic provider auth copy for 401 OAuth expiry", async () => { + // When the claude subprocess emits a 401 "Failed to authenticate" because + // its OAuth token has expired, the error is wrapped as a FailoverError with + // reason:"auth" and status:401. Without the ordering fix, this would be + // caught by classifyProviderRequestError before reaching classifyOAuthRefreshFailure, + // producing the generic "re-authenticate this provider" copy instead of the + // targeted claude-cli re-auth command. + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new FailoverError( + "Provider claude-cli failed: Failed to authenticate. API Error: 401 Invalid authentication credentials", + { + reason: "auth", + provider: "claude-cli", + model: "claude-sonnet-4-20250514", + status: 401, + }, + ), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + "⚠️ Model login expired on the gateway for claude-cli. Re-auth with `claude auth login && openclaw models auth login --provider anthropic --method cli` in a terminal, then try again.", + ); + } + }); + + it("surfaces claude-cli re-auth hint from structured provider metadata when the message omits claude-cli", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new FailoverError( + "Failed to authenticate. API Error: 401 Invalid authentication credentials", + { + reason: "auth", + provider: "claude-cli", + model: "claude-sonnet-4-20250514", + status: 401, + }, + ), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + "⚠️ Model login expired on the gateway for claude-cli. Re-auth with `claude auth login && openclaw models auth login --provider anthropic --method cli` in a terminal, then try again.", + ); + } + }); + + it("surfaces direct provider auth guidance for missing API keys", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error( + 'No API key found for provider "openai". You are authenticated with OpenAI Codex OAuth; OpenAI agent model runs use openai/gpt-* through the Codex runtime. Set OPENAI_API_KEY only for direct OpenAI API-key surfaces. | No API key found for provider "openai". You are authenticated with OpenAI Codex OAuth; OpenAI agent model runs use openai/gpt-* through the Codex runtime. Set OPENAI_API_KEY only for direct OpenAI API-key surfaces.', + ), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + "⚠️ Missing API key for OpenAI on the gateway. Use `openai/gpt-5.6-sol` with the OpenAI OAuth profile, or set `OPENAI_API_KEY` for direct OpenAI API-key runs.", + ); + } + }); + + it("surfaces typed missing API-key auth guidance without parsing the message", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new MissingProviderAuthError("openai", { + mode: "api-key", + source: "env: OPENAI_API_KEY", + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + '⚠️ Missing API key for provider "openai". Run `openclaw doctor --fix` to repair stale OpenAI model/session routes, restart the gateway if doctor asks, then try again. If doctor has nothing to repair or the error persists, re-auth with `openclaw models auth login --provider openai` or run `openclaw configure`.', + ); + } + }); + + it("formats auth-profile failover copy from typed FailoverError metadata", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new FailoverError("Auth profile failover exhausted for provider openai", { + reason: "auth", + provider: "openai", + status: 401, + authProfileFailure: { allInCooldown: true }, + cause: new Error("invalid_grant"), + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toContain("Couldn't sign in to openai."); + expect(result.payload.text).toContain("openclaw configure"); + expect(result.payload.text).toContain("(invalid_grant)"); + expect(result.payload.text).not.toContain("Auth profile failover exhausted"); + } + }); + + it("does not suggest re-authentication for typed format failures", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new FailoverError("Format failover exhausted for provider openai", { + reason: "format", + provider: "openai", + authProfileFailure: { allInCooldown: true }, + cause: new Error("messages must alternate roles"), + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toContain("Couldn't reach openai"); + expect(result.payload.text).toContain("messages must alternate roles"); + expect(result.payload.text).not.toContain("models auth login"); + expect(result.payload.text).not.toContain("openclaw configure"); + } + }); + + it("points stale openai missing-key failures at doctor repair with re-auth fallback", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error('No API key found for provider "openai".'), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + '⚠️ Missing API key for provider "openai". Run `openclaw doctor --fix` to repair stale OpenAI model/session routes, restart the gateway if doctor asks, then try again. If doctor has nothing to repair or the error persists, re-auth with `openclaw models auth login --provider openai` or run `openclaw configure`.', + ); + } + }); + + it("falls back to a generic provider message for unsafe missing-key provider ids", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error('No API key found for provider "openai`\nrm -rf /".'), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + "⚠️ Missing API key for the selected provider on the gateway. Configure provider auth, then try again.", + ); + } + }); + + it("falls back to a generic reauth command when the provider in the OAuth error is unsafe", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error( + "OAuth token refresh failed for openai`\nrm -rf /: invalid_grant. Please try again or re-authenticate.", + ), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + "⚠️ Model login expired on the gateway. Re-auth with `openclaw models auth login` in a terminal, then try again.", + ); + } + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-cli-progress.test.ts b/src/auto-reply/reply/agent-runner-execution-cli-progress.test.ts new file mode 100644 index 000000000000..2e8cfacfbc90 --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-cli-progress.test.ts @@ -0,0 +1,930 @@ +import { describe, expect, it, vi } from "vitest"; +import type { TemplateContext } from "../templating.js"; +import type { GetReplyOptions } from "../types.js"; +import { + setupAgentRunnerExecutionTestState, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, + createMinimalRunAgentTurnParams, +} from "./agent-runner-execution.test-support.js"; +import type { + FallbackRunnerParams, + EmbeddedAgentParams, +} from "./agent-runner-execution.test-support.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: CLI progress bridging", () => { + it("bridges CLI assistant agent events into onPartialReply for live preview (#76869)", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-opus-4-6"), + provider: "claude-cli", + model: "claude-opus-4-6", + attempts: [], + })); + state.runCliAgentMock.mockImplementationOnce( + async (params: { runId: string; emitCommentaryText?: boolean }) => { + expect(params.emitCommentaryText).toBe(false); + const realAgentEvents = await vi.importActual( + "../../infra/agent-events.js", + ); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "assistant", + data: { text: "Hello", delta: "Hello" }, + }); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "assistant", + data: { text: "Hello world", delta: " world" }, + }); + return { payloads: [{ text: "Hello world" }], meta: {} }; + }, + ); + + const onPartialReply = vi.fn>( + async (_payload) => undefined, + ); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-opus-4-6"; + + await runAgentTurnWithFallback({ + commandBody: "hi", + followupRun, + sessionCtx: { + Provider: "telegram", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onPartialReply }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + const partialTexts = onPartialReply.mock.calls.map((call) => call[0].text); + expect(partialTexts).toEqual(["Hello", "Hello world"]); + }); + + it("serializes and drains bridged CLI assistant previews before completing (#76869)", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-opus-4-6"), + provider: "claude-cli", + model: "claude-opus-4-6", + attempts: [], + })); + state.runCliAgentMock.mockImplementationOnce( + async (params: { runId: string; emitCommentaryText?: boolean }) => { + expect(params.emitCommentaryText).toBe(false); + const realAgentEvents = await vi.importActual( + "../../infra/agent-events.js", + ); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "assistant", + data: { text: "Hello", delta: "Hello" }, + }); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "assistant", + data: { text: "Hello world", delta: " world" }, + }); + return { payloads: [{ text: "Hello world" }], meta: {} }; + }, + ); + + let firstPreviewStarted: (() => void) | undefined; + let releaseFirstPreview: (() => void) | undefined; + const firstPreviewPromise = new Promise((resolve) => { + firstPreviewStarted = resolve; + }); + const previewOrder: string[] = []; + const onPartialReply = vi.fn>( + async (payload) => { + previewOrder.push(payload.text ?? ""); + if (payload.text === "Hello") { + firstPreviewStarted?.(); + await new Promise((resolve) => { + releaseFirstPreview = resolve; + }); + previewOrder.push("Hello released"); + } + }, + ); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-opus-4-6"; + + const runPromise = runAgentTurnWithFallback({ + commandBody: "hi", + followupRun, + sessionCtx: { + Provider: "telegram", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onPartialReply }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + await firstPreviewPromise; + await new Promise((resolve) => { + setImmediate(resolve); + }); + expect(previewOrder).toEqual(["Hello"]); + + releaseFirstPreview?.(); + await runPromise; + + expect(previewOrder).toEqual(["Hello", "Hello released", "Hello world"]); + }); + + it("bridges CLI tool agent events into onToolStart for live preview", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-opus-4-6"), + provider: "claude-cli", + model: "claude-opus-4-6", + attempts: [], + })); + state.runCliAgentMock.mockImplementationOnce( + async (params: { runId: string; emitCommentaryText?: boolean }) => { + expect(params.emitCommentaryText).toBe(false); + const realAgentEvents = await vi.importActual( + "../../infra/agent-events.js", + ); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "tool", + data: { + phase: "start", + name: "Bash", + toolCallId: "toolu_01ABCD", + args: { command: "ls -la" }, + }, + }); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "tool", + data: { + phase: "result", + name: "Bash", + toolCallId: "toolu_01ABCD", + isError: false, + }, + }); + return { payloads: [{ text: "done" }], meta: {} }; + }, + ); + + const onToolStart = vi.fn>(async () => undefined); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-opus-4-6"; + + await runAgentTurnWithFallback({ + commandBody: "hi", + followupRun, + sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, + opts: { onToolStart }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + + expect(onToolStart).toHaveBeenCalledTimes(1); + const call = onToolStart.mock.calls[0]?.[0]; + expect(call?.name).toBe("Bash"); + expect(call?.phase).toBe("start"); + expect(call?.args).toEqual({ command: "ls -la" }); + }); + + it("starts CLI assistant progress before a later tool while typing is slow", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-opus-4-6"), + provider: "claude-cli", + model: "claude-opus-4-6", + attempts: [], + })); + state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { + const agentEvents = await import("../../infra/agent-events.js"); + agentEvents.emitAgentEvent({ + runId: params.runId, + stream: "assistant", + data: { text: "answer before tool", delta: "answer before tool" }, + }); + agentEvents.emitAgentEvent({ + runId: params.runId, + stream: "assistant", + data: { text: "answer before tool 2", delta: " 2" }, + }); + agentEvents.emitAgentEvent({ + runId: params.runId, + stream: "tool", + data: { + phase: "start", + name: "Bash", + toolCallId: "toolu_order", + args: { command: "echo hi" }, + }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + let releaseTyping: (() => void) | undefined; + const typingPending = new Promise((resolve) => { + releaseTyping = resolve; + }); + const typingSignals = createMockTypingSignaler(); + vi.mocked(typingSignals.signalTextDelta).mockReturnValue(typingPending); + const callbackOrder: string[] = []; + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-opus-4-6"; + const runPromise = runAgentTurnWithFallback({ + commandBody: "hi", + followupRun, + sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, + opts: { + preserveProgressCallbackStartOrder: true, + onPartialReply: (payload) => { + callbackOrder.push(`partial:${payload.text}`); + }, + onToolStart: () => { + callbackOrder.push("tool"); + }, + }, + typingSignals, + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + try { + await vi.waitFor(() => { + expect(callbackOrder).toContain("tool"); + }); + expect(callbackOrder).toEqual([ + "partial:answer before tool", + "partial:answer before tool 2", + "tool", + ]); + } finally { + releaseTyping?.(); + await runPromise; + } + }); + + it("starts CLI tool progress before later assistant text while typing is slow", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-opus-4-6"), + provider: "claude-cli", + model: "claude-opus-4-6", + attempts: [], + })); + state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { + const agentEvents = await import("../../infra/agent-events.js"); + agentEvents.emitAgentEvent({ + runId: params.runId, + stream: "tool", + data: { + phase: "start", + name: "Bash", + toolCallId: "toolu_inverse_order", + args: { command: "echo hi" }, + }, + }); + agentEvents.emitAgentEvent({ + runId: params.runId, + stream: "tool", + data: { + phase: "update", + name: "Bash", + toolCallId: "toolu_inverse_order", + args: { command: "echo hi" }, + }, + }); + agentEvents.emitAgentEvent({ + runId: params.runId, + stream: "assistant", + data: { text: "answer after tool", delta: "answer after tool" }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + let releaseTyping: (() => void) | undefined; + const typingPending = new Promise((resolve) => { + releaseTyping = resolve; + }); + const typingSignals = createMockTypingSignaler(); + vi.mocked(typingSignals.signalToolStart).mockReturnValue(typingPending); + const callbackOrder: string[] = []; + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-opus-4-6"; + const runPromise = runAgentTurnWithFallback({ + commandBody: "hi", + followupRun, + sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, + opts: { + preserveProgressCallbackStartOrder: true, + onPartialReply: (payload) => { + callbackOrder.push(`partial:${payload.text}`); + }, + onToolStart: (payload) => { + callbackOrder.push(`tool:${payload.phase}`); + }, + }, + typingSignals, + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + try { + await vi.waitFor(() => { + expect(callbackOrder).toContain("partial:answer after tool"); + }); + expect(callbackOrder).toEqual(["tool:start", "tool:update", "partial:answer after tool"]); + } finally { + releaseTyping?.(); + await runPromise; + } + }); + + it("bridges CLI preambles for progress headlines when commentary is disabled", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-opus-4-6"), + provider: "claude-cli", + model: "claude-opus-4-6", + attempts: [], + })); + state.runCliAgentMock.mockImplementationOnce( + async (params: { runId: string; emitCommentaryText?: boolean }) => { + expect(params.emitCommentaryText).toBe(true); + const agentEvents = await import("../../infra/agent-events.js"); + // Inter-tool commentary surfaces as a stream:"item", kind:"preamble" agent event. + agentEvents.emitAgentEvent({ + runId: params.runId, + stream: "item", + data: { + kind: "preamble", + itemId: "commentary-1", + progressText: "Let me check the files.", + }, + }); + return { payloads: [{ text: "done" }], meta: {} }; + }, + ); + + const onItemEvent = vi.fn>(async () => undefined); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-opus-4-6"; + + await runAgentTurnWithFallback({ + commandBody: "hi", + followupRun, + sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, + opts: { + onItemEvent, + commentaryProgressEnabled: false, + progressPreambleEnabled: true, + }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + + expect(onItemEvent).toHaveBeenCalledTimes(1); + const call = onItemEvent.mock.calls[0]?.[0]; + expect(call?.kind).toBe("preamble"); + expect(call?.progressText).toBe("Let me check the files."); + expect(call?.itemId).toBe("commentary-1"); + }); + + it("does not emit CLI preambles when both progress lanes are disabled", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-opus-4-6"), + provider: "claude-cli", + model: "claude-opus-4-6", + attempts: [], + })); + state.runCliAgentMock.mockImplementationOnce( + async (params: { runId: string; emitCommentaryText?: boolean }) => { + // With no commentary lane or headline consumer, pre-tool text stays in + // the assistant stream instead of being split into progress events. + expect(params.emitCommentaryText).toBe(false); + return { payloads: [{ text: "done" }], meta: {} }; + }, + ); + + const onItemEvent = vi.fn>(); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-opus-4-6"; + + await runAgentTurnWithFallback({ + commandBody: "hi", + followupRun, + sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, + opts: { + onItemEvent, + commentaryProgressEnabled: false, + progressPreambleEnabled: false, + }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(state.runCliAgentMock).toHaveBeenCalledTimes(1); + expect(onItemEvent).not.toHaveBeenCalled(); + }); + + it("does not bridge CLI tool deltas when silentExpected is set", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-opus-4-6"), + provider: "claude-cli", + model: "claude-opus-4-6", + attempts: [], + })); + state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { + const realAgentEvents = await vi.importActual( + "../../infra/agent-events.js", + ); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "tool", + data: { + phase: "start", + name: "Bash", + toolCallId: "toolu_silent", + args: { command: "echo silent" }, + }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const onToolStart = vi.fn>(async () => undefined); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-opus-4-6"; + followupRun.run.silentExpected = true; + + await runAgentTurnWithFallback({ + commandBody: "hi", + followupRun, + sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, + opts: { onToolStart }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + + expect(onToolStart).not.toHaveBeenCalled(); + }); + + it("does not bridge CLI assistant deltas when silentExpected is set (#76869)", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-opus-4-6"), + provider: "claude-cli", + model: "claude-opus-4-6", + attempts: [], + })); + state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { + const realAgentEvents = await vi.importActual( + "../../infra/agent-events.js", + ); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "assistant", + data: { text: "secret heartbeat output", delta: "secret heartbeat output" }, + }); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "assistant", + data: { text: "NO_REPLY do not preview", delta: " do not preview" }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const onPartialReply = vi.fn>( + async (_payload) => undefined, + ); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-opus-4-6"; + followupRun.run.silentExpected = true; + + await runAgentTurnWithFallback({ + commandBody: "hi", + followupRun, + sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, + opts: { onPartialReply }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + + expect(onPartialReply).not.toHaveBeenCalled(); + }); + + it("bridges CLI thinking agent events into onReasoningStream with the reasoning opt-in gate", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-opus-4-7"), + provider: "claude-cli", + model: "claude-opus-4-7", + attempts: [], + })); + state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { + const realAgentEvents = await vi.importActual( + "../../infra/agent-events.js", + ); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "thinking", + data: { text: "Thinking", delta: "Thinking", isReasoningSnapshot: true }, + }); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "thinking", + data: { text: "Thinking", delta: "", isReasoningSnapshot: true }, + }); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "thinking", + data: { text: "Thinking about it", delta: " about it", isReasoningSnapshot: true }, + }); + return { payloads: [{ text: "Thinking about it" }], meta: {} }; + }); + + const onReasoningStream = vi.fn>( + async (_payload) => undefined, + ); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-opus-4-7"; + + await runAgentTurnWithFallback({ + commandBody: "hi", + followupRun, + sessionCtx: { + Provider: "telegram", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onReasoningStream }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(onReasoningStream.mock.calls.map((call) => call[0])).toEqual([ + { + text: "Thinking", + isReasoningSnapshot: true, + requiresReasoningProgressOptIn: true, + }, + { + text: "Thinking about it", + isReasoningSnapshot: true, + requiresReasoningProgressOptIn: true, + }, + ]); + }); + + it("does not bridge CLI thinking events to onReasoningStream when silentExpected is set", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-opus-4-7"), + provider: "claude-cli", + model: "claude-opus-4-7", + attempts: [], + })); + state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { + const realAgentEvents = await vi.importActual( + "../../infra/agent-events.js", + ); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "thinking", + data: { text: "heartbeat scratch text", delta: "heartbeat scratch text" }, + }); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "thinking", + data: { text: "NO_REPLY do not preview reasoning", delta: " do not preview reasoning" }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const onReasoningStream = vi.fn>( + async (_payload) => undefined, + ); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-opus-4-7"; + followupRun.run.silentExpected = true; + + await runAgentTurnWithFallback({ + commandBody: "hi", + followupRun, + sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, + opts: { onReasoningStream }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + + expect(onReasoningStream).not.toHaveBeenCalled(); + }); + + it("does not bridge non-Claude CLI assistant events to onReasoningStream", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("codex-cli", "gpt-5.5"), + provider: "codex-cli", + model: "gpt-5.5", + attempts: [], + })); + state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { + const realAgentEvents = await vi.importActual( + "../../infra/agent-events.js", + ); + realAgentEvents.emitAgentEvent({ + runId: params.runId, + stream: "assistant", + data: { text: "final answer", delta: "final answer" }, + }); + return { payloads: [{ text: "final answer" }], meta: {} }; + }); + + const onReasoningStream = vi.fn>( + async (_payload) => undefined, + ); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "codex-cli"; + followupRun.run.model = "gpt-5.5"; + + await runAgentTurnWithFallback({ + commandBody: "hi", + followupRun, + sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, + opts: { onReasoningStream }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + + expect(onReasoningStream).not.toHaveBeenCalled(); + }); + + it("does not double-fire onReasoningStream from the bridge when the API/native runtime path is active", async () => { + state.isCliProviderMock.mockReturnValue(false); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("anthropic", "claude-sonnet-4-7"), + provider: "anthropic", + model: "claude-sonnet-4-7", + attempts: [], + })); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + const realAgentEvents = await vi.importActual( + "../../infra/agent-events.js", + ); + realAgentEvents.emitAgentEvent({ + runId: "api-run", + stream: "assistant", + data: { text: "assistant text from API run", delta: "assistant text from API run" }, + }); + await params.onAgentEvent?.({ + stream: "assistant", + data: { text: "assistant text from API run", delta: "assistant text from API run" }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const onReasoningStream = vi.fn>( + async (_payload) => undefined, + ); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-sonnet-4-7"; + + await runAgentTurnWithFallback({ + commandBody: "hi", + followupRun, + sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, + opts: { onReasoningStream, runId: "api-run" }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + + expect(onReasoningStream).not.toHaveBeenCalled(); + }); + + it("preserves embedded reasoning stream opt-in markers", async () => { + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onReasoningStream?.({ text: "stream thought" }); + await params.onReasoningStream?.({ + text: "ambient thought", + requiresReasoningProgressOptIn: true, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const onReasoningStream = vi.fn>( + async (_payload) => undefined, + ); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + + await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + opts: { onReasoningStream }, + }), + ); + + expect( + onReasoningStream.mock.calls.map(([payload]) => ({ + text: payload.text, + requiresReasoningProgressOptIn: payload.requiresReasoningProgressOptIn, + })), + ).toEqual([ + { text: "stream thought", requiresReasoningProgressOptIn: undefined }, + { text: "ambient thought", requiresReasoningProgressOptIn: true }, + ]); + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-cli-sessions.test.ts b/src/auto-reply/reply/agent-runner-execution-cli-sessions.test.ts new file mode 100644 index 000000000000..b70ba7268fbf --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-cli-sessions.test.ts @@ -0,0 +1,465 @@ +import { describe, expect, it } from "vitest"; +import type { SessionEntry } from "../../config/sessions.js"; +import type { TemplateContext } from "../templating.js"; +import { SILENT_REPLY_TOKEN } from "../tokens.js"; +import { + setupAgentRunnerExecutionTestState, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, + createTestUserTurnRecorder, + requireRecord, + requireMockCall, + expectMockCallArgFields, + createMinimalRunAgentTurnParams, +} from "./agent-runner-execution.test-support.js"; +import type { FallbackRunnerParams } from "./agent-runner-execution.test-support.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: CLI session routing", () => { + it("forwards the static extra system prompt to CLI backends", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("codex-cli", "gpt-5.4"), + provider: "codex-cli", + model: "gpt-5.4", + attempts: [], + })); + state.runCliAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "final" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "codex-cli"; + followupRun.run.model = "gpt-5.4"; + followupRun.run.extraSystemPrompt = "dynamic inbound metadata\n\nstable group prompt"; + followupRun.run.extraSystemPromptStatic = "stable group prompt"; + followupRun.run.senderId = "sender-static"; + followupRun.run.senderName = "Sender Static"; + followupRun.run.senderUsername = "sender-static-user"; + followupRun.run.senderE164 = "+15550002222"; + followupRun.run.execOverrides = { host: "node", node: "mac-a" }; + followupRun.run.bashElevated = { + enabled: true, + allowed: true, + defaultLevel: "full", + }; + followupRun.run.groupId = "group-static"; + followupRun.run.groupChannel = "ops"; + followupRun.run.groupSpace = "workspace-static"; + followupRun.run.spawnedBy = "agent:main:telegram:group:parent"; + followupRun.run.runtimePolicySessionKey = "agent:main:telegram:default:direct:sender-static"; + followupRun.originatingChannel = "telegram"; + + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { + modelProvider: "codex-cli", + extraSystemPrompt: "dynamic inbound metadata\n\nstable group prompt", + extraSystemPromptStatic: "stable group prompt", + trigger: "user", + messageChannel: "telegram", + messageProvider: "telegram", + senderId: "sender-static", + senderName: "Sender Static", + senderUsername: "sender-static-user", + senderE164: "+15550002222", + execOverrides: { host: "node", node: "mac-a" }, + bashElevated: { enabled: true, allowed: true, defaultLevel: "full" }, + groupId: "group-static", + groupChannel: "ops", + groupSpace: "workspace-static", + spawnedBy: "agent:main:telegram:group:parent", + runtimePolicySessionKey: "agent:main:telegram:default:direct:sender-static", + }); + }); + + it("passes silent empty-reply policy to CLI backends for message-tool-only turns", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-sonnet-4-6"), + provider: "claude-cli", + model: "claude-sonnet-4-6", + attempts: [], + })); + state.runCliAgentMock.mockResolvedValueOnce({ + payloads: [{ text: SILENT_REPLY_TOKEN }], + meta: { executionTrace: { fallbackUsed: false } }, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-sonnet-4-6"; + followupRun.run.sourceReplyDeliveryMode = "message_tool_only"; + followupRun.run.allowEmptyAssistantReplyAsSilent = true; + followupRun.originatingChannel = "telegram"; + + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + followupRun, + sessionCtx: { + Provider: "telegram", + MessageSid: "msg", + ChatType: "group", + } as unknown as TemplateContext, + }), + ); + + expect(result.kind).toBe("success"); + expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { + provider: "claude-cli", + model: "claude-sonnet-4-6", + sourceReplyDeliveryMode: "message_tool_only", + allowEmptyAssistantReplyAsSilent: true, + messageChannel: "telegram", + messageProvider: "telegram", + }); + }); + + it("passes prepared CLI user turns to the runtime persistence boundary", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("codex-cli", "gpt-5.4"), + provider: "codex-cli", + model: "gpt-5.4", + attempts: [], + })); + state.runCliAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "final" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "codex-cli"; + followupRun.run.model = "gpt-5.4"; + const preparedUserTurnMessage = { + role: "user", + content: "describe this", + MediaPath: "/tmp/image.png", + MediaPaths: ["/tmp/image.png"], + MediaType: "image/png", + MediaTypes: ["image/png"], + } as never; + followupRun.userTurnTranscriptRecorder = createTestUserTurnRecorder(preparedUserTurnMessage); + const sessionEntry: SessionEntry = { + sessionId: "session", + sessionFile: "/tmp/session.jsonl", + updatedAt: 1, + }; + const activeSessionStore = { main: sessionEntry }; + + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + commandBody: "runtime prompt", + transcriptCommandBody: "display prompt", + activeSessionStore, + storePath: "/tmp/sessions.json", + getActiveSessionEntry: () => activeSessionStore.main, + }); + + expect(result.kind).toBe("success"); + expect(state.runCliAgentMock).toHaveBeenCalledOnce(); + expectMockCallArgFields(state.runCliAgentMock, 0, "CLI runtime", { + sessionKey: "main", + agentId: "agent", + sessionId: "session", + suppressNextUserMessagePersistence: false, + persistAssistantTranscript: true, + storePath: "/tmp/sessions.json", + }); + const call = requireMockCall(state.runCliAgentMock, 0, "CLI runtime"); + const callParams = requireRecord(call[0], "CLI runtime"); + expect(callParams.userTurnTranscriptRecorder).toEqual(expect.any(Object)); + expect(requireRecord(callParams.userTurnTranscriptRecorder, "user turn recorder").message).toBe( + preparedUserTurnMessage, + ); + expect(callParams.onUserMessagePersisted).toEqual(expect.any(Function)); + }); + + it("reuses CLI sessions for room-event turns", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("codex-cli", "gpt-5.4"), + provider: "codex-cli", + model: "gpt-5.4", + attempts: [], + })); + state.runCliAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "ambient" }], + meta: { + agentMeta: { + sessionId: "existing-cli-session", + cliSessionBinding: { + sessionId: "existing-cli-session", + authProfileId: "profile", + }, + }, + }, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.currentInboundEventKind = "room_event"; + followupRun.run.provider = "codex-cli"; + followupRun.run.model = "gpt-5.4"; + const sessionEntry = { + cliSessionBindings: { + "codex-cli": { sessionId: "existing-cli-session" }, + }, + } as unknown as SessionEntry; + const activeSessionStore = { main: sessionEntry }; + + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + activeSessionStore, + getActiveSessionEntry: () => sessionEntry, + }); + + expect(result.kind).toBe("success"); + expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { + currentInboundEventKind: "room_event", + persistAssistantTranscript: false, + cliSessionId: "existing-cli-session", + cliSessionBinding: { + sessionId: "existing-cli-session", + }, + }); + if (result.kind !== "success") { + throw new Error("expected success"); + } + expect(result.runResult.meta?.agentMeta?.sessionId).toBe("existing-cli-session"); + expect(result.runResult.meta?.agentMeta?.cliSessionBinding).toEqual({ + sessionId: "existing-cli-session", + authProfileId: "profile", + }); + }); + + it("keeps the first CLI session created by a room-event turn", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("codex-cli", "gpt-5.4"), + provider: "codex-cli", + model: "gpt-5.4", + attempts: [], + })); + state.runCliAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "ambient" }], + meta: { + agentMeta: { + sessionId: "new-cli-session", + cliSessionBinding: { + sessionId: "new-cli-session", + authProfileId: "profile", + }, + }, + }, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.currentInboundEventKind = "room_event"; + followupRun.run.provider = "codex-cli"; + followupRun.run.model = "gpt-5.4"; + const sessionEntry = {} as unknown as SessionEntry; + + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + getActiveSessionEntry: () => sessionEntry, + }); + + expect(result.kind).toBe("success"); + expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { + currentInboundEventKind: "room_event", + cliSessionId: undefined, + cliSessionBinding: undefined, + }); + if (result.kind !== "success") { + throw new Error("expected success"); + } + expect(result.runResult.meta?.agentMeta?.sessionId).toBe("new-cli-session"); + expect(result.runResult.meta?.agentMeta?.cliSessionBinding).toEqual({ + sessionId: "new-cli-session", + authProfileId: "profile", + }); + }); + + it("drops replacement room-event CLI sessions when reuse fails", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("codex-cli", "gpt-5.4"), + provider: "codex-cli", + model: "gpt-5.4", + attempts: [], + })); + state.runCliAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "ambient" }], + meta: { + agentMeta: { + sessionId: "transient-cli-session", + cliSessionBinding: { + sessionId: "transient-cli-session", + authProfileId: "profile", + }, + clearCliSessionBinding: true, + }, + }, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.currentInboundEventKind = "room_event"; + followupRun.run.provider = "codex-cli"; + followupRun.run.model = "gpt-5.4"; + const sessionEntry = { + cliSessionBindings: { + "codex-cli": { sessionId: "existing-cli-session" }, + }, + } as unknown as SessionEntry; + const activeSessionStore = { main: sessionEntry }; + + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + activeSessionStore, + getActiveSessionEntry: () => sessionEntry, + }); + + expect(result.kind).toBe("success"); + expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { + currentInboundEventKind: "room_event", + cliSessionId: "existing-cli-session", + cliSessionBinding: { + sessionId: "existing-cli-session", + }, + }); + if (result.kind !== "success") { + throw new Error("expected success"); + } + expect(result.runResult.meta?.agentMeta?.sessionId).toBe(""); + expect(result.runResult.meta?.agentMeta?.cliSessionBinding).toBeUndefined(); + expect(result.runResult.meta?.agentMeta?.clearCliSessionBinding).toBeUndefined(); + expect(activeSessionStore.main.cliSessionBindings?.["codex-cli"]).toBeUndefined(); + }); + + it("keeps room-event CLI bindings when synthetic hooks return no CLI binding", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("codex-cli", "gpt-5.4"), + provider: "codex-cli", + model: "gpt-5.4", + attempts: [], + })); + state.runCliAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "handled" }], + meta: { + agentMeta: { + sessionId: "openclaw-session", + provider: "codex-cli", + model: "gpt-5.4", + }, + }, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.currentInboundEventKind = "room_event"; + followupRun.run.provider = "codex-cli"; + followupRun.run.model = "gpt-5.4"; + const sessionEntry = { + cliSessionBindings: { + "codex-cli": { sessionId: "existing-cli-session" }, + }, + } as unknown as SessionEntry; + const activeSessionStore = { main: sessionEntry }; + + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + activeSessionStore, + getActiveSessionEntry: () => sessionEntry, + }); + + expect(result.kind).toBe("success"); + if (result.kind !== "success") { + throw new Error("expected success"); + } + expect(result.runResult.meta?.agentMeta?.sessionId).toBe(""); + expect(result.runResult.meta?.agentMeta?.cliSessionBinding).toBeUndefined(); + expect(activeSessionStore.main.cliSessionBindings?.["codex-cli"]).toEqual({ + sessionId: "existing-cli-session", + }); + }); + + it("clears room-event CLI bindings when an unflushed replacement is dropped", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("codex-cli", "gpt-5.4"), + provider: "codex-cli", + model: "gpt-5.4", + attempts: [], + })); + state.runCliAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "handled" }], + meta: { + agentMeta: { + sessionId: "", + provider: "codex-cli", + model: "gpt-5.4", + clearCliSessionBinding: true, + }, + }, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.currentInboundEventKind = "room_event"; + followupRun.run.provider = "codex-cli"; + followupRun.run.model = "gpt-5.4"; + const sessionEntry = { + cliSessionBindings: { + "codex-cli": { sessionId: "existing-cli-session" }, + }, + } as unknown as SessionEntry; + const activeSessionStore = { main: sessionEntry }; + + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + activeSessionStore, + getActiveSessionEntry: () => sessionEntry, + }); + + expect(result.kind).toBe("success"); + if (result.kind !== "success") { + throw new Error("expected success"); + } + expect(result.runResult.meta?.agentMeta?.sessionId).toBe(""); + expect(result.runResult.meta?.agentMeta?.cliSessionBinding).toBeUndefined(); + expect(result.runResult.meta?.agentMeta?.clearCliSessionBinding).toBeUndefined(); + expect(activeSessionStore.main.cliSessionBindings?.["codex-cli"]).toBeUndefined(); + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-command-events.test.ts b/src/auto-reply/reply/agent-runner-execution-command-events.test.ts new file mode 100644 index 000000000000..65c1fc99fdb2 --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-command-events.test.ts @@ -0,0 +1,315 @@ +import { describe, expect, it, vi } from "vitest"; +import type { TemplateContext } from "../templating.js"; +import type { GetReplyOptions } from "../types.js"; +import { + setupAgentRunnerExecutionTestState, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, +} from "./agent-runner-execution.test-support.js"; +import type { EmbeddedAgentParams } from "./agent-runner-execution.test-support.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: command events", () => { + it("forwards plan, approval, command output, and patch events", async () => { + const onPlanUpdate = vi.fn(); + const onApprovalEvent = vi.fn(); + const onCommandOutput = vi.fn(); + const onPatchSummary = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "plan", + data: { + phase: "update", + title: "Assistant proposed a plan", + explanation: "Inspect code, patch it, run tests.", + steps: ["Inspect code", "Patch code", "Run tests"], + }, + }); + await params.onAgentEvent?.({ + stream: "approval", + data: { + phase: "requested", + kind: "exec", + status: "pending", + title: "Command approval requested", + approvalId: "approval-1", + }, + }); + await params.onAgentEvent?.({ + stream: "command_output", + data: { + itemId: "command:exec-1", + phase: "delta", + title: "command ls", + toolCallId: "exec-1", + output: "README.md", + }, + }); + await params.onAgentEvent?.({ + stream: "patch", + data: { + itemId: "patch:patch-1", + phase: "end", + title: "apply patch", + toolCallId: "patch-1", + added: ["a.ts"], + modified: ["b.ts"], + deleted: [], + summary: "1 added, 1 modified", + }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const pendingToolTasks = new Set>(); + await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { + onPlanUpdate, + onApprovalEvent, + onCommandOutput, + onPatchSummary, + } satisfies GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks, + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(onPlanUpdate).toHaveBeenCalledWith({ + phase: "update", + title: "Assistant proposed a plan", + explanation: "Inspect code, patch it, run tests.", + steps: ["Inspect code", "Patch code", "Run tests"], + source: undefined, + }); + expect(onApprovalEvent).toHaveBeenCalledWith({ + phase: "requested", + kind: "exec", + status: "pending", + title: "Command approval requested", + itemId: undefined, + toolCallId: undefined, + approvalId: "approval-1", + approvalSlug: undefined, + command: undefined, + host: undefined, + reason: undefined, + scope: undefined, + message: undefined, + }); + expect(onCommandOutput).toHaveBeenCalledWith({ + itemId: "command:exec-1", + phase: "delta", + title: "command ls", + toolCallId: "exec-1", + name: undefined, + output: "README.md", + status: undefined, + exitCode: undefined, + durationMs: undefined, + cwd: undefined, + }); + expect(onPatchSummary).toHaveBeenCalledWith({ + itemId: "patch:patch-1", + phase: "end", + title: "apply patch", + toolCallId: "patch-1", + name: undefined, + added: ["a.ts"], + modified: ["b.ts"], + deleted: [], + summary: "1 added, 1 modified", + }); + }); + + it("forwards Codex command tool results as command output completion", async () => { + const onCommandOutput = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "tool", + data: { + phase: "result", + itemId: "command:exec-1", + toolCallId: "exec-1", + name: "exec", + status: "completed", + result: { + exitCode: 0, + durationMs: 42, + }, + }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onCommandOutput } satisfies GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(onCommandOutput).toHaveBeenCalledWith({ + itemId: "command:exec-1", + phase: "end", + title: undefined, + toolCallId: "exec-1", + name: "exec", + output: undefined, + status: "completed", + exitCode: 0, + durationMs: 42, + cwd: undefined, + }); + }); + + it("marks Codex command tool result errors as failed command output", async () => { + const onCommandOutput = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "tool", + data: { + phase: "result", + itemId: "command:exec-1", + toolCallId: "exec-1", + name: "exec", + isError: true, + result: { + content: [{ type: "text", text: "command failed" }], + }, + }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onCommandOutput } satisfies GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(onCommandOutput).toHaveBeenCalledWith( + expect.objectContaining({ + itemId: "command:exec-1", + phase: "end", + toolCallId: "exec-1", + name: "exec", + status: "failed", + }), + ); + }); + + it("does not synthesize command output from bare exec tool results", async () => { + const onCommandOutput = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "tool", + data: { + phase: "result", + name: "exec", + toolCallId: "exec-1", + isError: false, + }, + }); + await params.onAgentEvent?.({ + stream: "command_output", + data: { + itemId: "command:exec-1", + phase: "end", + title: "command ls", + toolCallId: "exec-1", + name: "exec", + status: "completed", + exitCode: 0, + }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onCommandOutput } satisfies GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(onCommandOutput).toHaveBeenCalledTimes(1); + expect(onCommandOutput).toHaveBeenCalledWith( + expect.objectContaining({ + itemId: "command:exec-1", + phase: "end", + status: "completed", + }), + ); + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-compaction.test.ts b/src/auto-reply/reply/agent-runner-execution-compaction.test.ts new file mode 100644 index 000000000000..9101ac67192f --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-compaction.test.ts @@ -0,0 +1,539 @@ +import { describe, expect, it, vi } from "vitest"; +import { resetLogger, setLoggerOverride } from "../../logging/logger.js"; +import { loggingState } from "../../logging/state.js"; +import type { TemplateContext } from "../templating.js"; +import { + setupAgentRunnerExecutionTestState, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, + expectBlockReplyCall, + createMinimalRunAgentTurnParams, +} from "./agent-runner-execution.test-support.js"; +import type { + FallbackRunnerParams, + EmbeddedAgentParams, +} from "./agent-runner-execution.test-support.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: compaction events", () => { + it("keeps compaction start notices silent by default", async () => { + const onBlockReply = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onBlockReply }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + expect(onBlockReply).not.toHaveBeenCalled(); + }); + + it("keeps compaction callbacks active when notices are silent by default", async () => { + const onBlockReply = vi.fn(); + const onCompactionStart = vi.fn(); + const onCompactionEnd = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); + await params.onAgentEvent?.({ + stream: "compaction", + data: { phase: "end", completed: true }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { + onBlockReply, + onCompactionStart, + onCompactionEnd, + }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + expect(onCompactionStart).toHaveBeenCalledTimes(1); + expect(onCompactionEnd).toHaveBeenCalledTimes(1); + expect(onBlockReply).not.toHaveBeenCalled(); + }); + + it("logs Codex app-server compaction completion while notices stay silent by default", async () => { + const onBlockReply = vi.fn(); + const consoleLog = vi.fn(); + setLoggerOverride({ level: "silent", consoleLevel: "info", consoleStyle: "compact" }); + loggingState.rawConsole = { + log: consoleLog, + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }; + try { + state.runWithModelFallbackMock.mockImplementationOnce( + async (params: FallbackRunnerParams) => ({ + result: await params.run("openai", "gpt-5.5"), + provider: "openai", + model: "gpt-5.5", + attempts: [{ provider: "anthropic", model: "claude", error: "rate limit" }], + }), + ); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "compaction", + data: { + phase: "start", + backend: "codex-app-server", + threadId: "thread-1", + turnId: "turn-1", + itemId: "compaction-1", + }, + }); + await params.onAgentEvent?.({ + stream: "compaction", + data: { + phase: "end", + completed: true, + backend: "codex-app-server", + threadId: "thread-1", + turnId: "turn-1", + itemId: "compaction-1", + }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ + opts: { onBlockReply }, + }), + }); + + expect(result.kind).toBe("success"); + expect(onBlockReply).not.toHaveBeenCalled(); + expect(consoleLog.mock.calls.map(([line]) => String(line)).join("\n")).toContain( + "codex app-server auto-compaction succeeded for openai/gpt-5.5; refreshed session context", + ); + } finally { + loggingState.rawConsole = null; + setLoggerOverride(null); + resetLogger(); + } + }); + + it("emits a compaction start notice when notifyUser is enabled", async () => { + const onBlockReply = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const followupRun = createFollowupRun(); + followupRun.run.config = { + agents: { + defaults: { + compaction: { + notifyUser: true, + }, + }, + }, + }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onBlockReply }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + expect(onBlockReply).toHaveBeenCalledTimes(1); + expectBlockReplyCall(onBlockReply, 0, { + text: "🧹 Compacting context...", + replyToId: "msg", + replyToCurrent: true, + isCompactionNotice: true, + }); + }); + + it("emits a compaction completion notice when notifyUser is enabled", async () => { + const onBlockReply = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); + await params.onAgentEvent?.({ + stream: "compaction", + data: { phase: "end", completed: true }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const followupRun = createFollowupRun(); + followupRun.run.config = { + agents: { + defaults: { + compaction: { + notifyUser: true, + }, + }, + }, + }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onBlockReply }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + expectBlockReplyCall(onBlockReply, 0, { + text: "🧹 Compacting context...", + replyToId: "msg", + replyToCurrent: true, + isCompactionNotice: true, + }); + expectBlockReplyCall(onBlockReply, 1, { + text: "🧹 Compaction complete", + replyToId: "msg", + replyToCurrent: true, + isCompactionNotice: true, + }); + }); + + it("delivers compaction hook messages alongside notifyUser notices (#90185)", async () => { + const onBlockReply = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "compaction", + data: { phase: "start", messages: ["Hook before"] }, + }); + await params.onAgentEvent?.({ + stream: "compaction", + data: { phase: "end", completed: true, messages: ["Hook after"] }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const followupRun = createFollowupRun(); + followupRun.run.config = { + agents: { + defaults: { + compaction: { + notifyUser: true, + }, + }, + }, + }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onBlockReply }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + expect(onBlockReply).toHaveBeenCalledTimes(4); + expectBlockReplyCall(onBlockReply, 0, { + text: "Hook before", + replyToId: "msg", + replyToCurrent: true, + isCompactionNotice: true, + }); + expectBlockReplyCall(onBlockReply, 1, { + text: "🧹 Compacting context...", + replyToId: "msg", + replyToCurrent: true, + isCompactionNotice: true, + }); + expectBlockReplyCall(onBlockReply, 2, { + text: "Hook after", + replyToId: "msg", + replyToCurrent: true, + isCompactionNotice: true, + }); + expectBlockReplyCall(onBlockReply, 3, { + text: "🧹 Compaction complete", + replyToId: "msg", + replyToCurrent: true, + isCompactionNotice: true, + }); + }); + + it("fires both notifyUser notices alongside onCompactionStart / onCompactionEnd callbacks (#87107)", async () => { + const onBlockReply = vi.fn(); + const onCompactionStart = vi.fn(); + const onCompactionEnd = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); + await params.onAgentEvent?.({ + stream: "compaction", + data: { phase: "end", completed: true }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const followupRun = createFollowupRun(); + followupRun.run.config = { + agents: { + defaults: { + compaction: { + notifyUser: true, + }, + }, + }, + }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onBlockReply, onCompactionStart, onCompactionEnd }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + // Internal callbacks (Control UI etc.) and the user-channel notifyUser + // notices are independent audiences; both must fire when opted in. + expect(onCompactionStart).toHaveBeenCalledTimes(1); + expect(onCompactionEnd).toHaveBeenCalledTimes(1); + expect(onBlockReply).toHaveBeenCalledTimes(2); + expectBlockReplyCall(onBlockReply, 0, { + text: "🧹 Compacting context...", + isCompactionNotice: true, + }); + expectBlockReplyCall(onBlockReply, 1, { + text: "🧹 Compaction complete", + isCompactionNotice: true, + }); + }); + + it("emits an incomplete compaction notice when compaction ends without completing", async () => { + const onBlockReply = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); + await params.onAgentEvent?.({ + stream: "compaction", + data: { phase: "end", completed: false }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const followupRun = createFollowupRun(); + followupRun.run.config = { + agents: { + defaults: { + compaction: { + notifyUser: true, + }, + }, + }, + }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onBlockReply }, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + expectBlockReplyCall(onBlockReply, 0, { + text: "🧹 Compacting context...", + isCompactionNotice: true, + }); + expectBlockReplyCall(onBlockReply, 1, { + text: "🧹 Compaction incomplete", + isCompactionNotice: true, + }); + }); + + it("uses the compaction notice fallback when no block-reply dispatcher is wired", async () => { + const onCompactionNoticePayload = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); + await params.onAgentEvent?.({ + stream: "compaction", + data: { phase: "end", completed: true }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const followupRun = createFollowupRun(); + followupRun.run.config = { + agents: { + defaults: { + compaction: { + notifyUser: true, + }, + }, + }, + }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + onCompactionNoticePayload, + }); + + expect(result.kind).toBe("success"); + expect(onCompactionNoticePayload).toHaveBeenCalledTimes(2); + expectBlockReplyCall(onCompactionNoticePayload, 0, { + text: "🧹 Compacting context...", + replyToId: "msg", + replyToCurrent: true, + isCompactionNotice: true, + }); + expectBlockReplyCall(onCompactionNoticePayload, 1, { + text: "🧹 Compaction complete", + replyToId: "msg", + replyToCurrent: true, + isCompactionNotice: true, + }); + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-context-failures.test.ts b/src/auto-reply/reply/agent-runner-execution-context-failures.test.ts new file mode 100644 index 000000000000..afba9bae4d25 --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-context-failures.test.ts @@ -0,0 +1,216 @@ +import { describe, expect, it, vi } from "vitest"; +import { FailoverError } from "../../agents/failover-error.js"; +import type { SessionEntry } from "../../config/sessions.js"; +import { getReplyPayloadMetadata } from "../reply-payload.js"; +import type { TemplateContext } from "../templating.js"; +import { + setupAgentRunnerExecutionTestState, + GENERIC_RUN_FAILURE_TEXT, + makeTestModel, + getRunAgentTurnWithFallback, + createFollowupRun, + createMockReplyOperation, + requireRecord, + expectRecordFields, + createMinimalRunAgentTurnParams, +} from "./agent-runner-execution.test-support.js"; +import type { FallbackRunnerParams } from "./agent-runner-execution.test-support.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: context failures", () => { + it("preserves the active session when embedded overflow recovery fails", async () => { + state.isContextOverflowErrorMock.mockReturnValue(true); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [], + meta: { + error: { + message: "400 The prompt is too long: 203557, model maximum context length: 196607", + }, + }, + }); + + const activeSessionEntry = { sessionId: "session", updatedAt: 1 } as SessionEntry; + const activeSessionStore = { "agent:main:main": activeSessionEntry }; + const { replyOperation, failMock, updateSessionIdMock } = createMockReplyOperation(); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ + sessionCtx: { + Provider: "webchat", + MessageSid: "msg", + } as unknown as TemplateContext, + }), + replyOperation, + sessionKey: "agent:main:main", + getActiveSessionEntry: () => activeSessionEntry, + activeSessionStore, + storePath: "/tmp/sessions.json", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toContain("kept this conversation mapped to the current session"); + expect(result.payload.text).toContain("reserveTokensFloor"); + expectRecordFields(requireRecord(getReplyPayloadMetadata(result.payload), "reply metadata"), { + deliverDespiteSourceReplySuppression: true, + }); + } + expect(failMock).toHaveBeenCalledWith( + "run_failed", + expect.objectContaining({ + message: "400 The prompt is too long: 203557, model maximum context length: 196607", + }), + ); + expect(activeSessionStore["agent:main:main"]?.sessionId).toBe("session"); + expect(updateSessionIdMock).not.toHaveBeenCalled(); + expect(state.updateSessionStoreMock).not.toHaveBeenCalled(); + }); + + it("preserves the active session when compaction failure is thrown before reply", async () => { + state.isCompactionFailureErrorMock.mockReturnValue(true); + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error("Auto-compaction failed: nothing to compact"), + ); + + const activeSessionEntry = { sessionId: "session", updatedAt: 1 } as SessionEntry; + const activeSessionStore = { "agent:main:main": activeSessionEntry }; + const { replyOperation, failMock, updateSessionIdMock } = createMockReplyOperation(); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ + sessionCtx: { + Provider: "webchat", + MessageSid: "msg", + } as unknown as TemplateContext, + }), + replyOperation, + sessionKey: "agent:main:main", + getActiveSessionEntry: () => activeSessionEntry, + activeSessionStore, + storePath: "/tmp/sessions.json", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toContain("kept this conversation mapped to the current session"); + expect(result.payload.text).toContain("reserveTokensFloor"); + expectRecordFields(requireRecord(getReplyPayloadMetadata(result.payload), "reply metadata"), { + deliverDespiteSourceReplySuppression: true, + }); + } + expect(failMock).toHaveBeenCalledWith( + "run_failed", + expect.objectContaining({ message: "Auto-compaction failed: nothing to compact" }), + ); + expect(activeSessionStore["agent:main:main"]?.sessionId).toBe("session"); + expect(updateSessionIdMock).not.toHaveBeenCalled(); + expect(state.updateSessionStoreMock).not.toHaveBeenCalled(); + }); + + it.each([ + { + reason: "server_error" as const, + message: "upstream provider failed briefly", + }, + { + reason: "timeout" as const, + message: "provider request timed out without status token", + }, + ])( + "retries once for structured FailoverError $reason without leading HTTP status text", + async ({ reason, message }) => { + vi.useFakeTimers(); + state.runEmbeddedAgentMock + .mockRejectedValueOnce( + new FailoverError(message, { + reason, + provider: "openai", + model: "gpt-5.5", + }), + ) + .mockResolvedValueOnce({ + payloads: [{ text: "recovered after transient failover" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const resultPromise = runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + await vi.advanceTimersByTimeAsync(2_500); + const result = await resultPromise; + + expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(2); + expect(result.kind).toBe("success"); + if (result.kind === "success") { + expect(result.runResult.payloads?.[0]?.text).toBe("recovered after transient failover"); + } + }, + ); + + it("uses structured FailoverError context_overflow over non-overflow message text", async () => { + state.isLikelyContextOverflowErrorMock.mockReturnValue(false); + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new FailoverError("provider rejected the request payload", { + reason: "context_overflow", + provider: "anthropic", + model: "claude", + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: { + Provider: "telegram", + Surface: "telegram", + ChatType: "direct", + MessageSid: "msg", + } as unknown as TemplateContext, + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + "⚠️ Context overflow — prompt too large for this model. Try a shorter message or a larger-context model.", + ); + expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); + expect(result.payload.text).not.toContain("provider rejected the request payload"); + } + }); + + it("uses the throwing fallback candidate model for compaction failure hints", async () => { + state.isCompactionFailureErrorMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + await params.run("custom", "uncataloged-32k"); + throw new Error("expected fallback candidate to throw"); + }); + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error("Auto-compaction failed: nothing to compact"), + ); + + const followupRun = createFollowupRun(); + followupRun.run.provider = "openrouter"; + followupRun.run.model = "qwen3.6-plus"; + followupRun.run.config = { + models: { + providers: { + openrouter: { + baseUrl: "https://openrouter.test", + models: [makeTestModel("qwen3.6-plus", 1_000_000)], + }, + }, + }, + }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams({ followupRun })); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toContain("reserveTokensFloor"); + expect(result.payload.text).toContain("20000"); + expect(result.payload.text).not.toContain("100000"); + } + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-conversation-failures.test.ts b/src/auto-reply/reply/agent-runner-execution-conversation-failures.test.ts new file mode 100644 index 000000000000..09404789e059 --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-conversation-failures.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, it, vi } from "vitest"; +import type { TemplateContext } from "../templating.js"; +import { + setupAgentRunnerExecutionTestState, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, +} from "./agent-runner-execution.test-support.js"; +import { PROVIDER_CONVERSATION_STATE_ERROR_USER_MESSAGE } from "./provider-request-error-classifier.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: conversation failures", () => { + it("returns a session reset hint for Bedrock tool mismatch errors on external chat channels", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error( + "The number of toolResult blocks at messages.186.content exceeds the number of toolUse blocks of previous turn.", + ), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe(PROVIDER_CONVERSATION_STATE_ERROR_USER_MESSAGE); + } + }); + + it("returns a provider conversation-state error for OpenAI missing custom tool output errors on external chat channels", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error("Custom tool call output is missing for call id: call_live_123."), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "slack", + ChannelId: "channel-1", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe(PROVIDER_CONVERSATION_STATE_ERROR_USER_MESSAGE); + } + }); + + it("does not auto-reset role-ordering provider conversation-state errors", async () => { + const resetSessionAfterRoleOrderingConflict = vi.fn(async () => true); + state.runEmbeddedAgentMock.mockRejectedValueOnce(new Error("400 Incorrect role information")); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "telegram", + ChatId: "chat-1", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(resetSessionAfterRoleOrderingConflict).not.toHaveBeenCalled(); + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe(PROVIDER_CONVERSATION_STATE_ERROR_USER_MESSAGE); + } + }); + + it("keeps raw generic errors on internal control surfaces", async () => { + state.isInternalMessageChannelMock.mockReturnValue(true); + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error("INVALID_ARGUMENT: some other failure"), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "chat", + Surface: "chat", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toContain("Agent failed before reply"); + expect(result.payload.text).toContain("INVALID_ARGUMENT: some other failure"); + expect(result.payload.text).toContain("Logs: openclaw logs --follow"); + } + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-lifecycle.test.ts b/src/auto-reply/reply/agent-runner-execution-lifecycle.test.ts new file mode 100644 index 000000000000..1c6ce47e83ac --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-lifecycle.test.ts @@ -0,0 +1,500 @@ +import { describe, expect, it, vi } from "vitest"; +import { createAgentRunRestartAbortError } from "../../agents/run-termination.js"; +import { HEARTBEAT_RUN_SCOPE } from "../../infra/heartbeat-run-scope.js"; +import { SILENT_REPLY_TOKEN } from "../tokens.js"; +import type { GetReplyOptions } from "../types.js"; +import { + setupAgentRunnerExecutionTestState, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, + createMockReplyOperation, + requireRecord, + expectMockCallArgFields, + createMinimalRunAgentTurnParams, +} from "./agent-runner-execution.test-support.js"; +import type { + FallbackRunnerParams, + EmbeddedAgentParams, +} from "./agent-runner-execution.test-support.js"; +import { createReplyOperation, type ReplyOperation } from "./reply-run-registry.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: run lifecycle and ownership", () => { + it("passes the reply abort signal to fallback orchestration and candidates", async () => { + const { replyOperation } = createMockReplyOperation(); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "ok" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams(), + replyOperation, + }); + + const fallbackCall = requireRecord( + state.runWithModelFallbackMock.mock.calls[0]?.[0], + "runWithModelFallback params", + ); + const embeddedCall = requireRecord( + state.runEmbeddedAgentMock.mock.calls[0]?.[0], + "runEmbeddedAgent params", + ); + expect(fallbackCall.abortSignal).toBe(replyOperation.abortSignal); + expect(fallbackCall.sessionId).toBe("session"); + expect(embeddedCall.abortSignal).toBe(replyOperation.abortSignal); + }); + + it("revalidates thinking for each main-chat fallback candidate without mutating the run", async () => { + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-5.6-sol"; + followupRun.run.thinkLevel = "ultra"; + followupRun.run.config = { + agents: { + defaults: { + models: { + "openai/gpt-5.6-sol": { agentRuntime: { id: "openclaw" } }, + "demo/basic": { agentRuntime: { id: "openclaw" } }, + }, + }, + }, + }; + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + await params.run("openai", "gpt-5.6-sol"); + const result = await params.run("demo", "basic"); + return { result, provider: "demo", model: "basic", attempts: [] }; + }); + state.runEmbeddedAgentMock.mockResolvedValue({ payloads: [{ text: "ok" }], meta: {} }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + }); + + expect(state.runEmbeddedAgentMock.mock.calls.map((call) => call[0]?.thinkLevel)).toEqual([ + "ultra", + "high", + ]); + expect(followupRun.run.thinkLevel).toBe("ultra"); + }); + + it("freezes abort ownership only after model fallback settles", async () => { + const { replyOperation, freezeAbortMock } = createMockReplyOperation(); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + expect(freezeAbortMock).not.toHaveBeenCalled(); + await params.run("anthropic", "claude").catch(() => undefined); + expect(freezeAbortMock).not.toHaveBeenCalled(); + const result = await params.run("openai", "gpt-5.5"); + expect(freezeAbortMock).not.toHaveBeenCalled(); + return { + result, + provider: "openai", + model: "gpt-5.5", + attempts: [], + }; + }); + state.runEmbeddedAgentMock + .mockRejectedValueOnce(new Error("primary failed")) + .mockResolvedValueOnce({ + payloads: [{ text: "ok" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams(), + replyOperation, + }); + + expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(2); + expect(freezeAbortMock).toHaveBeenCalledTimes(1); + }); + + it("suppresses a settled fallback result after an accepted user abort", async () => { + const { replyOperation, freezeAbortMock } = createMockReplyOperation(); + const abortController = new AbortController(); + let operationResult: ReplyOperation["result"] = null; + let releaseFallback: () => void = () => undefined; + let markCandidateSettled: () => void = () => undefined; + const candidateSettled = new Promise((resolve) => { + markCandidateSettled = resolve; + }); + const fallbackRelease = new Promise((resolve) => { + releaseFallback = resolve; + }); + let releaseToolTask: () => void = () => undefined; + const pendingToolTask = new Promise((resolve) => { + releaseToolTask = resolve; + }); + const pendingToolTasks = new Set([pendingToolTask]); + Object.defineProperty(replyOperation, "abortSignal", { + configurable: true, + get: () => abortController.signal, + }); + Object.defineProperty(replyOperation, "result", { + configurable: true, + get: () => operationResult, + }); + replyOperation.abortByUser = vi.fn(() => { + operationResult = { kind: "aborted", code: "aborted_by_user" }; + abortController.abort("user_abort"); + return true; + }); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "late reply" }], + meta: {}, + }); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + const result = await params.run("anthropic", "claude"); + markCandidateSettled(); + await fallbackRelease; + return { + result, + provider: "anthropic", + model: "claude", + attempts: [], + }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const pending = runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams(), + replyOperation, + pendingToolTasks, + }); + await candidateSettled; + expect(replyOperation.abortByUser()).toBe(true); + let settled = false; + void pending.then(() => { + settled = true; + }); + releaseFallback(); + await new Promise((resolve) => { + setImmediate(resolve); + }); + expect(settled).toBe(false); + expect(freezeAbortMock).not.toHaveBeenCalled(); + releaseToolTask(); + + await expect(pending).resolves.toEqual({ + kind: "final", + payload: { text: SILENT_REPLY_TOKEN }, + }); + expect(freezeAbortMock).toHaveBeenCalledTimes(1); + }); + + it("suppresses a settled fallback result after an upstream abort", async () => { + const upstreamAbort = new AbortController(); + const replyOperation = createReplyOperation({ + sessionKey: "agent:main:upstream-settled-fallback", + sessionId: "upstream-settled-fallback", + resetTriggered: false, + upstreamAbortSignal: upstreamAbort.signal, + }); + replyOperation.setPhase("running"); + let releaseFallback: () => void = () => undefined; + let markCandidateSettled: () => void = () => undefined; + const candidateSettled = new Promise((resolve) => { + markCandidateSettled = resolve; + }); + const fallbackRelease = new Promise((resolve) => { + releaseFallback = resolve; + }); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "late reply" }], + meta: {}, + }); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + const result = await params.run("anthropic", "claude"); + markCandidateSettled(); + await fallbackRelease; + return { + result, + provider: "anthropic", + model: "claude", + attempts: [], + }; + }); + + try { + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const pending = runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams(), + replyOperation, + }); + await candidateSettled; + upstreamAbort.abort(new Error("caller cancelled")); + expect(replyOperation.result).toEqual({ kind: "aborted", code: "aborted_by_user" }); + expect(replyOperation.abortSignal.aborted).toBe(true); + releaseFallback(); + + await expect(pending).resolves.toEqual({ + kind: "final", + payload: { text: SILENT_REPLY_TOKEN }, + }); + } finally { + replyOperation.complete(); + } + }); + + it("preserves restart reply classification after an upstream abort settles fallback", async () => { + const upstreamAbort = new AbortController(); + const replyOperation = createReplyOperation({ + sessionKey: "agent:main:upstream-settled-restart", + sessionId: "upstream-settled-restart", + resetTriggered: false, + upstreamAbortSignal: upstreamAbort.signal, + }); + replyOperation.setPhase("running"); + let releaseFallback: () => void = () => undefined; + let markCandidateSettled: () => void = () => undefined; + const candidateSettled = new Promise((resolve) => { + markCandidateSettled = resolve; + }); + const fallbackRelease = new Promise((resolve) => { + releaseFallback = resolve; + }); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "late reply" }], + meta: {}, + }); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + const result = await params.run("anthropic", "claude"); + markCandidateSettled(); + await fallbackRelease; + return { + result, + provider: "anthropic", + model: "claude", + attempts: [], + }; + }); + + try { + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const pending = runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams(), + replyOperation, + }); + await candidateSettled; + upstreamAbort.abort(createAgentRunRestartAbortError()); + releaseFallback(); + + await expect(pending).resolves.toEqual({ + kind: "final", + payload: { + isError: true, + text: "⚠️ Gateway is restarting. Please wait a few seconds and try again.", + }, + }); + } finally { + replyOperation.complete(); + } + }); + + it("passes the hydrated run account to embedded execution", async () => { + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "ok" }], + meta: {}, + }); + const followupRun = createFollowupRun(); + followupRun.run.agentAccountId = "work"; + followupRun.originatingChannel = "slack"; + followupRun.originatingTo = "user:U1"; + followupRun.originatingAccountId = "work"; + followupRun.originatingChatType = "direct"; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + followupRun, + sessionCtx: { + Provider: "cron-event", + }, + }), + ); + + expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { + messageProvider: "slack", + messageTo: "user:U1", + agentAccountId: "work", + chatType: "direct", + }); + }); + + it("signals typing from embedded harness execution phases before assistant text", async () => { + const typingSignals = createMockTypingSignaler(); + const onAgentRunStart = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + params.onExecutionPhase?.({ + phase: "model_call_started", + provider: "openai", + model: "gpt-5.4", + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ + opts: { + onAgentRunStart, + } satisfies GetReplyOptions, + }), + typingSignals, + }); + + expect(result.kind).toBe("success"); + expect(typingSignals.signalExecutionActivity).toHaveBeenCalledOnce(); + expect(typingSignals.signalRunStart).not.toHaveBeenCalled(); + expect(onAgentRunStart).toHaveBeenCalledOnce(); + }); + + it("forwards CLI harness execution phases into typing signals", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("codex-cli", "gpt-5.4"), + provider: "codex-cli", + model: "gpt-5.4", + attempts: [], + })); + state.runCliAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + params.onExecutionPhase?.({ + phase: "process_spawned", + provider: "codex-cli", + model: "gpt-5.4", + backend: "codex", + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + const followupRun = createFollowupRun(); + followupRun.run.provider = "codex-cli"; + followupRun.run.model = "gpt-5.4"; + followupRun.run.clientCaps = ["tool-events", "inline-widgets"]; + const typingSignals = createMockTypingSignaler(); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + followupRun, + typingSignals, + }), + ); + + expect(result.kind).toBe("success"); + expect(typingSignals.signalExecutionActivity).toHaveBeenCalledOnce(); + expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { + provider: "codex-cli", + model: "gpt-5.4", + clientCaps: ["tool-events", "inline-widgets"], + }); + }); + + it("propagates commitment-only bootstrap scope to CLI runs", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "sonnet-4.6"), + provider: "claude-cli", + model: "sonnet-4.6", + attempts: [], + })); + state.runCliAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "final" }], + meta: {}, + }); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "sonnet-4.6"; + const params = createMinimalRunAgentTurnParams({ + followupRun, + opts: { + isHeartbeat: true, + bootstrapContextMode: "lightweight", + [HEARTBEAT_RUN_SCOPE]: "commitment-only", + }, + }); + params.isHeartbeat = true; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback(params); + + expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { + trigger: "heartbeat", + bootstrapContextMode: "lightweight", + bootstrapContextRunKind: "commitment-only", + }); + }); + + it("registers run ownership before asynchronous image preflight", async () => { + const agentEvents = await import("../../infra/agent-events.js"); + const registerAgentRunContext = vi.mocked(agentEvents.registerAgentRunContext); + let resolveImages: (() => void) | undefined; + state.resolveCurrentTurnImagesMock.mockImplementationOnce( + () => + new Promise>((resolve) => { + resolveImages = () => resolve({}); + }), + ); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "ok" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const runPromise = runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(registerAgentRunContext).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + sessionKey: "main", + sessionId: "session", + }), + ); + expect(state.runWithModelFallbackMock).not.toHaveBeenCalled(); + + resolveImages?.(); + await runPromise; + }); + + it("clears run ownership when image preflight fails", async () => { + const agentEvents = await import("../../infra/agent-events.js"); + const clearAgentRunContext = vi.mocked(agentEvents.clearAgentRunContext); + state.resolveCurrentTurnImagesMock.mockRejectedValueOnce(new Error("invalid image metadata")); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await expect( + runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + opts: { runId: "preflight-failure" }, + }), + ), + ).rejects.toThrow("invalid image metadata"); + + expect(clearAgentRunContext).toHaveBeenCalledWith("preflight-failure", expect.any(String)); + expect(state.runWithModelFallbackMock).not.toHaveBeenCalled(); + }); + + it("passes runtime toolsAllow to embedded agent runs", async () => { + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "ok" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + opts: { + toolsAllow: ["message"], + }, + }), + ); + + expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { + toolsAllow: ["message"], + }); + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-message-tools.test.ts b/src/auto-reply/reply/agent-runner-execution-message-tools.test.ts new file mode 100644 index 000000000000..0a090632130f --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-message-tools.test.ts @@ -0,0 +1,393 @@ +import { describe, expect, it, vi } from "vitest"; +import type { TemplateContext } from "../templating.js"; +import type { GetReplyOptions } from "../types.js"; +import { + setupAgentRunnerExecutionTestState, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, +} from "./agent-runner-execution.test-support.js"; +import type { + FallbackRunnerParams, + EmbeddedAgentParams, +} from "./agent-runner-execution.test-support.js"; +import type { InternalGetReplyOptions } from "./get-reply.types.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: message tool progress", () => { + it("suppresses progress callbacks after message-tool-only delivery completes", async () => { + let releaseItemEvent: (() => void) | undefined; + const itemEventGate = new Promise((resolve) => { + releaseItemEvent = resolve; + }); + let markItemEventStarted: (() => void) | undefined; + const itemEventStarted = new Promise((resolve) => { + markItemEventStarted = resolve; + }); + const onItemEvent = vi.fn(async () => { + markItemEventStarted?.(); + await itemEventGate; + }); + const onCommandOutput = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "tool", + data: { + phase: "start", + name: "message", + toolCallId: "message-1", + args: { + action: "send", + message: "Visible reply", + }, + }, + }); + const itemEventPromise = params.onAgentEvent?.({ + stream: "item", + data: { + itemId: "tool-message-1", + phase: "end", + kind: "tool", + title: "message", + name: "message", + toolCallId: "message-1", + status: "completed", + }, + }); + await itemEventStarted; + await params.onAgentEvent?.({ + stream: "command_output", + data: { + itemId: "command:exec-1", + phase: "end", + title: "command false", + toolCallId: "exec-1", + name: "exec", + output: "failed command output", + status: "failed", + exitCode: 1, + }, + }); + await params.onAgentEvent?.({ + stream: "assistant", + data: { + phase: "commentary", + itemId: "commentary-1", + text: "This must stay suppressed.", + }, + }); + releaseItemEvent?.(); + await itemEventPromise; + return { payloads: [{ text: "NO_REPLY" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.sourceReplyDeliveryMode = "message_tool_only"; + await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "discord", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { + onItemEvent, + onCommandOutput, + progressPreambleEnabled: true, + } satisfies InternalGetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "on", + }); + + expect(onItemEvent).toHaveBeenCalledWith( + expect.objectContaining({ + name: "message", + phase: "end", + status: "completed", + }), + ); + expect(onItemEvent).toHaveBeenCalledTimes(1); + expect(onCommandOutput).not.toHaveBeenCalled(); + }); + + it("preserves message-tool-only suppression across fallback candidates", async () => { + const onItemEvent = vi.fn(); + const onCommandOutput = vi.fn(); + state.runEmbeddedAgentMock + .mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "tool", + data: { + phase: "start", + name: "message", + toolCallId: "message-1", + args: { action: "send", message: "Visible reply" }, + }, + }); + await params.onAgentEvent?.({ + stream: "item", + data: { + itemId: "tool-message-1", + phase: "end", + kind: "tool", + name: "message", + toolCallId: "message-1", + status: "completed", + }, + }); + return { payloads: [], meta: {} }; + }) + .mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "command_output", + data: { + itemId: "command:exec-1", + phase: "end", + name: "exec", + output: "must stay suppressed", + status: "completed", + exitCode: 0, + }, + }); + return { payloads: [{ text: "NO_REPLY" }], meta: {} }; + }); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + await params.run("anthropic", "primary"); + return { + result: await params.run("openai", "fallback"), + provider: "openai", + model: "fallback", + attempts: [], + }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.sourceReplyDeliveryMode = "message_tool_only"; + await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { Provider: "discord", MessageSid: "msg" } as unknown as TemplateContext, + opts: { onItemEvent, onCommandOutput } satisfies GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "on", + }); + + expect(onItemEvent).toHaveBeenCalledTimes(1); + expect(onCommandOutput).not.toHaveBeenCalled(); + }); + + it("keeps opted-in progress callbacks active after message-tool-only delivery completes", async () => { + const onToolStart = vi.fn(); + const onCommandOutput = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "tool", + data: { + phase: "start", + name: "message", + toolCallId: "message-1", + args: { + action: "send", + message: "Visible reply", + }, + }, + }); + await params.onAgentEvent?.({ + stream: "item", + data: { + itemId: "tool-message-1", + phase: "end", + kind: "tool", + title: "message", + name: "message", + toolCallId: "message-1", + status: "completed", + }, + }); + await params.onAgentEvent?.({ + stream: "tool", + data: { + phase: "start", + name: "bash", + toolCallId: "bash-1", + args: { + command: "sleep 6", + }, + }, + }); + await params.onAgentEvent?.({ + stream: "command_output", + data: { + itemId: "command:bash-1", + phase: "end", + title: "sleep 6", + toolCallId: "bash-1", + name: "bash", + output: "done", + status: "completed", + exitCode: 0, + }, + }); + return { payloads: [{ text: "NO_REPLY" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.sourceReplyDeliveryMode = "message_tool_only"; + await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "discord", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { + allowProgressCallbacksWhenSourceDeliverySuppressed: true, + onToolStart, + onCommandOutput, + } satisfies GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "on", + }); + + expect(onToolStart).toHaveBeenCalledWith( + expect.objectContaining({ + name: "bash", + phase: "start", + args: { command: "sleep 6" }, + detailMode: undefined, + }), + ); + expect(onCommandOutput).toHaveBeenCalledWith( + expect.objectContaining({ + name: "bash", + output: "done", + status: "completed", + }), + ); + }); + + it("keeps progress callbacks active after message-tool-only reads", async () => { + const onItemEvent = vi.fn(); + const onCommandOutput = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "tool", + data: { + phase: "start", + name: "message", + toolCallId: "message-read-1", + args: { + action: "read", + threadId: "thread-1", + }, + }, + }); + await params.onAgentEvent?.({ + stream: "item", + data: { + itemId: "tool-message-1", + phase: "end", + kind: "tool", + title: "message", + name: "message", + toolCallId: "message-read-1", + status: "completed", + }, + }); + await params.onAgentEvent?.({ + stream: "command_output", + data: { + itemId: "command:exec-1", + phase: "end", + title: "command false", + toolCallId: "exec-1", + name: "exec", + output: "failed command output", + status: "failed", + exitCode: 1, + }, + }); + return { payloads: [{ text: "NO_REPLY" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.sourceReplyDeliveryMode = "message_tool_only"; + await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "discord", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { + onItemEvent, + onCommandOutput, + } satisfies GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "on", + }); + + expect(onItemEvent).toHaveBeenCalledWith( + expect.objectContaining({ + name: "message", + phase: "end", + status: "completed", + }), + ); + expect(onCommandOutput).toHaveBeenCalledWith( + expect.objectContaining({ + output: "failed command output", + status: "failed", + }), + ); + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-probes.test.ts b/src/auto-reply/reply/agent-runner-execution-probes.test.ts new file mode 100644 index 000000000000..f8a8c239b549 --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-probes.test.ts @@ -0,0 +1,762 @@ +import { describe, expect, it, vi } from "vitest"; +import { FailoverError } from "../../agents/failover-error.js"; +import { LiveSessionModelSwitchError } from "../../agents/live-model-switch-error.js"; +import type { SessionEntry } from "../../config/sessions.js"; +import { resolveFallbackCandidateRun } from "./agent-runner-auth-profile.js"; +import { resolveRunAfterAutoFallbackPrimaryProbeRecheck } from "./agent-runner-auto-fallback.js"; +import { + setupAgentRunnerExecutionTestState, + GENERIC_RUN_FAILURE_TEXT, + getRunAgentTurnWithFallback, + createFollowupRun, + createMockReplyOperation, + expectRecordFields, + expectMockCallArgFields, + createMinimalRunAgentTurnParams, +} from "./agent-runner-execution.test-support.js"; +import type { + FallbackRunnerParams, + EmbeddedAgentParams, +} from "./agent-runner-execution.test-support.js"; +import { HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT } from "./agent-runner-failure-copy.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: primary probe routing", () => { + it("rechecks queued auto fallback primary probes before running", async () => { + const { markAutoFallbackPrimaryProbe } = await import("../../agents/agent-scope.js"); + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + fallbackAuthProfileId: "google:fallback", + fallbackAuthProfileIdSource: "auto" as const, + }; + markAutoFallbackPrimaryProbe({ + probe, + sessionKey: "main", + now: Date.now(), + }); + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + authProfileOverride: "google:fallback", + authProfileOverrideSource: "auto", + }; + const run = createFollowupRun().run; + run.provider = "anthropic"; + run.model = "claude-sonnet-4-6"; + run.authProfileId = "anthropic:primary"; + run.authProfileIdSource = "auto"; + run.autoFallbackPrimaryProbe = probe; + + expect( + resolveRunAfterAutoFallbackPrimaryProbeRecheck({ + run, + entry: sessionEntry, + sessionKey: "main", + }), + ).toMatchObject({ + provider: "google", + model: "gemini-3.1-pro-preview", + authProfileId: "google:fallback", + authProfileIdSource: "auto", + autoFallbackPrimaryProbe: undefined, + }); + }); + + it("drops stale queued primary probes after a user model switch", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + }; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + modelOverride: "openai/gpt-5.4", + modelOverrideSource: "user", + authProfileOverride: "openai:work", + authProfileOverrideSource: "user", + }; + const run = createFollowupRun().run; + run.provider = "anthropic"; + run.model = "claude-sonnet-4-6"; + run.autoFallbackPrimaryProbe = probe; + + expect( + resolveRunAfterAutoFallbackPrimaryProbeRecheck({ + run, + entry: sessionEntry, + sessionKey: "main", + }), + ).toMatchObject({ + provider: "openai", + model: "gpt-5.4", + authProfileId: "openai:work", + authProfileIdSource: "user", + modelOverrideSource: "user", + autoFallbackPrimaryProbe: undefined, + }); + }); + + it("propagates rechecked user selections to post-run state", async () => { + const sessionKey = "rechecked-user-selection"; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "openai", + modelOverride: "gpt-5.4", + modelOverrideSource: "user", + authProfileOverride: "openai:work", + authProfileOverrideSource: "user", + }; + const activeSessionStore = { [sessionKey]: sessionEntry }; + const staleAutoEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + }; + const followupRun = createFollowupRun(); + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-sonnet-4-6"; + followupRun.run.autoFallbackPrimaryProbe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + }; + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run(params.provider, params.model), + provider: params.provider, + model: params.model, + attempts: [], + })); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "user model" }], + meta: { + agentMeta: { + provider: "openai", + model: "gpt-5.4", + }, + }, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + sessionKey, + activeSessionStore, + getActiveSessionEntry: () => staleAutoEntry, + }); + + expectRecordFields(followupRun.run as unknown as Record, { + provider: "openai", + model: "gpt-5.4", + authProfileId: "openai:work", + authProfileIdSource: "user", + modelOverrideSource: "user", + }); + expect(followupRun.run.autoFallbackPrimaryProbe).toBeUndefined(); + expectRecordFields(activeSessionStore[sessionKey] as unknown as Record, { + providerOverride: "openai", + modelOverride: "gpt-5.4", + modelOverrideSource: "user", + }); + }); + + it("drops stale queued probe metadata after the auto fallback pin is cleared", () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + }; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + authProfileOverride: "google:fallback", + authProfileOverrideSource: "user", + }; + const run = createFollowupRun().run; + run.provider = "anthropic"; + run.model = "claude-sonnet-4-6"; + run.hasSessionModelOverride = true; + run.modelOverrideSource = "auto"; + run.hasAutoFallbackProvenance = true; + run.autoFallbackPrimaryProbe = probe; + + expect( + resolveRunAfterAutoFallbackPrimaryProbeRecheck({ + run, + entry: sessionEntry, + sessionKey: "main", + }), + ).toMatchObject({ + provider: "anthropic", + model: "claude-sonnet-4-6", + autoFallbackPrimaryProbe: undefined, + }); + const rechecked = resolveRunAfterAutoFallbackPrimaryProbeRecheck({ + run, + entry: sessionEntry, + sessionKey: "main", + }); + expect(rechecked.authProfileId).toBeUndefined(); + expect(rechecked.authProfileIdSource).toBeUndefined(); + expect(rechecked.hasSessionModelOverride).toBeUndefined(); + expect(rechecked.modelOverrideSource).toBeUndefined(); + expect(rechecked.hasAutoFallbackProvenance).toBeUndefined(); + }); + + it("keeps fallback auth available when a primary probe falls back", () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + fallbackAuthProfileId: "google:fallback", + fallbackAuthProfileIdSource: "auto" as const, + }; + const followupRun = createFollowupRun(); + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-sonnet-4-6"; + followupRun.run.authProfileId = "anthropic:primary"; + followupRun.run.authProfileIdSource = "auto"; + followupRun.run.autoFallbackPrimaryProbe = probe; + expect(resolveFallbackCandidateRun(followupRun.run, "google", "gemini-3-pro")).toMatchObject({ + provider: "google", + model: "gemini-3-pro", + authProfileId: "google:fallback", + authProfileIdSource: "auto", + }); + }); + + it("does not clear an auto-fallback pin for an exhausted preserved result", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + fallbackAuthProfileId: "google:fallback", + fallbackAuthProfileIdSource: "auto" as const, + }; + const followupRun = createFollowupRun(); + followupRun.run.provider = probe.provider; + followupRun.run.model = probe.model; + followupRun.run.autoFallbackPrimaryProbe = probe; + const sessionKey = "exhausted-primary-probe"; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: probe.fallbackProvider, + modelOverride: probe.fallbackModel, + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: probe.provider, + modelOverrideFallbackOriginModel: probe.model, + authProfileOverride: probe.fallbackAuthProfileId, + authProfileOverrideSource: "auto", + }; + const activeSessionStore: Record = { [sessionKey]: sessionEntry }; + const exhaustedResult = { + payloads: [{ text: "Terminal tool summary", isError: true }], + meta: { + error: { + kind: "incomplete_turn", + message: "All fallback candidates ended incomplete", + fallbackSafe: true, + terminalPresentation: true, + }, + }, + }; + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "lifecycle", + data: { + phase: "finishing", + error: "All fallback candidates ended incomplete", + livenessState: "blocked", + providerStarted: true, + replayInvalid: true, + timeoutPhase: "provider", + }, + }); + return exhaustedResult; + }); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + outcome: "exhausted", + result: await params.run(probe.provider, probe.model), + provider: probe.provider, + model: probe.model, + attempts: [ + { provider: probe.provider, model: probe.model, error: "incomplete" }, + { + provider: probe.fallbackProvider, + model: probe.fallbackModel, + error: "incomplete", + }, + ], + })); + const { replyOperation, failMock, retainFailureUntilCompleteMock } = createMockReplyOperation(); + const emitAgentEvent = vi.mocked((await import("../../infra/agent-events.js")).emitAgentEvent); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun, replyOperation }), + sessionKey, + activeSessionStore, + getActiveSessionEntry: () => activeSessionStore[sessionKey], + }); + + expect(result).toMatchObject({ + kind: "success", + fallbackExhausted: true, + fallbackProvider: probe.provider, + fallbackModel: probe.model, + runResult: exhaustedResult, + }); + expect(activeSessionStore[sessionKey]).toMatchObject({ + providerOverride: probe.fallbackProvider, + modelOverride: probe.fallbackModel, + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: probe.provider, + modelOverrideFallbackOriginModel: probe.model, + }); + expect(retainFailureUntilCompleteMock).toHaveBeenCalledTimes(1); + expect(failMock).toHaveBeenCalledWith("run_failed", expect.any(Error)); + expect( + emitAgentEvent.mock.calls + .map((call) => call[0]) + .find( + (event) => + event.stream === "lifecycle" && + event.data.phase === "error" && + event.data.fallbackExhaustedFailure === true && + event.data.livenessState === "blocked" && + event.data.providerStarted === true && + event.data.replayInvalid === true && + event.data.timeoutPhase === "provider", + ), + ).toBeDefined(); + }); + + it("reports a completed non-fallbackable error result as a failure terminal", async () => { + const terminalErrorResult = { + payloads: [{ text: "Command may have changed state", isError: true }], + meta: { + replayInvalid: true, + error: { + kind: "incomplete_turn", + message: "raw provider detail should stay private", + fallbackSafe: false, + }, + }, + }; + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "lifecycle", + data: { + phase: "finishing", + error: "Command may have changed state", + replayInvalid: true, + }, + }); + return terminalErrorResult; + }); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + outcome: "completed", + result: await params.run("anthropic", "claude"), + provider: "anthropic", + model: "claude", + attempts: [], + })); + const { replyOperation, failMock, retainFailureUntilCompleteMock } = createMockReplyOperation(); + const emitAgentEvent = vi.mocked((await import("../../infra/agent-events.js")).emitAgentEvent); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + replyOperation, + opts: { runId: "run-non-fallbackable-error" }, + }), + ); + + expect(result).toMatchObject({ kind: "success", runResult: terminalErrorResult }); + expect(retainFailureUntilCompleteMock).toHaveBeenCalledTimes(1); + expect(failMock).toHaveBeenCalledWith("run_failed", expect.any(Error)); + const lifecycleEvents = emitAgentEvent.mock.calls + .map((call) => call[0]) + .filter( + (event) => event.runId === "run-non-fallbackable-error" && event.stream === "lifecycle", + ); + expect(lifecycleEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + data: expect.objectContaining({ + phase: "error", + error: "Command may have changed state", + replayInvalid: true, + }), + }), + ]), + ); + expect( + lifecycleEvents.some( + (event) => event.data.phase === "end" || event.data.fallbackExhaustedFailure === true, + ), + ).toBe(false); + expect(JSON.stringify(lifecycleEvents)).not.toContain("raw provider detail"); + }); + + it.each([ + { + label: "exhausted", + outcome: "exhausted" as const, + attempts: [{ error: "missing tool result" }], + isHeartbeat: false, + expectedText: GENERIC_RUN_FAILURE_TEXT, + }, + { + label: "completed", + outcome: "completed" as const, + attempts: [], + isHeartbeat: false, + expectedText: GENERIC_RUN_FAILURE_TEXT, + }, + { + label: "heartbeat", + outcome: "completed" as const, + attempts: [], + isHeartbeat: true, + expectedText: HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT, + }, + ])("surfaces an empty $label terminal result through the normal reply path", async (testCase) => { + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [], + meta: { + error: { + kind: "tool_result_mismatch", + message: "Agent run reached a terminal error before reply delivery.", + }, + }, + }); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + outcome: testCase.outcome, + result: await params.run("anthropic", "claude"), + provider: "anthropic", + model: "claude", + attempts: testCase.attempts, + })); + const { replyOperation, failMock } = createMockReplyOperation(); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ replyOperation }), + isHeartbeat: testCase.isHeartbeat, + }); + + expect(result).toMatchObject({ + kind: "success", + terminalFailurePayload: { + text: testCase.expectedText, + isError: true, + }, + runResult: { + payloads: [], + }, + }); + expect(failMock).toHaveBeenCalledWith("run_failed", expect.any(Error)); + }); + + it("reports exhausted CLI results without a success lifecycle terminal", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + outcome: "exhausted", + result: await params.run("codex-cli", "gpt-5.4"), + provider: "codex-cli", + model: "gpt-5.4", + attempts: [{ provider: "codex-cli", model: "gpt-5.4", error: "incomplete" }], + })); + state.runCliAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "Terminal tool summary", isError: true }], + meta: { + error: { + kind: "incomplete_turn", + message: "CLI turn ended incomplete", + }, + }, + }); + const followupRun = createFollowupRun(); + followupRun.run.provider = "codex-cli"; + followupRun.run.model = "gpt-5.4"; + const { replyOperation, failMock, retainFailureUntilCompleteMock } = createMockReplyOperation(); + const emitAgentEvent = vi.mocked((await import("../../infra/agent-events.js")).emitAgentEvent); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + followupRun, + replyOperation, + opts: { runId: "run-cli-exhausted" }, + }), + ); + + expect(result).toMatchObject({ + kind: "success", + fallbackExhausted: true, + fallbackProvider: "codex-cli", + fallbackModel: "gpt-5.4", + }); + expect(retainFailureUntilCompleteMock).toHaveBeenCalledTimes(1); + expect(failMock).toHaveBeenCalledWith("run_failed", expect.any(Error)); + const lifecycleEvents = emitAgentEvent.mock.calls + .map((call) => call[0]) + .filter((event) => event.runId === "run-cli-exhausted" && event.stream === "lifecycle"); + expect(lifecycleEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + data: expect.objectContaining({ + phase: "error", + fallbackExhaustedFailure: true, + }), + }), + ]), + ); + expect(lifecycleEvents.some((event) => event.data.phase === "end")).toBe(false); + }); + + it("preserves a CLI watchdog timeout through the lifecycle backstop", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + try { + return await params.run("codex-cli", "gpt-5.4"); + } catch (cause) { + throw new Error("All model fallback candidates failed", { cause }); + } + }); + state.runCliAgentMock.mockRejectedValueOnce( + new FailoverError("CLI produced no output", { reason: "timeout" }), + ); + const followupRun = createFollowupRun(); + followupRun.run.provider = "codex-cli"; + followupRun.run.model = "gpt-5.4"; + const emitAgentEvent = vi.mocked((await import("../../infra/agent-events.js")).emitAgentEvent); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + followupRun, + opts: { runId: "run-cli-timeout" }, + }), + ); + + expect( + emitAgentEvent.mock.calls + .map((call) => call[0]) + .find( + (event) => + event.runId === "run-cli-timeout" && + event.stream === "lifecycle" && + event.data.phase === "error", + )?.data, + ).toMatchObject({ + stopReason: "timeout", + timeoutPhase: "provider", + fallbackExhaustedFailure: true, + }); + }); + + it("keeps primary auth on same-provider primary probes", async () => { + const probe = { + provider: "openai", + model: "gpt-5.5", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + fallbackAuthProfileId: "openai:fallback", + fallbackAuthProfileIdSource: "auto" as const, + }; + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-5.5"; + followupRun.run.authProfileId = "openai:primary"; + followupRun.run.authProfileIdSource = "auto"; + followupRun.run.autoFallbackPrimaryProbe = probe; + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + await params.run("openai", "gpt-5.5"); + return { + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [{ provider: "openai", model: "gpt-5.5", error: "rate limit" }], + }; + }); + state.runEmbeddedAgentMock + .mockResolvedValueOnce({ payloads: [], meta: {} }) + .mockResolvedValueOnce({ payloads: [{ text: "fallback" }], meta: {} }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback(createMinimalRunAgentTurnParams({ followupRun })); + + expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "primary run", { + provider: "openai", + model: "gpt-5.5", + authProfileId: "openai:primary", + authProfileIdSource: "auto", + }); + expectMockCallArgFields(state.runEmbeddedAgentMock, 1, "fallback run", { + provider: "openai", + model: "gpt-5.4", + authProfileId: "openai:fallback", + authProfileIdSource: "auto", + }); + }); + + it("does not clear a concurrent user selection after primary probe success", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + }; + const sessionKey = "concurrent-user-switch-during-probe"; + const staleAutoEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + }; + const activeSessionStore = { [sessionKey]: staleAutoEntry }; + const followupRun = createFollowupRun(); + followupRun.run.sessionKey = sessionKey; + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-sonnet-4-6"; + followupRun.run.autoFallbackPrimaryProbe = probe; + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + const result = await params.run(params.provider, params.model); + activeSessionStore[sessionKey] = { + sessionId: "session", + updatedAt: 2, + providerOverride: "openai", + modelOverride: "gpt-5.4", + modelOverrideSource: "user", + }; + return { + result, + provider: params.provider, + model: params.model, + attempts: [], + }; + }); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "primary recovered" }], + meta: { + agentMeta: { + provider: "anthropic", + model: "claude-sonnet-4-6", + }, + }, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + sessionKey, + activeSessionStore, + getActiveSessionEntry: () => staleAutoEntry, + }); + + expectRecordFields(activeSessionStore[sessionKey] as unknown as Record, { + providerOverride: "openai", + modelOverride: "gpt-5.4", + modelOverrideSource: "user", + }); + }); + + it("keeps rechecked primary probe runs in sync after live model switches", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "openai", + fallbackModel: "gpt-5.5", + fallbackAuthProfileId: "openai:fallback", + fallbackAuthProfileIdSource: "auto" as const, + }; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "openai", + modelOverride: "gpt-5.5", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + }; + const sessionKey = "live-switch-probe"; + const activeSessionStore = { [sessionKey]: sessionEntry }; + const followupRun = createFollowupRun(); + followupRun.run.sessionKey = sessionKey; + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-sonnet-4-6"; + followupRun.run.autoFallbackPrimaryProbe = probe; + const attemptedProviders: Array = []; + state.runWithModelFallbackMock.mockImplementation(async (params: FallbackRunnerParams) => { + attemptedProviders.push(params.provider); + const provider = params.provider ?? "anthropic"; + const model = params.model ?? "claude-sonnet-4-6"; + return { + result: await params.run(provider, model), + provider, + model, + attempts: [], + }; + }); + state.runEmbeddedAgentMock + .mockImplementationOnce(async () => { + throw new LiveSessionModelSwitchError({ + provider: "openai", + model: "gpt-5.4", + authProfileId: "openai:primary", + authProfileIdSource: "auto", + }); + }) + .mockResolvedValueOnce({ + payloads: [{ text: "switched" }], + meta: { + agentMeta: { + provider: "openai", + model: "gpt-5.4", + }, + }, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + sessionKey, + activeSessionStore, + getActiveSessionEntry: () => activeSessionStore[sessionKey], + }); + + expect(result.kind).toBe("success"); + expect(attemptedProviders).toEqual(["anthropic", "openai"]); + expectMockCallArgFields(state.runEmbeddedAgentMock, 1, "embedded run", { + provider: "openai", + model: "gpt-5.4", + authProfileId: "openai:primary", + authProfileIdSource: "auto", + }); + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-progress.test.ts b/src/auto-reply/reply/agent-runner-execution-progress.test.ts new file mode 100644 index 000000000000..42a454071adb --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-progress.test.ts @@ -0,0 +1,769 @@ +import { describe, expect, it, vi } from "vitest"; +import type { TemplateContext } from "../templating.js"; +import type { GetReplyOptions } from "../types.js"; +import { + setupAgentRunnerExecutionTestState, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, + requireRecord, + expectRecordFields, + expectNoMockCallWithFields, + requireMockCallArgWithFields, + createMinimalRunAgentTurnParams, +} from "./agent-runner-execution.test-support.js"; +import type { + FallbackRunnerParams, + EmbeddedAgentParams, +} from "./agent-runner-execution.test-support.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: lifecycle progress", () => { + it("forwards item lifecycle events to reply options", async () => { + const onItemEvent = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "item", + data: { + itemId: "tool:read-1", + toolCallId: "read-1", + kind: "tool", + title: "read", + name: "read", + phase: "start", + status: "running", + }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const pendingToolTasks = new Set>(); + const typingSignals = createMockTypingSignaler(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { + onItemEvent, + } satisfies GetReplyOptions, + typingSignals, + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks, + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + await Promise.all(pendingToolTasks); + + expect(result.kind).toBe("success"); + expect(onItemEvent).toHaveBeenCalledWith({ + itemId: "tool:read-1", + toolCallId: "read-1", + kind: "tool", + title: "read", + name: "read", + phase: "start", + status: "running", + }); + }); + + it("skips channel item progress when a matching tool event carries the progress", async () => { + const onItemEvent = vi.fn(); + const onToolStart = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "item", + data: { + itemId: "cmd-1", + toolCallId: "cmd-1", + kind: "command", + title: "Command", + name: "bash", + phase: "start", + status: "running", + suppressChannelProgress: true, + }, + }); + await params.onAgentEvent?.({ + stream: "tool", + data: { + itemId: "cmd-1", + toolCallId: "cmd-1", + name: "bash", + phase: "start", + args: { command: "pnpm test" }, + }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ + opts: { + onItemEvent, + onToolStart, + } satisfies GetReplyOptions, + }), + }); + + expect(result.kind).toBe("success"); + expect(onItemEvent).not.toHaveBeenCalled(); + expect(onToolStart).toHaveBeenCalledWith({ + itemId: "cmd-1", + toolCallId: "cmd-1", + name: "bash", + phase: "start", + args: { command: "pnpm test" }, + detailMode: undefined, + }); + }); + + it("preserves suppressed item progress when no tool-start callback is registered", async () => { + const onItemEvent = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "item", + data: { + itemId: "cmd-1", + toolCallId: "cmd-1", + kind: "command", + title: "Command", + name: "bash", + phase: "start", + status: "running", + suppressChannelProgress: true, + }, + }); + await params.onAgentEvent?.({ + stream: "tool", + data: { + itemId: "cmd-1", + toolCallId: "cmd-1", + name: "bash", + phase: "start", + args: { command: "pnpm test" }, + }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ + opts: { + onItemEvent, + } satisfies GetReplyOptions, + }), + }); + + expect(result.kind).toBe("success"); + expect(onItemEvent).toHaveBeenCalledWith({ + itemId: "cmd-1", + toolCallId: "cmd-1", + kind: "command", + title: "Command", + name: "bash", + phase: "start", + status: "running", + }); + }); + + it("hides internal lifecycle events while preserving visible tool progress", async () => { + const onItemEvent = vi.fn(); + const onToolStart = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "tool", + data: { + name: "exec", + phase: "start", + args: { command: "pwd" }, + }, + }); + await params.onAgentEvent?.({ + stream: "item", + data: { + itemId: "tool:exec-1", + kind: "tool", + title: "exec pwd", + name: "exec", + phase: "start", + status: "running", + }, + }); + await params.onAgentEvent?.({ + stream: "tool", + data: { + name: "wait", + phase: "start", + args: { runId: "ordinary_wait" }, + }, + }); + await params.onAgentEvent?.({ + stream: "tool", + data: { + name: "wait", + phase: "start", + args: { runId: "cm_1" }, + hideFromChannelProgress: true, + }, + }); + await params.onAgentEvent?.({ + stream: "item", + data: { + itemId: "tool:wait-1", + kind: "tool", + title: "wait", + name: "wait", + phase: "start", + status: "running", + hideFromChannelProgress: true, + }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ + opts: { onItemEvent, onToolStart } satisfies GetReplyOptions, + }), + }); + + expect(result.kind).toBe("success"); + expect(onToolStart).toHaveBeenCalledTimes(2); + expect(onToolStart).toHaveBeenCalledWith( + expect.objectContaining({ name: "exec", phase: "start" }), + ); + expect(onToolStart).toHaveBeenCalledWith( + expect.objectContaining({ name: "wait", phase: "start" }), + ); + expect(onItemEvent).toHaveBeenCalledTimes(1); + expect(onItemEvent).toHaveBeenCalledWith( + expect.objectContaining({ name: "exec", phase: "start" }), + ); + }); + + it("forwards raw tool progress detail mode to tool-start reply options", async () => { + const onToolStart = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "tool", + data: { + name: "exec", + phase: "start", + args: { command: "pnpm test -- --watch=false" }, + }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ + opts: { + onToolStart, + } satisfies GetReplyOptions, + }), + toolProgressDetail: "raw", + }); + + expect(result.kind).toBe("success"); + expect(onToolStart).toHaveBeenCalledWith({ + itemId: undefined, + toolCallId: undefined, + name: "exec", + phase: "start", + args: { command: "pnpm test -- --watch=false" }, + detailMode: "raw", + }); + }); + + it("fires tool-start progress before slow typing signals resolve for best-effort agent events", async () => { + const onToolStart = vi.fn(async () => {}); + let releaseTyping: (() => void) | undefined; + const typingSignals = createMockTypingSignaler(); + vi.mocked(typingSignals.signalToolStart).mockImplementation( + () => + new Promise((resolve) => { + releaseTyping = resolve; + }), + ); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + void params.onAgentEvent?.({ + stream: "tool", + data: { + name: "exec", + phase: "start", + args: { command: "echo hi" }, + }, + }); + await Promise.resolve(); + await Promise.resolve(); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ + opts: { + onToolStart, + } satisfies GetReplyOptions, + }), + typingSignals, + }); + + try { + expect(result.kind).toBe("success"); + expect(onToolStart).toHaveBeenCalledWith({ + itemId: undefined, + toolCallId: undefined, + name: "exec", + phase: "start", + args: { command: "echo hi" }, + detailMode: undefined, + }); + } finally { + releaseTyping?.(); + await Promise.resolve(); + } + }); + + it("starts presentation callbacks in source order while typing signals are slow", async () => { + const callbackOrder: string[] = []; + let releaseTyping: (() => void) | undefined; + const typingPending = new Promise((resolve) => { + releaseTyping = resolve; + }); + const typingSignals = createMockTypingSignaler(); + vi.mocked(typingSignals.signalMessageStart).mockReturnValue(typingPending); + vi.mocked(typingSignals.signalTextDelta).mockReturnValue(typingPending); + vi.mocked(typingSignals.signalReasoningDelta).mockReturnValue(typingPending); + vi.mocked(typingSignals.signalToolStart).mockReturnValue(typingPending); + + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + void params.onAssistantMessageStart?.(); + void params.onPartialReply?.({ text: "answer before tool" }); + void params.onReasoningStream?.({ text: "reasoning before tool" }); + void params.onReasoningEnd?.(); + void params.onAgentEvent?.({ + stream: "tool", + data: { + name: "exec", + phase: "start", + args: { command: "echo hi" }, + }, + }); + await Promise.resolve(); + await Promise.resolve(); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ + opts: { + preserveProgressCallbackStartOrder: true, + onAssistantMessageStart: () => { + callbackOrder.push("message-start"); + }, + onPartialReply: () => { + callbackOrder.push("partial"); + }, + onReasoningStream: () => { + callbackOrder.push("reasoning"); + }, + onReasoningEnd: () => { + callbackOrder.push("reasoning-end"); + }, + onToolStart: () => { + callbackOrder.push("tool"); + }, + } satisfies GetReplyOptions, + }), + typingSignals, + }); + + try { + expect(result.kind).toBe("success"); + expect(callbackOrder).toEqual([ + "message-start", + "partial", + "reasoning", + "reasoning-end", + "tool", + ]); + } finally { + releaseTyping?.(); + await Promise.resolve(); + } + }); + + it("starts typing when a presentation callback throws inline", async () => { + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await expect(params.onPartialReply?.({ text: "before failure" })).rejects.toThrow( + "presentation failed", + ); + return { payloads: [{ text: "final" }], meta: {} }; + }); + const typingSignals = createMockTypingSignaler(); + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ + opts: { + preserveProgressCallbackStartOrder: true, + onPartialReply: () => { + throw new Error("presentation failed"); + }, + }, + }), + typingSignals, + }); + + expect(result.kind).toBe("success"); + expect(typingSignals.signalTextDelta).toHaveBeenCalledWith("before failure"); + }); + + it("leaves Codex app-server telemetry publication to the harness", async () => { + const agentEvents = await import("../../infra/agent-events.js"); + const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "codex_app_server.guardian", + sessionKey: "agent:main:subagent:codex-child", + data: { + phase: "blocked", + message: "command requires approval", + }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { runId: "run-codex" } as GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + expectNoMockCallWithFields(emitAgentEvent, { + runId: "run-codex", + stream: "codex_app_server.guardian", + }); + }); + + it("emits an embedded lifecycle terminal backstop when the runner returns without one", async () => { + const agentEvents = await import("../../infra/agent-events.js"); + const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "lifecycle", + data: { phase: "start", startedAt: 1_000 }, + }); + return { + payloads: [{ text: "Request timed out before a response was generated.", isError: true }], + meta: { aborted: true, livenessState: "blocked", replayInvalid: true }, + }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { runId: "run-timeout" } as GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + const lifecycleEvent = requireRecord( + requireMockCallArgWithFields( + emitAgentEvent, + { runId: "run-timeout", sessionKey: "main", stream: "lifecycle" }, + "agent event", + ), + "agent event", + ); + expectRecordFields(lifecycleEvent, { + runId: "run-timeout", + sessionKey: "main", + stream: "lifecycle", + }); + const lifecycleData = requireRecord(lifecycleEvent.data, "lifecycle data"); + expectRecordFields(lifecycleData, { + phase: "end", + startedAt: 1_000, + aborted: true, + livenessState: "blocked", + replayInvalid: true, + }); + expect(typeof lifecycleData.endedAt).toBe("number"); + }); + + it("uses a rebound lifecycle generation for embedded terminal events", async () => { + const agentEvents = await import("../../infra/agent-events.js"); + const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + params.onExecutionStarted?.({ lifecycleGeneration: "post-restart" }); + await params.onAgentEvent?.({ + stream: "lifecycle", + data: { phase: "start", startedAt: 1_000 }, + }); + throw new Error("rebound failure"); + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { runId: "run-rebound" } as GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + const lifecycleEvents = emitAgentEvent.mock.calls + .map((call) => call[0]) + .filter( + (event) => + event.runId === "run-rebound" && + event.stream === "lifecycle" && + (event.data.phase === "error" || event.data.fallbackExhaustedFailure === true), + ); + expect(lifecycleEvents.length).toBeGreaterThan(0); + expect(lifecycleEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + lifecycleGeneration: "post-restart", + }), + ]), + ); + expect(lifecycleEvents.every((event) => event.lifecycleGeneration === "post-restart")).toBe( + true, + ); + }); + + it("does not duplicate embedded lifecycle terminal events already reported by the runner", async () => { + const agentEvents = await import("../../infra/agent-events.js"); + const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onAgentEvent?.({ + stream: "lifecycle", + data: { phase: "start", startedAt: 1_000 }, + }); + await params.onAgentEvent?.({ + stream: "lifecycle", + data: { phase: "end", endedAt: 1_500 }, + }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { runId: "run-complete" } as GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + expectNoMockCallWithFields(emitAgentEvent, { + runId: "run-complete", + stream: "lifecycle", + }); + }); + + it("preserves GPT ack-turn final prose without reply-side truncation", async () => { + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + })); + state.runEmbeddedAgentMock.mockImplementationOnce(async () => ({ + payloads: [ + { + text: [ + "I updated the prompt overlay and tightened the runtime guard.", + "I also added the ack-turn fast path so short approvals skip the recap.", + "The reply-side output now keeps long prose-heavy GPT confirmations intact.", + "I updated tests for the overlay, retry guard, and reply normalization.", + "Everything is wired together and ready for verification.", + ].join(" "), + }, + ], + meta: {}, + })); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-5.4"; + const result = await runAgentTurnWithFallback({ + commandBody: "ok do it", + followupRun, + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + if (result.kind === "success") { + expect(result.runResult.payloads?.[0]?.text).toBe( + [ + "I updated the prompt overlay and tightened the runtime guard.", + "I also added the ack-turn fast path so short approvals skip the recap.", + "The reply-side output now keeps long prose-heavy GPT confirmations intact.", + "I updated tests for the overlay, retry guard, and reply normalization.", + "Everything is wired together and ready for verification.", + ].join(" "), + ); + } + }); + + it("does not trim GPT replies when the user asked for depth", async () => { + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + })); + const longDetailedReply = [ + "Here is the detailed breakdown.", + "First, the runner now detects short approval turns and skips the recap path.", + "Second, the reply layer scores long prose-heavy GPT confirmations and trims them only in chat-style turns.", + "Third, code fences and richer structured outputs are left untouched so technical answers stay intact.", + "Finally, the overlay reinforces that this is a live chat and nudges the model toward short natural replies.", + ].join(" "); + state.runEmbeddedAgentMock.mockImplementationOnce(async () => ({ + payloads: [{ text: longDetailedReply }], + meta: {}, + })); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-5.4"; + const result = await runAgentTurnWithFallback({ + commandBody: "explain in detail what changed", + followupRun, + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + if (result.kind === "success") { + expect(result.runResult.payloads?.[0]?.text).toBe(longDetailedReply); + } + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-provider-failures.test.ts b/src/auto-reply/reply/agent-runner-execution-provider-failures.test.ts new file mode 100644 index 000000000000..21d0cecdf31a --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-provider-failures.test.ts @@ -0,0 +1,579 @@ +import { describe, expect, it } from "vitest"; +import { formatBillingErrorMessage } from "../../agents/embedded-agent-helpers.js"; +import { FailoverError } from "../../agents/failover-error.js"; +import type { TemplateContext } from "../templating.js"; +import { SILENT_REPLY_TOKEN } from "../tokens.js"; +import { + PROVIDER_AUTHENTICATION_ERROR_USER_MESSAGE, + PROVIDER_RATE_LIMIT_OR_QUOTA_ERROR_USER_MESSAGE, + PROVIDER_INTERNAL_ERROR_USER_MESSAGE, + setupAgentRunnerExecutionTestState, + GENERIC_RUN_FAILURE_TEXT, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, + createMinimalRunAgentTurnParams, + NON_DIRECT_FAILURE_SURFACE_CASES, + createNonDirectFailureSessionCtx, +} from "./agent-runner-execution.test-support.js"; +import { buildKnownAgentRunFailureReplyPayload } from "./agent-runner-failure-reply.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: provider failures", () => { + it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( + "keeps raw runner failure boilerplate out of $label chats", + async (testCase) => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error("openai/gpt-5.5 ended with an incomplete terminal response"), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: createNonDirectFailureSessionCtx(testCase), + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe(SILENT_REPLY_TOKEN); + } + }, + ); + + it.each(["group", "channel"] as const)( + "surfaces raw runner failure copy in Discord %s chats when silentReply.group is set to disallow", + async (chatType) => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error("openai/gpt-5.5 ended with an incomplete terminal response"), + ); + + const followupRun = createFollowupRun(); + followupRun.run.config = { + agents: { + defaults: { + silentReply: { group: "disallow" }, + }, + }, + }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + followupRun, + sessionCtx: { + Provider: "discord", + Surface: "discord", + ChatType: chatType, + GroupSubject: "agent group", + GroupChannel: "#general", + MessageSid: "msg", + } as unknown as TemplateContext, + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); + expect(result.payload.text).toBe(GENERIC_RUN_FAILURE_TEXT); + } + }, + ); + + it("surfaces raw runner failure copy when per-surface silentReply.group is set to disallow", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error("openai/gpt-5.5 ended with an incomplete terminal response"), + ); + + const followupRun = createFollowupRun(); + followupRun.run.config = { + agents: { + defaults: { + silentReply: { group: "allow" }, + }, + }, + surfaces: { + discord: { + silentReply: { group: "disallow" }, + }, + }, + }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + followupRun, + sessionCtx: { + Provider: "discord", + Surface: "discord", + ChatType: "group", + GroupSubject: "agent group", + GroupChannel: "#general", + MessageSid: "msg", + } as unknown as TemplateContext, + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe(GENERIC_RUN_FAILURE_TEXT); + } + }); + + it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( + "keeps default silent behavior in $label chats when silentReply policy is unset", + async (testCase) => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error("openai/gpt-5.5 ended with an incomplete terminal response"), + ); + + const followupRun = createFollowupRun(); + followupRun.run.config = {}; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + followupRun, + sessionCtx: createNonDirectFailureSessionCtx(testCase), + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe(SILENT_REPLY_TOKEN); + } + }, + ); + + it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( + "keeps classified non-transient failures visible in $label chats", + async (testCase) => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error('No API key found for provider "openai"'), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: createNonDirectFailureSessionCtx(testCase), + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); + expect(result.payload.text).toContain('Missing API key for provider "openai"'); + } + }, + ); + + it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( + "surfaces provider authentication failures in $label chats", + async (testCase) => { + const rawError = + "unexpected status 401 Unauthorized: Missing bearer or basic authentication in header, url: https://api.openai.com/v1/responses"; + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new FailoverError("LLM request unauthorized.", { + reason: "auth", + provider: "openai", + model: "gpt-5.5", + status: 401, + rawError, + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: createNonDirectFailureSessionCtx(testCase), + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.isError).toBe(true); + expect(result.payload.text).toBe(PROVIDER_AUTHENTICATION_ERROR_USER_MESSAGE); + expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); + expect(result.payload.text).not.toContain(rawError); + } + }, + ); + + it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( + "surfaces rate-limit fallback copy in $label chats", + async (testCase) => { + state.runEmbeddedAgentMock.mockRejectedValueOnce(new Error("429 rate limit exceeded")); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: createNonDirectFailureSessionCtx(testCase), + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.isError).toBe(true); + expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); + expect(result.payload.text).toContain("rate-limited"); + } + }, + ); + + it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( + "surfaces typed periodic rate-limit details in $label chats", + async (testCase) => { + const periodicLimitMessage = "You've hit your weekly limit · resets 6pm (UTC)"; + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new FailoverError(periodicLimitMessage, { + reason: "rate_limit", + provider: "anthropic", + model: "claude-opus-4-1", + rawError: periodicLimitMessage, + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: createNonDirectFailureSessionCtx(testCase), + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.isError).toBe(true); + expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); + expect(result.payload.text).toContain("weekly limit"); + expect(result.payload.text).toContain("resets 6pm"); + expect(result.payload.text).not.toContain("few minutes"); + } + }, + ); + + it("surfaces typed periodic rate-limit details through known failure payloads in group chats", () => { + const periodicLimitMessage = "You've hit your weekly limit · resets 6pm (UTC)"; + const payload = buildKnownAgentRunFailureReplyPayload({ + err: new FailoverError(periodicLimitMessage, { + reason: "rate_limit", + provider: "anthropic", + model: "claude-opus-4-1", + rawError: periodicLimitMessage, + }), + sessionCtx: createNonDirectFailureSessionCtx(NON_DIRECT_FAILURE_SURFACE_CASES[0]), + resolvedVerboseLevel: "off", + }); + + expect(payload).toBeDefined(); + expect(payload?.isError).toBe(true); + expect(payload?.text).not.toBe(SILENT_REPLY_TOKEN); + expect(payload?.text).toContain("weekly limit"); + expect(payload?.text).toContain("resets 6pm"); + expect(payload?.text).not.toContain("few minutes"); + }); + + it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( + "surfaces overloaded fallback copy in $label chats", + async (testCase) => { + state.runEmbeddedAgentMock.mockRejectedValueOnce(new Error("model is overloaded")); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: createNonDirectFailureSessionCtx(testCase), + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.isError).toBe(true); + expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); + expect(result.payload.text).toContain("overloaded"); + } + }, + ); + + it("surfaces typed overloaded failures without rate-limit cooldown copy", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new FailoverError("529 Please try again", { + reason: "overloaded", + provider: "anthropic", + model: "claude-opus-4-1", + status: 529, + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: createNonDirectFailureSessionCtx(NON_DIRECT_FAILURE_SURFACE_CASES[0]), + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.isError).toBe(true); + expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); + expect(result.payload.text).toContain("overloaded"); + expect(result.payload.text).not.toContain("rate-limited"); + expect(result.payload.text).not.toContain("few minutes"); + } + }); + + it("surfaces rate-limit fallback copy in Discord group chats when silentReply.group is disallow", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce(new Error("429 rate limit exceeded")); + + const followupRun = createFollowupRun(); + followupRun.run.config = { + agents: { + defaults: { + silentReply: { group: "disallow" }, + }, + }, + }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + followupRun, + sessionCtx: { + Provider: "discord", + Surface: "discord", + ChatType: "group", + GroupSubject: "agent group", + GroupChannel: "#general", + MessageSid: "msg", + } as unknown as TemplateContext, + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.isError).toBe(true); + expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); + expect(result.payload.text).toContain("rate-limited"); + } + }); + + it("uses compact generic copy for raw runner failures in normal Discord direct chats", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error("openai/gpt-5.5 ended with an incomplete terminal response"), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: { + Provider: "discord", + Surface: "discord", + ChatType: "direct", + MessageSid: "msg", + } as unknown as TemplateContext, + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe(GENERIC_RUN_FAILURE_TEXT); + } + }); + + it("keeps raw runner failure guidance visible in verbose Discord direct chats", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error("openai/gpt-5.5 ended with an incomplete terminal response"), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ + sessionCtx: { + Provider: "discord", + Surface: "discord", + ChatType: "direct", + MessageSid: "msg", + } as unknown as TemplateContext, + }), + resolvedVerboseLevel: "on", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toContain("Agent failed before reply"); + expect(result.payload.text).toContain("incomplete terminal response"); + } + }); + + it("surfaces provider quota guidance for generic HTTP 429 failures before reply", async () => { + const error = new Error( + "Something went wrong while processing your request. Please try again.", + ); + Object.assign(error, { status: 429 }); + state.runEmbeddedAgentMock.mockRejectedValueOnce(error); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: { + Provider: "discord", + Surface: "discord", + ChatType: "direct", + MessageSid: "msg", + } as unknown as TemplateContext, + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe(PROVIDER_RATE_LIMIT_OR_QUOTA_ERROR_USER_MESSAGE); + expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); + } + }); + + it("surfaces provider internal errors without session reset guidance before reply", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new FailoverError( + "The AI service returned an internal error. Please try again in a moment.", + { + reason: "server_error", + provider: "fyapis", + model: "gpt-5.5", + status: 500, + }, + ), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: { + Provider: "telegram", + Surface: "telegram", + ChatType: "direct", + MessageSid: "msg", + } as unknown as TemplateContext, + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe(PROVIDER_INTERNAL_ERROR_USER_MESSAGE); + expect(result.payload.text).not.toContain("/new"); + expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); + } + }); + + it("surfaces billing guidance for Volcengine Coding Plan subscription failures before reply", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error( + 'HTTP 400 Bad Request: {"error":{"code":"InvalidSubscription","message":"Your account does not have a valid CodingPlan subscription, or your subscription has expired."}}', + ), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: { + Provider: "discord", + Surface: "discord", + ChatType: "direct", + MessageSid: "msg", + } as unknown as TemplateContext, + }), + ); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe("billing"); + expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); + } + }); + + it("preserves neutral billing guidance for OAuth failover errors", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new FailoverError(formatBillingErrorMessage("Anthropic", "claude-sonnet-4-5", "oauth"), { + reason: "billing", + provider: "Anthropic", + model: "claude-sonnet-4-5", + authMode: "oauth", + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toContain("check your account for subscription or usage limits"); + expect(result.payload.text).not.toContain("API key"); + expect(result.payload.text).not.toContain("top up"); + } + }); + + it("preserves neutral billing guidance after fallback exhaustion", async () => { + state.runWithModelFallbackMock.mockRejectedValueOnce( + Object.assign(new Error("All models failed (1): openai/gpt-5.5: billing"), { + name: "FallbackSummaryError", + attempts: [ + { + provider: "openai", + model: "gpt-5.5", + error: "billing", + reason: "billing", + authMode: "oauth", + }, + ], + soonestCooldownExpiry: null, + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toContain("check your account for subscription or usage limits"); + expect(result.payload.text).not.toContain("API key"); + expect(result.payload.text).not.toContain("top up"); + } + }); + + it("formats raw Codex API payloads before forwarding verbose external errors", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error( + 'Codex error: {"type":"error","error":{"type":"server_error","message":"Something exploded"},"sequence_number":2}', + ), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "on", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + "⚠️ Agent failed before reply: LLM error server_error: Something exploded. Please try again, or use /new to start a fresh session.", + ); + } + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-results.test.ts b/src/auto-reply/reply/agent-runner-execution-results.test.ts new file mode 100644 index 000000000000..be1f2611fd57 --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-results.test.ts @@ -0,0 +1,525 @@ +import { describe, expect, it, vi } from "vitest"; +import type { SessionEntry } from "../../config/sessions.js"; +import type { TemplateContext } from "../templating.js"; +import type { GetReplyOptions } from "../types.js"; +import { + setupAgentRunnerExecutionTestState, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, + requireRecord, + expectRecordFields, + requireMockCall, + expectMockCallArgFields, + createMinimalRunAgentTurnParams, + NON_DIRECT_FAILURE_SURFACE_CASES, + createNonDirectFailureSessionCtx, +} from "./agent-runner-execution.test-support.js"; +import type { + FallbackRunnerParams, + EmbeddedAgentParams, +} from "./agent-runner-execution.test-support.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: result and tool delivery", () => { + it("forwards media-only tool results without typing text", async () => { + const onToolResult = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onToolResult?.({ mediaUrls: ["/tmp/generated.png"] }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const pendingToolTasks = new Set>(); + const typingSignals = createMockTypingSignaler(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { + onToolResult, + } satisfies GetReplyOptions, + typingSignals, + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks, + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + await Promise.all(pendingToolTasks); + + expect(result.kind).toBe("success"); + expect(typingSignals.signalTextDelta).not.toHaveBeenCalled(); + expect(onToolResult).toHaveBeenCalledTimes(1); + expectMockCallArgFields(onToolResult, 0, "tool result payload", { + mediaUrls: ["/tmp/generated.png"], + }); + expect( + requireRecord( + requireMockCall(onToolResult, 0, "tool result payload")[0], + "tool result payload", + ).text, + ).toBeUndefined(); + }); + + it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( + "surfaces model capacity errors from no-text mid-turn failures in $label chats", + async (testCase) => { + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "thinking", isReasoning: true }], + meta: { + error: { + kind: "server_overloaded", + message: "Selected model is at capacity. Please try a different model.", + }, + }, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + sessionCtx: createNonDirectFailureSessionCtx(testCase), + }), + ); + + expect(result.kind).toBe("success"); + if (result.kind === "success") { + expect(result.runResult.payloads).toEqual([ + { + text: "⚠️ Selected model is at capacity. Try a different model, or wait and retry.", + isError: true, + }, + ]); + } + }, + ); + + it("surfaces model capacity errors from pre-reply CLI failures", async () => { + state.runWithModelFallbackMock.mockRejectedValueOnce( + new Error("Selected model is at capacity. Please try a different model."), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-5.5"; + + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result).toEqual({ + kind: "final", + payload: { + isError: true, + text: "⚠️ Selected model is at capacity. Try a different model, or wait and retry.", + }, + }); + }); + + it("classifies structured harness plan-only terminal results as fallback-eligible", async () => { + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-5.4"; + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [], + meta: { + agentHarnessResultClassification: "planning-only", + }, + }); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + const first = (await params.run("openai", "gpt-5.4")) as { + payloads?: Array<{ text?: string; isError?: boolean; isReasoning?: boolean }>; + }; + const classification = await params.classifyResult?.({ + result: first, + provider: "openai", + model: "gpt-5.4", + attempt: 1, + total: 2, + }); + expectRecordFields(requireRecord(classification, "fallback classification"), { + reason: "format", + code: "planning_only_result", + }); + return { + result: { payloads: [{ text: "fallback ok" }], meta: {} }, + provider: "anthropic", + model: "claude", + attempts: [ + { + provider: "openai", + model: "gpt-5.4", + error: "planning-only", + reason: "format", + }, + ], + }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams({ followupRun })); + + expect(result.kind).toBe("success"); + if (result.kind === "success") { + expect(result.runResult.payloads?.[0]?.text).toBe("fallback ok"); + expect(result.fallbackProvider).toBe("anthropic"); + expect(result.fallbackAttempts[0]?.reason).toBe("format"); + } + }); + + it("does not classify silent NO_REPLY terminal results for fallback", async () => { + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + const result = { payloads: [{ text: "NO_REPLY" }], meta: {} }; + expect( + await params.classifyResult?.({ + result, + provider: "openai", + model: "gpt-5.4", + attempt: 1, + total: 2, + }), + ).toBeNull(); + return { + result, + provider: "openai", + model: "gpt-5.4", + attempts: [], + }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("success"); + }); + + it("does not classify empty final payloads after block replies were sent", async () => { + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-5.4"; + state.createBlockReplyDeliveryHandlerMock.mockImplementationOnce( + (params: { directlySentBlockKeys?: Set }) => async () => { + params.directlySentBlockKeys?.add("block:1"); + }, + ); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onBlockReply?.({ text: "streamed block" }); + return { payloads: [], meta: {} }; + }); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + const result = (await params.run("openai", "gpt-5.4")) as { + payloads?: Array<{ text?: string; isError?: boolean; isReasoning?: boolean }>; + }; + expect( + await params.classifyResult?.({ + result, + provider: "openai", + model: "gpt-5.4", + attempt: 1, + total: 2, + }), + ).toBeNull(); + return { + result, + provider: "openai", + model: "gpt-5.4", + attempts: [], + }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback( + createMinimalRunAgentTurnParams({ + followupRun, + opts: { onBlockReply: vi.fn() } satisfies GetReplyOptions, + }), + ); + + expect(result.kind).toBe("success"); + }); + + it("does not classify empty final payloads while block replies are buffered", async () => { + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-5.4"; + const blockReplyPipeline = { + enqueue: vi.fn(), + flush: vi.fn(async () => {}), + stop: vi.fn(), + hasBuffered: vi.fn(() => true), + didStream: vi.fn(() => false), + isAborted: vi.fn(() => false), + hasSentPayload: vi.fn(() => false), + getSentMediaUrls: vi.fn(() => []), + }; + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + const result = { payloads: [], meta: {} }; + expect( + await params.classifyResult?.({ + result, + provider: "openai", + model: "gpt-5.4", + attempt: 1, + total: 2, + }), + ).toBeNull(); + return { + result, + provider: "openai", + model: "gpt-5.4", + attempts: [], + }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + blockReplyPipeline, + blockStreamingEnabled: true, + opts: { onBlockReply: vi.fn() } satisfies GetReplyOptions, + }); + + expect(result.kind).toBe("success"); + }); + + it("classifies final GPT-5 terminal-empty results instead of silently succeeding", async () => { + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + const result = { payloads: [], meta: {} }; + const classification = await params.classifyResult?.({ + result, + provider: "openai", + model: "gpt-5.4", + attempt: 1, + total: 1, + }); + expectRecordFields(requireRecord(classification, "fallback classification"), { + reason: "format", + code: "empty_result", + }); + return { + result, + provider: "openai", + model: "gpt-5.4", + attempts: [], + }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("success"); + }); + + it("keeps fallback candidate selection turn-local during result classification", async () => { + const followupRun = createFollowupRun(); + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude"; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + totalTokens: 1, + compactionCount: 0, + }; + const activeSessionStore = { main: sessionEntry }; + state.runEmbeddedAgentMock.mockResolvedValueOnce({ payloads: [], meta: {} }); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + const failedResult = await params.run("openai", "gpt-5.4"); + expect(sessionEntry.providerOverride).toBeUndefined(); + expect(sessionEntry.modelOverride).toBeUndefined(); + const classification = await params.classifyResult?.({ + result: failedResult as { payloads?: [] }, + provider: "openai", + model: "gpt-5.4", + attempt: 1, + total: 2, + }); + expectRecordFields(requireRecord(classification, "fallback classification"), { + code: "empty_result", + }); + return { + result: { payloads: [{ text: "fallback ok" }], meta: {} }, + provider: "anthropic", + model: "claude", + attempts: [], + }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + activeSessionStore, + getActiveSessionEntry: () => sessionEntry, + }); + + expect(result.kind).toBe("success"); + expect(sessionEntry.providerOverride).toBeUndefined(); + expect(sessionEntry.modelOverride).toBeUndefined(); + }); + + it("strips a glued leading NO_REPLY token from streamed tool results", async () => { + const onToolResult = vi.fn(); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + await params.onToolResult?.({ text: "NO_REPLYThe user is saying hello" }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const pendingToolTasks = new Set>(); + const typingSignals = createMockTypingSignaler(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { + onToolResult, + } satisfies GetReplyOptions, + typingSignals, + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks, + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + await Promise.all(pendingToolTasks); + + expect(result.kind).toBe("success"); + expect(typingSignals.signalTextDelta).toHaveBeenCalledWith("The user is saying hello"); + expect(onToolResult).toHaveBeenCalledWith({ text: "The user is saying hello" }); + }); + + it("continues delivering later streamed tool results after an earlier delivery failure", async () => { + const delivered: string[] = []; + const onToolResult = vi.fn(async (payload: { text?: string }) => { + if (payload.text === "first") { + throw new Error("simulated delivery failure"); + } + delivered.push(payload.text ?? ""); + }); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + void params.onToolResult?.({ text: "first", mediaUrls: [] }); + void params.onToolResult?.({ text: "second", mediaUrls: [] }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const pendingToolTasks = new Set>(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onToolResult } satisfies GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks, + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + await Promise.all(pendingToolTasks); + + expect(result.kind).toBe("success"); + expect(onToolResult).toHaveBeenCalledTimes(2); + expect(delivered).toEqual(["second"]); + }); + + it("delivers streamed tool results in callback order even when dispatch latency differs", async () => { + const deliveryOrder: string[] = []; + const onToolResult = vi.fn(async (payload: { text?: string }) => { + const delay = payload.text === "first" ? 5 : 1; + await new Promise((resolve) => { + setTimeout(resolve, delay); + }); + deliveryOrder.push(payload.text ?? ""); + }); + state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { + void params.onToolResult?.({ text: "first", mediaUrls: [] }); + void params.onToolResult?.({ text: "second", mediaUrls: [] }); + return { payloads: [{ text: "final" }], meta: {} }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const pendingToolTasks = new Set>(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { onToolResult } satisfies GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks, + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + await Promise.all(pendingToolTasks); + + expect(result.kind).toBe("success"); + expect(onToolResult).toHaveBeenCalledTimes(2); + expect(deliveryOrder).toEqual(["first", "second"]); + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-runtime.test.ts b/src/auto-reply/reply/agent-runner-execution-runtime.test.ts new file mode 100644 index 000000000000..9e3bb36b337b --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-runtime.test.ts @@ -0,0 +1,313 @@ +import { describe, expect, it } from "vitest"; +import { testing as cliBackendsTesting } from "../../agents/cli-backends.js"; +import type { SessionEntry } from "../../config/sessions.js"; +import type { TemplateContext } from "../templating.js"; +import { + setupAgentRunnerExecutionTestState, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, + requireRecord, + requireMockCall, + expectMockCallArgFields, + createMinimalRunAgentTurnParams, +} from "./agent-runner-execution.test-support.js"; +import type { FallbackRunnerParams } from "./agent-runner-execution.test-support.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: runtime selection", () => { + it("resolves CLI messageProvider from the live session surface when no origin channel is set", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("codex-cli", "gpt-5.4"), + provider: "codex-cli", + model: "gpt-5.4", + attempts: [], + })); + state.runCliAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "final" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "codex-cli"; + followupRun.run.model = "gpt-5.4"; + followupRun.run.messageProvider = "stale-provider"; + + await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "discord", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { + messageChannel: undefined, + messageProvider: "discord", + }); + }); + + it("does not pass CLI runtime overrides as embedded harness ids for fallback providers", async () => { + cliBackendsTesting.setDepsForTest({ + resolveRuntimeCliBackends: () => [], + resolvePluginSetupCliBackend: ({ backend, config }) => + backend === "claude-cli" && config + ? { + pluginId: "anthropic", + backend: { + id: "claude-cli", + modelProvider: "anthropic", + config: { command: "claude" }, + bundleMcp: false, + }, + } + : undefined, + }); + state.isCliProviderMock.mockImplementation((provider: unknown) => provider === "claude-cli"); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + })); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "fallback" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-opus-4-7"; + followupRun.run.config = { + agents: { + defaults: { + agentRuntime: { id: "claude-cli" }, + }, + }, + }; + + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + getActiveSessionEntry: () => + ({ + sessionId: "session", + updatedAt: Date.now(), + agentRuntimeOverride: "claude-cli", + }) as SessionEntry, + }); + + expect(result.kind).toBe("success"); + expect(state.runCliAgentMock).not.toHaveBeenCalled(); + expect(state.runEmbeddedAgentMock).toHaveBeenCalledOnce(); + expect( + requireRecord( + requireMockCall(state.runEmbeddedAgentMock, 0, "embedded run params")[0], + "embedded run params", + ), + ).not.toHaveProperty("agentHarnessId", "claude-cli"); + }); + + it("passes OpenAI session runtime overrides as embedded harness ids", async () => { + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + })); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "openai" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-5.4"; + + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + getActiveSessionEntry: () => + ({ + sessionId: "session", + updatedAt: Date.now(), + agentRuntimeOverride: "codex", + }) as SessionEntry, + }); + + expect(result.kind).toBe("success"); + expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { + provider: "openai", + model: "gpt-5.4", + agentHarnessId: "codex", + }); + }); + + it("keeps catalog-adopted Codex sessions on Codex during heartbeat model overrides", async () => { + state.isCliProviderMock.mockImplementation((provider: unknown) => provider === "claude-cli"); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("anthropic", "claude-opus-4-6"), + provider: "anthropic", + model: "claude-opus-4-6", + attempts: [], + })); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "heartbeat" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-opus-4-6"; + followupRun.run.config = { + agents: { + defaults: { + models: { + "anthropic/claude-opus-4-6": { agentRuntime: { id: "claude-cli" } }, + }, + }, + }, + }; + + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + isHeartbeat: true, + getActiveSessionEntry: () => + ({ + sessionId: "catalog-adopted-session", + updatedAt: Date.now(), + agentHarnessId: "codex", + modelSelectionLocked: true, + pluginExtensions: { + codex: { + supervision: { + sourceThreadId: "019f-codex-thread", + modelLocked: true, + }, + }, + }, + }) as SessionEntry, + }); + + expect(result.kind).toBe("success"); + expect(state.runCliAgentMock).not.toHaveBeenCalled(); + expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { + provider: "anthropic", + model: "claude-opus-4-6", + trigger: "heartbeat", + agentHarnessId: "codex", + agentHarnessRuntimeOverride: "codex", + }); + }); + + it("keeps a locked Codex harness embedded when cliBackends.codex is configured", async () => { + state.isCliProviderMock.mockImplementation((provider: unknown) => provider === "codex"); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + })); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "continued" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-5.4"; + followupRun.run.config = { + agents: { + defaults: { + cliBackends: { + codex: { command: "codex" }, + }, + }, + }, + }; + + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + getActiveSessionEntry: () => + ({ + sessionId: "catalog-adopted-session", + updatedAt: Date.now(), + agentHarnessId: "codex", + modelSelectionLocked: true, + }) as SessionEntry, + }); + + expect(result.kind).toBe("success"); + expect(state.runCliAgentMock).not.toHaveBeenCalled(); + expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { + provider: "openai", + model: "gpt-5.4", + agentHarnessId: "codex", + agentHarnessRuntimeOverride: "codex", + }); + }); + + it("honors agent session runtime overrides before CLI runtime aliases", async () => { + state.isCliProviderMock.mockImplementation((provider: unknown) => provider === "claude-cli"); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + })); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "agent" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-5.4"; + followupRun.run.config = { + agents: { + defaults: { + agentRuntime: { id: "claude-cli" }, + }, + }, + }; + + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + getActiveSessionEntry: () => + ({ + sessionId: "session", + updatedAt: Date.now(), + agentRuntimeOverride: "codex", + }) as SessionEntry, + }); + + expect(result.kind).toBe("success"); + expect(state.runCliAgentMock).not.toHaveBeenCalled(); + expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { + provider: "openai", + model: "gpt-5.4", + agentHarnessId: "codex", + }); + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-state.test.ts b/src/auto-reply/reply/agent-runner-execution-state.test.ts new file mode 100644 index 000000000000..4c297a69d8e2 --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-state.test.ts @@ -0,0 +1,667 @@ +import { describe, expect, it } from "vitest"; +import { LiveSessionModelSwitchError } from "../../agents/live-model-switch-error.js"; +import type { SessionEntry } from "../../config/sessions.js"; +import type { TemplateContext } from "../templating.js"; +import { + setupAgentRunnerExecutionTestState, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, + expectMockCallArgFields, + createMinimalRunAgentTurnParams, +} from "./agent-runner-execution.test-support.js"; +import type { FallbackRunnerParams } from "./agent-runner-execution.test-support.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: session state", () => { + it("restarts the active prompt when a live model switch is requested", async () => { + let fallbackInvocation = 0; + state.runWithModelFallbackMock.mockImplementation( + async (params: { run: (provider: string, model: string) => Promise }) => { + const isInitialInvocation = fallbackInvocation++ === 0; + const provider = isInitialInvocation ? "anthropic" : "openai"; + const model = isInitialInvocation ? "claude" : "gpt-5.4"; + return { + result: await params.run(provider, model), + provider, + model, + attempts: [], + }; + }, + ); + state.runEmbeddedAgentMock + .mockImplementationOnce(async () => { + throw new LiveSessionModelSwitchError({ + provider: "openai", + model: "gpt-5.4", + agentRuntimeOverride: "codex", + }); + }) + .mockImplementationOnce(async () => { + return { + payloads: [{ text: "switched" }], + meta: { + agentMeta: { + sessionId: "session", + provider: "openai", + model: "gpt-5.4", + }, + }, + }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(2); + expect(followupRun.run.provider).toBe("openai"); + expect(followupRun.run.model).toBe("gpt-5.4"); + expect(state.runEmbeddedAgentMock.mock.calls[1]?.[0]).toEqual( + expect.objectContaining({ agentHarnessRuntimeOverride: "codex" }), + ); + }); + + it("breaks out of the retry loop when LiveSessionModelSwitchError is thrown repeatedly (#58348)", async () => { + // Simulate a scenario where the persisted session selection keeps conflicting + // with the fallback model, causing LiveSessionModelSwitchError on every attempt. + // The outer loop must be bounded to prevent a session death loop. + let switchCallCount = 0; + state.runWithModelFallbackMock.mockImplementation( + async (params: { run: (provider: string, model: string) => Promise }) => { + switchCallCount++; + return { + result: await params.run("anthropic", "claude"), + provider: "anthropic", + model: "claude", + attempts: [], + }; + }, + ); + state.runEmbeddedAgentMock.mockImplementation(async () => { + throw new LiveSessionModelSwitchError({ + provider: "openai", + model: "gpt-5.4", + }); + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + // After two retries the loop must break instead of continuing + // forever. The result should be a final error, not an infinite hang. + expect(result.kind).toBe("final"); + // One initial attempt plus two retries. + expect(switchCallCount).toBe(3); + }); + + it("propagates auth profile state on bounded live model switch retries (#58348)", async () => { + let invocation = 0; + state.runWithModelFallbackMock.mockImplementation( + async (params: { run: (provider: string, model: string) => Promise }) => { + invocation++; + if (invocation <= 2) { + return { + result: await params.run("anthropic", "claude"), + provider: "anthropic", + model: "claude", + attempts: [], + }; + } + // Third invocation succeeds with the switched model + return { + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + }; + }, + ); + state.runEmbeddedAgentMock + .mockImplementationOnce(async () => { + throw new LiveSessionModelSwitchError({ + provider: "openai", + model: "gpt-5.4", + authProfileId: "profile-b", + authProfileIdSource: "user", + }); + }) + .mockImplementationOnce(async () => { + throw new LiveSessionModelSwitchError({ + provider: "openai", + model: "gpt-5.4", + authProfileId: "profile-c", + authProfileIdSource: "auto", + }); + }) + .mockImplementationOnce(async () => { + return { + payloads: [{ text: "finally ok" }], + meta: { + agentMeta: { + sessionId: "session", + provider: "openai", + model: "gpt-5.4", + }, + }, + }; + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const followupRun = createFollowupRun(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + // Two switches (within the limit of 2) then success on third attempt + expect(result.kind).toBe("success"); + expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(3); + expect(followupRun.run.provider).toBe("openai"); + expect(followupRun.run.model).toBe("gpt-5.4"); + expect(followupRun.run.authProfileId).toBe("profile-c"); + expect(followupRun.run.authProfileIdSource).toBe("auto"); + }); + + it("does not roll back newer override changes after a failed fallback candidate", async () => { + state.runWithModelFallbackMock.mockImplementation( + async (params: { run: (provider: string, model: string) => Promise }) => { + await expect(params.run("openai", "gpt-5.4")).rejects.toThrow("fallback failed"); + throw new Error("fallback failed"); + }, + ); + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + providerOverride: "anthropic", + modelOverride: "claude", + authProfileOverride: "anthropic:default", + authProfileOverrideSource: "user", + }; + const sessionStore = { main: sessionEntry }; + state.runEmbeddedAgentMock.mockImplementationOnce(async () => { + sessionEntry.providerOverride = "zai"; + sessionEntry.modelOverride = "glm-5"; + sessionEntry.authProfileOverride = "zai:work"; + sessionEntry.authProfileOverrideSource = "user"; + throw new Error("fallback failed"); + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => sessionEntry, + activeSessionStore: sessionStore, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + expect(sessionEntry.providerOverride).toBe("zai"); + expect(sessionEntry.modelOverride).toBe("glm-5"); + expect(sessionEntry.authProfileOverride).toBe("zai:work"); + expect(sessionEntry.authProfileOverrideSource).toBe("user"); + expect(sessionStore.main.providerOverride).toBe("zai"); + expect(sessionStore.main.modelOverride).toBe("glm-5"); + }); + + it("keeps cross-provider fallback selection turn-local", async () => { + state.runWithModelFallbackMock.mockImplementation( + async (params: { run: (provider: string, model: string) => Promise }) => ({ + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + }), + ); + state.runEmbeddedAgentMock.mockResolvedValue({ + payloads: [{ text: "ok" }], + meta: {}, + }); + + const followupRun = createFollowupRun(); + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-opus"; + followupRun.run.authProfileId = "anthropic:openclaw"; + followupRun.run.authProfileIdSource = "user"; + + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + totalTokens: 1, + compactionCount: 0, + }; + const sessionStore = { main: sessionEntry }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "telegram", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => sessionEntry, + activeSessionStore: sessionStore, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(1); + expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { + provider: "openai", + model: "gpt-5.4", + authProfileId: undefined, + authProfileIdSource: undefined, + }); + expect(sessionEntry.providerOverride).toBeUndefined(); + expect(sessionEntry.modelOverride).toBeUndefined(); + expect(sessionEntry.modelOverrideSource).toBeUndefined(); + expect(sessionEntry.authProfileOverride).toBeUndefined(); + expect(sessionEntry.authProfileOverrideSource).toBeUndefined(); + expect(sessionStore.main.authProfileOverride).toBeUndefined(); + }); + + it("does not persist fallback selection for legacy user overrides without modelOverrideSource", async () => { + // Regression: older persisted sessions can have a user-selected override + // (modelOverride set) but no modelOverrideSource field, because the field + // was added later. These legacy entries must still be protected from + // fallback overwrite, matching the backward-compat treatment in + // session-reset-service. + state.runWithModelFallbackMock.mockImplementation( + async (params: { run: (provider: string, model: string) => Promise }) => ({ + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + }), + ); + state.runEmbeddedAgentMock.mockResolvedValue({ + payloads: [{ text: "ok" }], + meta: {}, + }); + + const followupRun = createFollowupRun(); + followupRun.run.provider = "bailian"; + followupRun.run.model = "qwen3.6-plus"; + + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + totalTokens: 1, + compactionCount: 0, + // Legacy entry: override is set but the source field is missing. + providerOverride: "anthropic", + modelOverride: "claude-opus-4-6", + // modelOverrideSource intentionally absent + }; + const sessionStore = { main: sessionEntry }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "telegram", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => sessionEntry, + activeSessionStore: sessionStore, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + // Legacy user override must survive the fallback unchanged. + expect(sessionEntry.providerOverride).toBe("anthropic"); + expect(sessionEntry.modelOverride).toBe("claude-opus-4-6"); + expect(sessionEntry.modelOverrideSource).toBeUndefined(); + }); + + it("does not replace a recovered auto override during fallback", async () => { + state.runWithModelFallbackMock.mockImplementation( + async (params: { run: (provider: string, model: string) => Promise }) => ({ + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + }), + ); + state.runEmbeddedAgentMock.mockResolvedValue({ + payloads: [{ text: "ok" }], + meta: {}, + }); + + const followupRun = createFollowupRun(); + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-opus-4-6"; + + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + totalTokens: 1, + compactionCount: 0, + providerOverride: "bailian", + modelOverride: "qwen3.6-plus", + modelOverrideFallbackOriginProvider: "minimax", + modelOverrideFallbackOriginModel: "MiniMax-M2.7", + // modelOverrideSource intentionally absent + }; + const sessionStore = { main: sessionEntry }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "telegram", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => sessionEntry, + activeSessionStore: sessionStore, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + expect(sessionEntry.providerOverride).toBe("bailian"); + expect(sessionEntry.modelOverride).toBe("qwen3.6-plus"); + expect(sessionEntry.modelOverrideSource).toBeUndefined(); + expect(sessionEntry.modelOverrideFallbackOriginProvider).toBe("minimax"); + expect(sessionEntry.modelOverrideFallbackOriginModel).toBe("MiniMax-M2.7"); + }); + + it("does not persist fallback selection when modelOverrideSource is user", async () => { + // Regression: fallback persistence overwrote user-initiated /models + // selections. When the user explicitly picked a model, the fallback + // should NOT clobber it even when the primary model fails. + state.runWithModelFallbackMock.mockImplementation( + async (params: { run: (provider: string, model: string) => Promise }) => ({ + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + }), + ); + state.runEmbeddedAgentMock.mockResolvedValue({ + payloads: [{ text: "ok" }], + meta: {}, + }); + + const followupRun = createFollowupRun(); + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-opus-4-6"; + + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + totalTokens: 1, + compactionCount: 0, + // User explicitly selected this model via /models + providerOverride: "anthropic", + modelOverride: "claude-opus-4-6", + modelOverrideSource: "user", + }; + const sessionStore = { main: sessionEntry }; + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun, + sessionCtx: { + Provider: "telegram", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => sessionEntry, + activeSessionStore: sessionStore, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("success"); + // The user's /models selection must survive the fallback. + expect(sessionEntry.providerOverride).toBe("anthropic"); + expect(sessionEntry.modelOverride).toBe("claude-opus-4-6"); + expect(sessionEntry.modelOverrideSource).toBe("user"); + }); + + it("latches assistant error stub suppression across main reply fallback candidates", async () => { + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + await params.run("anthropic", "claude-opus-4-7").catch(() => undefined); + await params.run("anthropic", "claude-opus-4-6").catch(() => undefined); + return { + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + }; + }); + state.runEmbeddedAgentMock.mockImplementationOnce( + async (args: { + onAssistantErrorMessagePersisted?: (message: { + role: "assistant"; + content: string; + stopReason: "error"; + }) => void; + }) => { + args.onAssistantErrorMessagePersisted?.({ + role: "assistant", + content: "[assistant turn failed before producing content]", + stopReason: "error", + }); + throw new Error("upstream 500"); + }, + ); + state.runEmbeddedAgentMock.mockRejectedValueOnce(new Error("upstream 500")); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "ok" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(3); + expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "primary candidate", { + suppressAssistantErrorPersistence: false, + }); + expectMockCallArgFields(state.runEmbeddedAgentMock, 1, "first fallback candidate", { + suppressAssistantErrorPersistence: true, + }); + expectMockCallArgFields(state.runEmbeddedAgentMock, 2, "second fallback candidate", { + suppressAssistantErrorPersistence: true, + }); + }); + + it("does not suppress the first embedded assistant error after a CLI fallback failure", async () => { + state.isCliProviderMock.mockImplementation((provider: unknown) => provider === "anthropic"); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + await params.run("anthropic", "claude-opus-4-7").catch(() => undefined); + return { + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + }; + }); + state.runCliAgentMock.mockRejectedValueOnce(new Error("cli failed")); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "ok" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(state.runCliAgentMock).toHaveBeenCalledOnce(); + expect(state.runEmbeddedAgentMock).toHaveBeenCalledOnce(); + expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded fallback candidate", { + suppressAssistantErrorPersistence: false, + }); + }); + + it("latches queued user message persistence across main reply fallback candidates", async () => { + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + await params.run("anthropic", "claude-opus-4-7").catch(() => undefined); + return { + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [], + }; + }); + state.runEmbeddedAgentMock.mockImplementationOnce( + async (args: { + onUserMessagePersisted?: (m: { + role: "user"; + content: Array<{ type: "text"; text: string }>; + }) => void; + }) => { + args.onUserMessagePersisted?.({ + role: "user", + content: [{ type: "text", text: "queued" }], + }); + throw new Error("upstream 500"); + }, + ); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "ok" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(2); + expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "primary candidate", { + suppressNextUserMessagePersistence: false, + }); + expectMockCallArgFields(state.runEmbeddedAgentMock, 1, "fallback candidate", { + suppressNextUserMessagePersistence: true, + }); + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution-terminal-failures.test.ts b/src/auto-reply/reply/agent-runner-execution-terminal-failures.test.ts new file mode 100644 index 000000000000..e94bb4ee0950 --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution-terminal-failures.test.ts @@ -0,0 +1,654 @@ +import { describe, expect, it, vi } from "vitest"; +import { FailoverError } from "../../agents/failover-error.js"; +import { CommandLaneClearedError, GatewayDrainingError } from "../../process/command-queue.js"; +import { getReplyPayloadMetadata } from "../reply-payload.js"; +import type { TemplateContext } from "../templating.js"; +import { SILENT_REPLY_TOKEN } from "../tokens.js"; +import type { GetReplyOptions } from "../types.js"; +import { + setupAgentRunnerExecutionTestState, + GENERIC_RUN_FAILURE_TEXT, + getRunAgentTurnWithFallback, + createMockTypingSignaler, + createFollowupRun, + createMockReplyOperation, + requireRecord, + expectRecordFields, + requireMockCall, + createMinimalRunAgentTurnParams, +} from "./agent-runner-execution.test-support.js"; +import { HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT } from "./agent-runner-failure-copy.js"; + +const state = setupAgentRunnerExecutionTestState(); + +describe("runAgentTurnWithFallback: terminal failures", () => { + it("surfaces billing guidance for mixed-cause fallback exhaustion", async () => { + state.runWithModelFallbackMock.mockRejectedValueOnce( + Object.assign( + new Error( + "All models failed (2): anthropic/claude: 429 (rate_limit) | openai/gpt-5.4: 402 (billing)", + ), + { + name: "FallbackSummaryError", + attempts: [ + { provider: "anthropic", model: "claude", error: "429", reason: "rate_limit" }, + { provider: "openai", model: "gpt-5.4", error: "402", reason: "billing" }, + ], + soonestCooldownExpiry: Date.now() + 60_000, + }, + ), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe("billing"); + expect(result.payload.text).not.toContain("All models failed"); + expect(result.payload.text).not.toContain("402 (billing)"); + expect(result.payload.text).not.toContain("Rate-limited"); + } + }); + + it("surfaces Codex usage-limit reset details for pure fallback exhaustion", async () => { + const codexMessage = + "You've reached your Codex subscription usage limit. Next reset in 42 minutes (2026-05-04T21:34:00.000Z). Run /codex account for current usage details."; + state.runWithModelFallbackMock.mockRejectedValueOnce( + Object.assign(new Error(`All models failed (1): openai/gpt-5.5: ${codexMessage}`), { + name: "FallbackSummaryError", + attempts: [ + { + provider: "openai", + model: "gpt-5.5", + error: codexMessage, + reason: "rate_limit", + }, + ], + soonestCooldownExpiry: null, + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "telegram", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe(`⚠️ ${codexMessage}`); + expect(result.payload.text).not.toContain("All models failed"); + expectRecordFields(requireRecord(getReplyPayloadMetadata(result.payload), "reply metadata"), { + deliverDespiteSourceReplySuppression: true, + }); + } + }); + + it("surfaces direct Codex usage-limit errors when fallback does not wrap one attempt", async () => { + const codexMessage = + "You've reached your Codex subscription usage limit. Codex did not return a reset time for this limit. Run /codex account for current usage details."; + state.runWithModelFallbackMock.mockRejectedValueOnce(new Error(codexMessage)); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "telegram", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe(`⚠️ ${codexMessage}`); + expectRecordFields(requireRecord(getReplyPayloadMetadata(result.payload), "reply metadata"), { + deliverDespiteSourceReplySuppression: true, + }); + } + }); + + it("surfaces billing guidance for pure billing cooldown fallback exhaustion", async () => { + state.runWithModelFallbackMock.mockRejectedValueOnce( + Object.assign( + new Error( + "All models failed (2): anthropic/claude-opus-4-6: Provider anthropic has billing issue (skipping all models) (billing) | anthropic/claude-sonnet-4-6: Provider anthropic has billing issue (skipping all models) (billing)", + ), + { + name: "FallbackSummaryError", + attempts: [ + { + provider: "anthropic", + model: "claude-opus-4-6", + error: "Provider anthropic has billing issue (skipping all models)", + reason: "billing", + }, + { + provider: "anthropic", + model: "claude-sonnet-4-6", + error: "Provider anthropic has billing issue (skipping all models)", + reason: "billing", + }, + ], + soonestCooldownExpiry: Date.now() + 60_000, + }, + ), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe("billing"); + } + }); + + it("surfaces restart text when fallback exhaustion wraps a drain error, keeping fail bookkeeping", async () => { + const { replyOperation, failMock } = createMockReplyOperation(); + state.runWithModelFallbackMock.mockRejectedValueOnce( + Object.assign(new Error("fallback exhausted"), { + name: "FallbackSummaryError", + attempts: [ + { + provider: "anthropic", + model: "claude", + error: new GatewayDrainingError(), + }, + ], + soonestCooldownExpiry: null, + cause: new GatewayDrainingError(), + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + replyOperation, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + "⚠️ Gateway is restarting. Please wait a few seconds and try again.", + ); + } + const failCall = requireMockCall(failMock, 0, "reply operation fail"); + expect(failCall[0]).toBe("gateway_draining"); + expect(failCall[1]).toBeInstanceOf(GatewayDrainingError); + }); + + it("surfaces restart text when fallback exhaustion wraps a cleared lane error, keeping fail bookkeeping", async () => { + const { replyOperation, failMock } = createMockReplyOperation(); + state.runWithModelFallbackMock.mockRejectedValueOnce( + Object.assign(new Error("fallback exhausted"), { + name: "FallbackSummaryError", + attempts: [ + { + provider: "anthropic", + model: "claude", + error: new CommandLaneClearedError("session:main"), + }, + ], + soonestCooldownExpiry: null, + cause: new CommandLaneClearedError("session:main"), + }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + replyOperation, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + "⚠️ Gateway is restarting. Please wait a few seconds and try again.", + ); + } + const failCall = requireMockCall(failMock, 0, "reply operation fail"); + expect(failCall[0]).toBe("command_lane_cleared"); + expect(failCall[1]).toBeInstanceOf(CommandLaneClearedError); + }); + + it("stays silent (NO_REPLY) when the reply operation was aborted for restart", async () => { + const agentEvents = await import("../../infra/agent-events.js"); + const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); + const { replyOperation, failMock } = createMockReplyOperation(); + Object.defineProperty(replyOperation, "result", { + value: { kind: "aborted", code: "aborted_for_restart" } as const, + configurable: true, + }); + state.runEmbeddedAgentMock.mockRejectedValueOnce( + Object.assign(new Error("aborted"), { name: "AbortError" }), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + replyOperation, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + isRestartRecoveryArmed: () => true, + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe(SILENT_REPLY_TOKEN); + } + expect(failMock).not.toHaveBeenCalled(); + expect( + emitAgentEvent.mock.calls.some( + ([event]) => + event.stream === "lifecycle" && + event.data.phase === "end" && + event.data.aborted === true && + event.data.stopReason === "restart", + ), + ).toBe(true); + }); + + it("preserves restart ownership when an aborted embedded runner resolves normally", async () => { + const agentEvents = await import("../../infra/agent-events.js"); + const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); + const { replyOperation } = createMockReplyOperation(); + Object.defineProperty(replyOperation, "result", { + value: { kind: "aborted", code: "aborted_for_restart" } as const, + configurable: true, + }); + state.runEmbeddedAgentMock.mockResolvedValueOnce({ + payloads: [], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + replyOperation, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + isRestartRecoveryArmed: () => true, + }); + + expect(result).toEqual({ + kind: "final", + payload: expect.objectContaining({ + text: SILENT_REPLY_TOKEN, + }), + }); + expect( + emitAgentEvent.mock.calls.some( + ([event]) => + event.stream === "lifecycle" && + event.data.phase === "end" && + event.data.aborted === true && + event.data.stopReason === "restart", + ), + ).toBe(true); + }); + + it("uses compact generic copy for raw external chat errors when verbose is off", async () => { + const agentEvents = await import("../../infra/agent-events.js"); + const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error("INVALID_ARGUMENT: some other failure"), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: { runId: "run-provider-failure" } as GetReplyOptions, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe(GENERIC_RUN_FAILURE_TEXT); + } + const terminalFailureEvent = emitAgentEvent.mock.calls + .map((call) => call[0]) + .find((event) => { + if (!event || typeof event !== "object") { + return false; + } + const data = (event as { data?: Record }).data; + return ( + (event as { runId?: unknown }).runId === "run-provider-failure" && + (event as { stream?: unknown }).stream === "lifecycle" && + data?.phase === "error" && + data.fallbackExhaustedFailure === true + ); + }); + expect(terminalFailureEvent).toBeDefined(); + }); + + it("surfaces CLI max-turn recovery context at normal verbosity", async () => { + const recoveryText = + "Claude CLI stopped after reaching the maximum number of turns (limit: 1). " + + "OpenClaw run: run-max-turns. OpenClaw session: session-1. Claude session: claude-session-1. " + + "Tool actions may already have run; verify their effects before retrying. " + + "Retry with a higher --max-turns value or a narrower task."; + const maxTurns = new FailoverError(recoveryText, { + reason: "unknown", + code: "cli_max_turns", + provider: "claude-cli", + model: "sonnet", + }); + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new AggregateError( + [maxTurns, new Error("fork successor persistence failed")], + "CLI turn failed and its fork successor could not be persisted", + ), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.isError).toBe(true); + expect(result.payload.text).toBe(recoveryText); + expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); + } + }); + + it("uses heartbeat failure copy for raw external errors during heartbeat runs", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error('Command lane "main" task timed out after 120000ms'), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams(), + isHeartbeat: true, + }); + + expect(result.kind).toBe("final"); + if (result.kind !== "final") { + throw new Error("expected final reply"); + } + expect(result.payload.text).toBe(HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT); + expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); + expect(result.payload.text).not.toContain("/new"); + }); + + it.each([ + { + rejection: new Error("CLI exceeded timeout (300s) and was terminated."), + modeLabel: "overall CLI turn budget" as const, + routingSubstring: undefined as string | undefined, + }, + { + rejection: new Error("CLI produced no output for 120s and was terminated."), + modeLabel: "no-output stall" as const, + routingSubstring: undefined, + }, + { + rejection: new Error( + "All models failed (2): anthropic/claude-opus-4-7: CLI exceeded timeout (300s) and was terminated. | anthropic/foo: bar", + ), + modeLabel: "overall CLI turn budget" as const, + routingSubstring: "(routing anthropic/claude-opus-4-7)", + }, + { + rejection: new Error("codex-cli/gpt-5.5: CLI exceeded timeout (60s) and was terminated."), + modeLabel: "overall CLI turn budget" as const, + routingSubstring: "(routing codex-cli/gpt-5.5)", + }, + ])( + "surfaces CLI subprocess timeout copy instead of generic failure when verbose is off ($modeLabel)", + async ({ rejection, modeLabel, routingSubstring }) => { + state.runWithModelFallbackMock.mockRejectedValueOnce(rejection); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams(), + }); + + expect(result.kind).toBe("final"); + if (result.kind !== "final") { + throw new Error("expected final reply"); + } + expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); + expect(result.payload.text).toContain("CLI subprocess"); + expect(result.payload.text).not.toContain("Claude CLI"); + expect(result.payload.text).toContain(modeLabel); + expect(result.payload.text).toContain("gateway may still be healthy"); + expect(result.payload.text).toContain("cliBackends."); + if (routingSubstring) { + expect(result.payload.text).toContain(routingSubstring); + } + }, + ); + + it.each([ + { + rejection: new Error("codex app-server client closed before turn completed"), + expected: "connection closed", + }, + { + rejection: new Error("codex app-server turn idle timed out waiting for turn/completed"), + expected: "did not replay the turn automatically", + }, + ])( + "surfaces Codex app-server bridge failures instead of generic copy", + async ({ rejection, expected }) => { + state.runWithModelFallbackMock.mockRejectedValueOnce(rejection); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams(), + }); + + expect(result.kind).toBe("final"); + if (result.kind !== "final") { + throw new Error("expected final reply"); + } + expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); + expect(result.payload.text).toContain("Codex app-server"); + expect(result.payload.text).toContain(expected); + }, + ); + + it("forwards sanitized generic errors on external chat channels when verbose is on", async () => { + state.runEmbeddedAgentMock.mockRejectedValueOnce( + new Error("INVALID_ARGUMENT: some other failure"), + ); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + commandBody: "hello", + followupRun: createFollowupRun(), + sessionCtx: { + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext, + opts: {}, + typingSignals: createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end", + applyReplyToMode: (payload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "on", + }); + + expect(result.kind).toBe("final"); + if (result.kind === "final") { + expect(result.payload.text).toBe( + "⚠️ Agent failed before reply: INVALID_ARGUMENT: some other failure. Please try again, or use /new to start a fresh session.", + ); + } + }); +}); diff --git a/src/auto-reply/reply/agent-runner-execution.test-support.ts b/src/auto-reply/reply/agent-runner-execution.test-support.ts new file mode 100644 index 000000000000..ec262a4cfdb6 --- /dev/null +++ b/src/auto-reply/reply/agent-runner-execution.test-support.ts @@ -0,0 +1,587 @@ +// Shared mocks and fixtures for agent-runner execution tests. +import { afterEach, beforeEach, expect, vi } from "vitest"; +import { testing as cliBackendsTesting } from "../../agents/cli-backends.js"; +import { AUTH_INVALID_TOKEN_USER_TEXT } from "../../agents/embedded-agent-helpers/errors.js"; +import type { ModelDefinitionConfig } from "../../config/types.models.js"; +import type { ReplyOptionsWithHeartbeatRunScope } from "../../infra/heartbeat-run-scope.js"; +import { + createUserTurnTranscriptRecorder, + type PersistedUserTurnMessage, +} from "../../sessions/user-turn-transcript.js"; +import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js"; +import type { TemplateContext } from "../templating.js"; +import type { GetReplyOptions, ReplyPayload } from "../types.js"; +import type { FollowupRun } from "./queue.js"; +import type { ReplyOperation } from "./reply-run-registry.js"; +import type { TypingSignaler } from "./typing-mode.js"; + +export const PROVIDER_AUTHENTICATION_ERROR_USER_MESSAGE = `⚠️ ${AUTH_INVALID_TOKEN_USER_TEXT}`; +export const PROVIDER_RATE_LIMIT_OR_QUOTA_ERROR_USER_MESSAGE = + "⚠️ The model provider returned HTTP 429 before replying. This can mean rate limiting, exhausted quota, or an account balance/billing issue. Check the selected provider/model, API key, and provider billing/quota dashboard, then try again."; +export const PROVIDER_INTERNAL_ERROR_USER_MESSAGE = + "⚠️ The model provider returned a temporary internal error before replying. Try again in a moment, or switch to another model if it keeps happening."; + +const state = vi.hoisted(() => ({ + runEmbeddedAgentMock: vi.fn(), + runCliAgentMock: vi.fn(), + runWithModelFallbackMock: vi.fn(), + isCliProviderMock: vi.fn((_: unknown) => false), + isInternalMessageChannelMock: vi.fn((_: unknown) => false), + createBlockReplyDeliveryHandlerMock: vi.fn(), + isCompactionFailureErrorMock: vi.fn((_: string | undefined) => false), + isContextOverflowErrorMock: vi.fn((_: string | undefined) => false), + isLikelyContextOverflowErrorMock: vi.fn((_: string | undefined) => false), + updateSessionStoreMock: vi.fn(), + resolveCurrentTurnImagesMock: vi.fn(), +})); + +export const GENERIC_RUN_FAILURE_TEXT = + "⚠️ Something went wrong while processing your request. Please try again, or use /new to start a fresh session."; +export function makeTestModel(id: string, contextTokens: number): ModelDefinitionConfig { + return { + id, + name: id, + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: contextTokens, + contextTokens, + maxTokens: 4096, + }; +} + +vi.mock("../../agents/embedded-agent.js", () => ({ + runEmbeddedAgent: (params: unknown) => state.runEmbeddedAgentMock(params), +})); + +vi.mock("../../agents/cli-runner.js", () => ({ + runCliAgent: (params: unknown) => state.runCliAgentMock(params), +})); + +vi.mock("../../agents/model-fallback.js", () => ({ + runWithModelFallback: (params: unknown) => state.runWithModelFallbackMock(params), + isFallbackSummaryError: (err: unknown) => + err instanceof Error && + err.name === "FallbackSummaryError" && + Array.isArray((err as { attempts?: unknown[] }).attempts), +})); + +vi.mock("../../agents/model-selection.js", async () => { + const actual = await vi.importActual( + "../../agents/model-selection.js", + ); + return { + ...actual, + isCliProvider: (provider: unknown) => state.isCliProviderMock(provider), + }; +}); + +vi.mock("../../agents/bootstrap-budget.js", () => ({ + resolveBootstrapWarningSignaturesSeen: () => [], +})); + +vi.mock("../../agents/embedded-agent-helpers.js", async () => { + const actual = await vi.importActual( + "../../agents/embedded-agent-helpers.js", + ); + return { + BILLING_ERROR_USER_MESSAGE: "billing", + formatBillingErrorMessage: actual.formatBillingErrorMessage, + formatRateLimitOrOverloadedErrorCopy: (message: string) => { + if (/model\s+(?:is\s+)?at capacity/i.test(message)) { + return "⚠️ Selected model is at capacity. Try a different model, or wait and retry."; + } + if (/rate.limit|too many requests|429/i.test(message)) { + return "⚠️ API rate limit reached. Please try again later."; + } + if (/overloaded/i.test(message)) { + return "The AI service is temporarily overloaded. Please try again in a moment."; + } + return undefined; + }, + isCompactionFailureError: (message?: string) => state.isCompactionFailureErrorMock(message), + isContextOverflowError: (message?: string) => state.isContextOverflowErrorMock(message), + isBillingErrorMessage: actual.isBillingErrorMessage, + isLikelyContextOverflowError: (message?: string) => + state.isLikelyContextOverflowErrorMock(message), + isOverloadedErrorMessage: (message: string) => /overloaded|capacity/i.test(message), + isRateLimitErrorMessage: (message: string) => + /rate.limit|too many requests|429|usage limit/i.test(message), + isTransientHttpError: () => false, + sanitizeUserFacingText: (text?: string) => text ?? "", + }; +}); + +vi.mock("../../config/sessions.js", () => ({ + resolveGroupSessionKey: vi.fn(() => null), + resolveSessionTranscriptPath: vi.fn(), + updateSessionStore: state.updateSessionStoreMock, +})); + +vi.mock("../../globals.js", () => ({ + logVerbose: vi.fn(), +})); + +vi.mock("../../infra/agent-events.js", async () => { + const actual = await vi.importActual( + "../../infra/agent-events.js", + ); + const emitAgentEvent = vi.fn((...args: Parameters) => + actual.emitAgentEvent(...args), + ); + return { + ...actual, + clearAgentRunContext: vi.fn(), + emitAgentEvent, + registerAgentRunContext: vi.fn(), + }; +}); + +vi.mock("../../runtime.js", () => ({ + defaultRuntime: { + error: vi.fn(), + }, +})); + +vi.mock("../../utils/message-channel.js", () => ({ + isMarkdownCapableMessageChannel: () => true, + resolveMessageChannel: () => "whatsapp", + isInternalMessageChannel: (value: unknown) => state.isInternalMessageChannelMock(value), +})); + +vi.mock("../heartbeat.js", () => ({ + stripHeartbeatToken: (text: string) => ({ + text, + didStrip: false, + shouldSkip: false, + }), +})); + +vi.mock("./current-turn-images.js", () => ({ + resolveCurrentTurnImages: (params: unknown) => state.resolveCurrentTurnImagesMock(params), +})); + +vi.mock("./agent-runner-utils.js", () => ({ + buildEmbeddedRunExecutionParams: (params: { + provider: string; + model: string; + run: { + provider?: string; + thinkLevel?: string; + authProfileId?: string; + authProfileIdSource?: "auto" | "user"; + agentAccountId?: string; + chatType?: string; + }; + replyRoute?: { + originatingChannel?: string; + originatingTo?: string; + originatingAccountId?: string; + originatingChatType?: string; + }; + sessionCtx: { AccountId?: string; ChatType?: string }; + }) => ({ + embeddedContext: { + messageProvider: params.replyRoute?.originatingChannel, + messageTo: params.replyRoute?.originatingTo, + agentAccountId: + params.replyRoute?.originatingAccountId ?? + params.sessionCtx.AccountId ?? + params.run.agentAccountId, + chatType: + params.replyRoute?.originatingChatType ?? params.sessionCtx.ChatType ?? params.run.chatType, + }, + senderContext: {}, + runBaseParams: { + provider: params.provider, + model: params.model, + thinkLevel: params.run.thinkLevel, + authProfileId: params.provider === params.run.provider ? params.run.authProfileId : undefined, + authProfileIdSource: + params.provider === params.run.provider ? params.run.authProfileIdSource : undefined, + }, + }), + resolveQueuedReplyRuntimeConfig: (config: T) => config, + resolveModelFallbackOptions: vi.fn( + (run: { provider?: string; model?: string; config?: unknown; agentDir?: string }) => ({ + provider: run.provider, + model: run.model, + cfg: run.config, + agentDir: run.agentDir, + }), + ), + resolveRunFastModeForFallbackCandidate: (params: { + run: { fastMode?: unknown; fastModeAutoOnSeconds?: unknown }; + }) => ({ + fastMode: params.run.fastMode, + fastModeAutoOnSeconds: params.run.fastModeAutoOnSeconds, + }), +})); + +vi.mock("./reply-delivery.js", () => ({ + createBlockReplyDeliveryHandler: (params: unknown) => + state.createBlockReplyDeliveryHandlerMock(params), +})); + +vi.mock("./reply-media-paths.runtime.js", () => ({ + createReplyMediaContext: () => ({ + normalizePayload: (payload: unknown) => payload, + }), + createReplyMediaPathNormalizer: () => (payload: unknown) => payload, +})); + +export async function getRunAgentTurnWithFallback() { + return (await import("./agent-runner-execution.js")).runAgentTurnWithFallback; +} + +export type FallbackRunnerParams = { + provider: string; + model: string; + sessionId?: string; + abortSignal?: AbortSignal; + run: (provider: string, model: string) => Promise; + classifyResult?: (params: { + result: { payloads?: Array<{ text?: string; isError?: boolean; isReasoning?: boolean }> }; + provider: string; + model: string; + attempt: number; + total: number; + }) => Promise; +}; + +export type EmbeddedAgentParams = { + lifecycleGeneration?: string; + onExecutionStarted?: (info?: { lifecycleGeneration?: string }) => void; + onExecutionPhase?: (info: { + phase: + | "runner_entered" + | "workspace" + | "runtime_plugins" + | "before_agent_reply" + | "model_resolution" + | "auth" + | "context_engine" + | "attempt_dispatch" + | "context_assembled" + | "turn_accepted" + | "process_spawned" + | "tool_execution_started" + | "assistant_output_started" + | "model_call_started"; + provider?: string; + model?: string; + backend?: string; + source?: string; + tool?: string; + toolCallId?: string; + itemId?: string; + }) => void; + onBlockReply?: (payload: { text?: string; mediaUrls?: string[] }) => Promise | void; + onPartialReply?: (payload: { text?: string; mediaUrls?: string[] }) => Promise | void; + onAssistantMessageStart?: () => Promise | void; + onToolResult?: (payload: { text?: string; mediaUrls?: string[] }) => Promise | void; + onReasoningStream?: (payload: { + text?: string; + mediaUrls?: string[]; + isReasoningSnapshot?: boolean; + requiresReasoningProgressOptIn?: boolean; + }) => Promise | void; + onReasoningEnd?: () => Promise | void; + onItemEvent?: (payload: { + itemId?: string; + toolCallId?: string; + kind?: string; + title?: string; + name?: string; + phase?: string; + status?: string; + summary?: string; + progressText?: string; + approvalId?: string; + approvalSlug?: string; + }) => Promise | void; + onAgentEvent?: (payload: { + stream: string; + data: Record; + sessionKey?: string; + }) => Promise | void; +}; + +export function createMockTypingSignaler(): TypingSignaler { + return { + mode: "message", + shouldStartImmediately: false, + shouldStartOnMessageStart: true, + shouldStartOnText: true, + shouldStartOnReasoning: false, + signalRunStart: vi.fn(async () => {}), + signalMessageStart: vi.fn(async () => {}), + signalTextDelta: vi.fn(async () => {}), + signalReasoningDelta: vi.fn(async () => {}), + signalToolStart: vi.fn(async () => {}), + signalExecutionActivity: vi.fn(async () => {}), + }; +} + +export function createFollowupRun(): FollowupRun { + return { + prompt: "hello", + summaryLine: "hello", + enqueuedAt: Date.now(), + run: { + agentId: "agent", + agentDir: "/tmp/agent", + sessionId: "session", + sessionKey: "main", + messageProvider: "whatsapp", + sessionFile: "/tmp/session.jsonl", + workspaceDir: "/tmp", + config: {}, + skillsSnapshot: {}, + provider: "anthropic", + model: "claude", + verboseLevel: "off", + elevatedLevel: "off", + bashElevated: { + enabled: false, + allowed: false, + defaultLevel: "off", + }, + timeoutMs: 1_000, + blockReplyBreak: "message_end", + }, + } as unknown as FollowupRun; +} + +export function createTestUserTurnRecorder(message: PersistedUserTurnMessage) { + return createUserTurnTranscriptRecorder({ + message, + target: createTestUserTurnTranscriptTarget(), + updateMode: "none", + }); +} + +export function createMockReplyOperation(): { + replyOperation: ReplyOperation; + failMock: ReturnType; + freezeAbortMock: ReturnType; + retainFailureUntilCompleteMock: ReturnType; + updateSessionIdMock: ReturnType; +} { + const failMock = vi.fn(); + const freezeAbortMock = vi.fn(); + const retainFailureUntilCompleteMock = vi.fn(); + const updateSessionIdMock = vi.fn(); + return { + failMock, + freezeAbortMock, + retainFailureUntilCompleteMock, + updateSessionIdMock, + replyOperation: { + key: "main", + sessionId: "session", + abortSignal: new AbortController().signal, + resetTriggered: false, + terminalRecovery: false, + acceptedSteeredInboundAudio: false, + phase: "running", + result: null, + startedAtMs: Date.now(), + lastActivityAtMs: Date.now(), + hasOwnedSessionId: vi.fn((sessionId: string) => sessionId === "session"), + recordActivity: vi.fn(), + setPhase: vi.fn(), + markWaitingForDeferredMaintenance: vi.fn(), + markDeferredMaintenanceWaitEnded: vi.fn(), + updateSessionId: updateSessionIdMock, + updateSessionKey: vi.fn(), + attachBackend: vi.fn(), + detachBackend: vi.fn(), + freezeAbort: freezeAbortMock, + retainFailureUntilComplete: retainFailureUntilCompleteMock, + complete: vi.fn(), + completeThen: vi.fn((afterClear: () => void) => afterClear()), + completeWithAfterClearBarrier: vi.fn(), + fail: failMock, + abortByUser: vi.fn(() => true), + abortForRestart: vi.fn(() => true), + markTerminalRecovery: vi.fn(), + markAcceptedSteeredInboundAudio: vi.fn(), + }, + }; +} + +export function requireRecord(value: unknown, label: string): Record { + if (typeof value !== "object" || value === null) { + throw new Error(`${label} was not an object`); + } + return value as Record; +} + +export function expectRecordFields( + record: Record, + fields: Record, +) { + for (const [key, value] of Object.entries(fields)) { + expect(record[key]).toEqual(value); + } +} + +export function requireMockCall(mock: unknown, index: number, label: string): unknown[] { + const call = (mock as { mock?: { calls?: unknown[][] } }).mock?.calls?.[index]; + if (!call) { + throw new Error(`missing ${label} call ${index + 1}`); + } + return call; +} + +export function expectMockCallArgFields( + mock: unknown, + index: number, + label: string, + fields: Record, +) { + expectRecordFields(requireRecord(requireMockCall(mock, index, label)[0], label), fields); +} + +export function expectNoMockCallWithFields(mock: unknown, fields: Record) { + const calls = (mock as { mock?: { calls?: unknown[][] } }).mock?.calls ?? []; + const hasMatchingCall = calls.some((call) => { + const value = call[0]; + if (typeof value !== "object" || value === null) { + return false; + } + const record = value as Record; + return Object.entries(fields).every(([key, expected]) => record[key] === expected); + }); + expect(hasMatchingCall).toBe(false); +} + +export function requireMockCallArgWithFields( + mock: unknown, + fields: Record, + label: string, +) { + const calls = (mock as { mock?: { calls?: unknown[][] } }).mock?.calls ?? []; + const found = calls + .map((call) => call[0]) + .find((value) => { + if (typeof value !== "object" || value === null) { + return false; + } + const record = value as Record; + return Object.entries(fields).every(([key, expected]) => record[key] === expected); + }); + if (!found) { + throw new Error(`missing ${label}`); + } + return requireRecord(found, label); +} + +export function expectBlockReplyCall( + onBlockReply: unknown, + index: number, + fields: Record, +) { + expectMockCallArgFields(onBlockReply, index, "block reply payload", fields); +} + +export function createMinimalRunAgentTurnParams(overrides?: { + followupRun?: FollowupRun; + opts?: GetReplyOptions & ReplyOptionsWithHeartbeatRunScope; + replyOperation?: ReplyOperation; + sessionCtx?: TemplateContext; + typingSignals?: TypingSignaler; +}) { + return { + commandBody: "fix it", + followupRun: overrides?.followupRun ?? createFollowupRun(), + sessionCtx: + overrides?.sessionCtx ?? + ({ + Provider: "whatsapp", + MessageSid: "msg", + } as unknown as TemplateContext), + opts: overrides?.opts ?? ({} satisfies GetReplyOptions), + replyOperation: overrides?.replyOperation, + typingSignals: overrides?.typingSignals ?? createMockTypingSignaler(), + blockReplyPipeline: null, + blockStreamingEnabled: false, + resolvedBlockStreamingBreak: "message_end" as const, + applyReplyToMode: (payload: ReplyPayload) => payload, + shouldEmitToolResult: () => true, + shouldEmitToolOutput: () => false, + pendingToolTasks: new Set>(), + resetSessionAfterRoleOrderingConflict: async () => false, + isHeartbeat: false, + sessionKey: "main", + getActiveSessionEntry: () => undefined, + resolvedVerboseLevel: "off" as const, + }; +} + +export const NON_DIRECT_FAILURE_SURFACE_CASES = [ + { label: "Discord group", provider: "discord", chatType: "group" }, + { label: "Discord channel", provider: "discord", chatType: "channel" }, + { label: "Slack channel", provider: "slack", chatType: "channel" }, + { label: "Telegram group", provider: "telegram", chatType: "group" }, + { label: "WhatsApp group", provider: "whatsapp", chatType: "group" }, + { label: "Microsoft Teams channel", provider: "msteams", chatType: "channel" }, +] as const; + +export function createNonDirectFailureSessionCtx( + testCase: (typeof NON_DIRECT_FAILURE_SURFACE_CASES)[number], +): TemplateContext { + return { + Provider: testCase.provider, + Surface: testCase.provider, + ChatType: testCase.chatType, + GroupSubject: `${testCase.label} fixture`, + GroupChannel: "#general", + MessageSid: "msg", + } as unknown as TemplateContext; +} + +export function setupAgentRunnerExecutionTestState() { + beforeEach(() => { + vi.useRealTimers(); + state.runEmbeddedAgentMock.mockReset(); + state.runCliAgentMock.mockReset(); + state.runWithModelFallbackMock.mockReset(); + state.isCliProviderMock.mockReset(); + state.isCliProviderMock.mockReturnValue(false); + state.isInternalMessageChannelMock.mockReset(); + state.isInternalMessageChannelMock.mockReturnValue(false); + state.createBlockReplyDeliveryHandlerMock.mockReset(); + state.createBlockReplyDeliveryHandlerMock.mockReturnValue(undefined); + state.isCompactionFailureErrorMock.mockReset(); + state.isCompactionFailureErrorMock.mockReturnValue(false); + state.isContextOverflowErrorMock.mockReset(); + state.isContextOverflowErrorMock.mockReturnValue(false); + state.isLikelyContextOverflowErrorMock.mockReset(); + state.isLikelyContextOverflowErrorMock.mockReturnValue(false); + state.updateSessionStoreMock.mockReset(); + state.resolveCurrentTurnImagesMock.mockReset(); + state.resolveCurrentTurnImagesMock.mockImplementation( + async (params: { images?: unknown[]; imageOrder?: unknown[] }) => ({ + images: params.images, + imageOrder: params.imageOrder, + }), + ); + state.runWithModelFallbackMock.mockImplementation(async (params: FallbackRunnerParams) => ({ + result: await params.run("anthropic", "claude"), + provider: "anthropic", + model: "claude", + attempts: [], + })); + }); + + afterEach(() => { + // Fake-timer tests must not leak into --isolate=false peers. + vi.useRealTimers(); + cliBackendsTesting.resetDepsForTest(); + vi.clearAllMocks(); + }); + + return state; +} diff --git a/src/auto-reply/reply/agent-runner-execution.test.ts b/src/auto-reply/reply/agent-runner-execution.test.ts deleted file mode 100644 index 4cbba9f26d38..000000000000 --- a/src/auto-reply/reply/agent-runner-execution.test.ts +++ /dev/null @@ -1,9320 +0,0 @@ -// Tests agent runner execution setup, command args, and model fallback routing. -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { OAuthRefreshFailureError } from "../../agents/auth-profiles/oauth-refresh-failure.js"; -import { testing as cliBackendsTesting } from "../../agents/cli-backends.js"; -import { formatBillingErrorMessage } from "../../agents/embedded-agent-helpers.js"; -import { AUTH_INVALID_TOKEN_USER_TEXT } from "../../agents/embedded-agent-helpers/errors.js"; -import { FailoverError } from "../../agents/failover-error.js"; -import { LiveSessionModelSwitchError } from "../../agents/live-model-switch-error.js"; -import { MissingProviderAuthError } from "../../agents/model-auth.js"; -import { createAgentRunRestartAbortError } from "../../agents/run-termination.js"; -import { resolveSessionRuntimeOverrideForProvider } from "../../agents/session-runtime-compat.js"; -import type { SessionEntry } from "../../config/sessions.js"; -import type { ModelDefinitionConfig } from "../../config/types.models.js"; -import { - HEARTBEAT_RUN_SCOPE, - type ReplyOptionsWithHeartbeatRunScope, -} from "../../infra/heartbeat-run-scope.js"; -import { resetLogger, setLoggerOverride } from "../../logging/logger.js"; -import { loggingState } from "../../logging/state.js"; -import { CommandLaneClearedError, GatewayDrainingError } from "../../process/command-queue.js"; -import { - createUserTurnTranscriptRecorder, - type PersistedUserTurnMessage, -} from "../../sessions/user-turn-transcript.js"; -import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js"; -import { getReplyPayloadMetadata } from "../reply-payload.js"; -import type { TemplateContext } from "../templating.js"; -import { SILENT_REPLY_TOKEN } from "../tokens.js"; -import type { GetReplyOptions, ReplyPayload } from "../types.js"; -import { resolveFallbackCandidateRun } from "./agent-runner-auth-profile.js"; -import { resolveRunAfterAutoFallbackPrimaryProbeRecheck } from "./agent-runner-auto-fallback.js"; -import { - buildContextOverflowRecoveryText, - computeContextAwareReserveTokensFloor, -} from "./agent-runner-context-recovery.js"; -import { HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT } from "./agent-runner-failure-copy.js"; -import { - buildEmptyInteractiveReplyPayload, - buildKnownAgentRunFailureReplyPayload, -} from "./agent-runner-failure-reply.js"; -import type { InternalGetReplyOptions } from "./get-reply.types.js"; -import { PROVIDER_CONVERSATION_STATE_ERROR_USER_MESSAGE } from "./provider-request-error-classifier.js"; -import type { FollowupRun } from "./queue.js"; -import { createReplyOperation, type ReplyOperation } from "./reply-run-registry.js"; -import type { TypingSignaler } from "./typing-mode.js"; - -const PROVIDER_AUTHENTICATION_ERROR_USER_MESSAGE = `⚠️ ${AUTH_INVALID_TOKEN_USER_TEXT}`; -const PROVIDER_RATE_LIMIT_OR_QUOTA_ERROR_USER_MESSAGE = - "⚠️ The model provider returned HTTP 429 before replying. This can mean rate limiting, exhausted quota, or an account balance/billing issue. Check the selected provider/model, API key, and provider billing/quota dashboard, then try again."; -const PROVIDER_INTERNAL_ERROR_USER_MESSAGE = - "⚠️ The model provider returned a temporary internal error before replying. Try again in a moment, or switch to another model if it keeps happening."; - -const state = vi.hoisted(() => ({ - runEmbeddedAgentMock: vi.fn(), - runCliAgentMock: vi.fn(), - runWithModelFallbackMock: vi.fn(), - isCliProviderMock: vi.fn((_: unknown) => false), - isInternalMessageChannelMock: vi.fn((_: unknown) => false), - createBlockReplyDeliveryHandlerMock: vi.fn(), - isCompactionFailureErrorMock: vi.fn((_: string | undefined) => false), - isContextOverflowErrorMock: vi.fn((_: string | undefined) => false), - isLikelyContextOverflowErrorMock: vi.fn((_: string | undefined) => false), - updateSessionStoreMock: vi.fn(), - resolveCurrentTurnImagesMock: vi.fn(), -})); - -const GENERIC_RUN_FAILURE_TEXT = - "⚠️ Something went wrong while processing your request. Please try again, or use /new to start a fresh session."; -const EMPTY_INTERACTIVE_REPLY_TEXT = - "I finished the turn, but it did not produce a visible reply. Please try again, or start a new session if this keeps happening."; - -describe("resolveSessionRuntimeOverrideForProvider", () => { - it("honors an explicit OpenClaw override for OpenAI", () => { - expect( - resolveSessionRuntimeOverrideForProvider({ - provider: "openai", - entry: { agentRuntimeOverride: "openclaw" } as SessionEntry, - }), - ).toBe("openclaw"); - }); - - afterEach(() => { - cliBackendsTesting.resetDepsForTest(); - }); - - it("ignores unsupported session runtime pins", () => { - expect( - resolveSessionRuntimeOverrideForProvider({ - provider: "openai", - entry: { agentRuntimeOverride: "unsupported-runtime" }, - }), - ).toBeUndefined(); - }); - - it.each([ - { provider: "openai", expected: "codex" }, - { provider: "codex", expected: "codex" }, - { provider: "anthropic", expected: undefined }, - ])("resolves Codex runtime compatibility for $provider", ({ provider, expected }) => { - expect( - resolveSessionRuntimeOverrideForProvider({ - provider, - entry: { agentRuntimeOverride: "codex" }, - }), - ).toBe(expected); - }); - - it("does not treat an observed harness as a future-turn override", () => { - expect( - resolveSessionRuntimeOverrideForProvider({ - provider: "anthropic", - entry: { agentHarnessId: "codex" }, - }), - ).toBeUndefined(); - }); - - it("keeps a locked harness pin ahead of a conflicting runtime override", () => { - expect( - resolveSessionRuntimeOverrideForProvider({ - provider: "anthropic", - entry: { - agentHarnessId: "codex", - agentRuntimeOverride: "claude-cli", - modelSelectionLocked: true, - }, - }), - ).toBe("codex"); - }); - it("keeps CLI runtime pins only when the runtime serves the selected provider", () => { - cliBackendsTesting.setDepsForTest({ - resolveRuntimeCliBackends: () => [], - resolvePluginSetupCliBackend: ({ backend, config }) => - backend === "claude-cli" && config - ? { - pluginId: "anthropic", - backend: { - id: "claude-cli", - modelProvider: "anthropic", - config: { command: "claude" }, - bundleMcp: false, - }, - } - : undefined, - }); - const cfg = { - agents: { - defaults: { - cliBackends: { - "claude-cli": { command: "claude" }, - }, - }, - }, - }; - - expect( - resolveSessionRuntimeOverrideForProvider({ - provider: "anthropic", - entry: { agentRuntimeOverride: "claude-cli" }, - cfg, - }), - ).toBe("claude-cli"); - expect( - resolveSessionRuntimeOverrideForProvider({ - provider: "openai", - entry: { agentRuntimeOverride: "claude-cli" }, - cfg, - }), - ).toBeUndefined(); - }); -}); - -function makeTestModel(id: string, contextTokens: number): ModelDefinitionConfig { - return { - id, - name: id, - reasoning: false, - input: ["text"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: contextTokens, - contextTokens, - maxTokens: 4096, - }; -} - -vi.mock("../../agents/embedded-agent.js", () => ({ - runEmbeddedAgent: (params: unknown) => state.runEmbeddedAgentMock(params), -})); - -vi.mock("../../agents/cli-runner.js", () => ({ - runCliAgent: (params: unknown) => state.runCliAgentMock(params), -})); - -vi.mock("../../agents/model-fallback.js", () => ({ - runWithModelFallback: (params: unknown) => state.runWithModelFallbackMock(params), - isFallbackSummaryError: (err: unknown) => - err instanceof Error && - err.name === "FallbackSummaryError" && - Array.isArray((err as { attempts?: unknown[] }).attempts), -})); - -vi.mock("../../agents/model-selection.js", async () => { - const actual = await vi.importActual( - "../../agents/model-selection.js", - ); - return { - ...actual, - isCliProvider: (provider: unknown) => state.isCliProviderMock(provider), - }; -}); - -vi.mock("../../agents/bootstrap-budget.js", () => ({ - resolveBootstrapWarningSignaturesSeen: () => [], -})); - -vi.mock("../../agents/embedded-agent-helpers.js", async () => { - const actual = await vi.importActual( - "../../agents/embedded-agent-helpers.js", - ); - return { - BILLING_ERROR_USER_MESSAGE: "billing", - formatBillingErrorMessage: actual.formatBillingErrorMessage, - formatRateLimitOrOverloadedErrorCopy: (message: string) => { - if (/model\s+(?:is\s+)?at capacity/i.test(message)) { - return "⚠️ Selected model is at capacity. Try a different model, or wait and retry."; - } - if (/rate.limit|too many requests|429/i.test(message)) { - return "⚠️ API rate limit reached. Please try again later."; - } - if (/overloaded/i.test(message)) { - return "The AI service is temporarily overloaded. Please try again in a moment."; - } - return undefined; - }, - isCompactionFailureError: (message?: string) => state.isCompactionFailureErrorMock(message), - isContextOverflowError: (message?: string) => state.isContextOverflowErrorMock(message), - isBillingErrorMessage: actual.isBillingErrorMessage, - isLikelyContextOverflowError: (message?: string) => - state.isLikelyContextOverflowErrorMock(message), - isOverloadedErrorMessage: (message: string) => /overloaded|capacity/i.test(message), - isRateLimitErrorMessage: (message: string) => - /rate.limit|too many requests|429|usage limit/i.test(message), - isTransientHttpError: () => false, - sanitizeUserFacingText: (text?: string) => text ?? "", - }; -}); - -vi.mock("../../config/sessions.js", () => ({ - resolveGroupSessionKey: vi.fn(() => null), - resolveSessionTranscriptPath: vi.fn(), - updateSessionStore: state.updateSessionStoreMock, -})); - -vi.mock("../../globals.js", () => ({ - logVerbose: vi.fn(), -})); - -vi.mock("../../infra/agent-events.js", async () => { - const actual = await vi.importActual( - "../../infra/agent-events.js", - ); - const emitAgentEvent = vi.fn((...args: Parameters) => - actual.emitAgentEvent(...args), - ); - return { - ...actual, - clearAgentRunContext: vi.fn(), - emitAgentEvent, - registerAgentRunContext: vi.fn(), - }; -}); - -vi.mock("../../runtime.js", () => ({ - defaultRuntime: { - error: vi.fn(), - }, -})); - -vi.mock("../../utils/message-channel.js", () => ({ - isMarkdownCapableMessageChannel: () => true, - resolveMessageChannel: () => "whatsapp", - isInternalMessageChannel: (value: unknown) => state.isInternalMessageChannelMock(value), -})); - -vi.mock("../heartbeat.js", () => ({ - stripHeartbeatToken: (text: string) => ({ - text, - didStrip: false, - shouldSkip: false, - }), -})); - -vi.mock("./current-turn-images.js", () => ({ - resolveCurrentTurnImages: (params: unknown) => state.resolveCurrentTurnImagesMock(params), -})); - -vi.mock("./agent-runner-utils.js", () => ({ - buildEmbeddedRunExecutionParams: (params: { - provider: string; - model: string; - run: { - provider?: string; - thinkLevel?: string; - authProfileId?: string; - authProfileIdSource?: "auto" | "user"; - agentAccountId?: string; - chatType?: string; - }; - replyRoute?: { - originatingChannel?: string; - originatingTo?: string; - originatingAccountId?: string; - originatingChatType?: string; - }; - sessionCtx: { AccountId?: string; ChatType?: string }; - }) => ({ - embeddedContext: { - messageProvider: params.replyRoute?.originatingChannel, - messageTo: params.replyRoute?.originatingTo, - agentAccountId: - params.replyRoute?.originatingAccountId ?? - params.sessionCtx.AccountId ?? - params.run.agentAccountId, - chatType: - params.replyRoute?.originatingChatType ?? params.sessionCtx.ChatType ?? params.run.chatType, - }, - senderContext: {}, - runBaseParams: { - provider: params.provider, - model: params.model, - thinkLevel: params.run.thinkLevel, - authProfileId: params.provider === params.run.provider ? params.run.authProfileId : undefined, - authProfileIdSource: - params.provider === params.run.provider ? params.run.authProfileIdSource : undefined, - }, - }), - resolveQueuedReplyRuntimeConfig: (config: T) => config, - resolveModelFallbackOptions: vi.fn( - (run: { provider?: string; model?: string; config?: unknown; agentDir?: string }) => ({ - provider: run.provider, - model: run.model, - cfg: run.config, - agentDir: run.agentDir, - }), - ), - resolveRunFastModeForFallbackCandidate: (params: { - run: { fastMode?: unknown; fastModeAutoOnSeconds?: unknown }; - }) => ({ - fastMode: params.run.fastMode, - fastModeAutoOnSeconds: params.run.fastModeAutoOnSeconds, - }), -})); - -vi.mock("./reply-delivery.js", () => ({ - createBlockReplyDeliveryHandler: (params: unknown) => - state.createBlockReplyDeliveryHandlerMock(params), -})); - -vi.mock("./reply-media-paths.runtime.js", () => ({ - createReplyMediaContext: () => ({ - normalizePayload: (payload: unknown) => payload, - }), - createReplyMediaPathNormalizer: () => (payload: unknown) => payload, -})); - -async function getRunAgentTurnWithFallback() { - return (await import("./agent-runner-execution.js")).runAgentTurnWithFallback; -} - -type FallbackRunnerParams = { - provider: string; - model: string; - sessionId?: string; - abortSignal?: AbortSignal; - run: (provider: string, model: string) => Promise; - classifyResult?: (params: { - result: { payloads?: Array<{ text?: string; isError?: boolean; isReasoning?: boolean }> }; - provider: string; - model: string; - attempt: number; - total: number; - }) => Promise; -}; - -type EmbeddedAgentParams = { - lifecycleGeneration?: string; - onExecutionStarted?: (info?: { lifecycleGeneration?: string }) => void; - onExecutionPhase?: (info: { - phase: - | "runner_entered" - | "workspace" - | "runtime_plugins" - | "before_agent_reply" - | "model_resolution" - | "auth" - | "context_engine" - | "attempt_dispatch" - | "context_assembled" - | "turn_accepted" - | "process_spawned" - | "tool_execution_started" - | "assistant_output_started" - | "model_call_started"; - provider?: string; - model?: string; - backend?: string; - source?: string; - tool?: string; - toolCallId?: string; - itemId?: string; - }) => void; - onBlockReply?: (payload: { text?: string; mediaUrls?: string[] }) => Promise | void; - onPartialReply?: (payload: { text?: string; mediaUrls?: string[] }) => Promise | void; - onAssistantMessageStart?: () => Promise | void; - onToolResult?: (payload: { text?: string; mediaUrls?: string[] }) => Promise | void; - onReasoningStream?: (payload: { - text?: string; - mediaUrls?: string[]; - isReasoningSnapshot?: boolean; - requiresReasoningProgressOptIn?: boolean; - }) => Promise | void; - onReasoningEnd?: () => Promise | void; - onItemEvent?: (payload: { - itemId?: string; - toolCallId?: string; - kind?: string; - title?: string; - name?: string; - phase?: string; - status?: string; - summary?: string; - progressText?: string; - approvalId?: string; - approvalSlug?: string; - }) => Promise | void; - onAgentEvent?: (payload: { - stream: string; - data: Record; - sessionKey?: string; - }) => Promise | void; -}; - -function createMockTypingSignaler(): TypingSignaler { - return { - mode: "message", - shouldStartImmediately: false, - shouldStartOnMessageStart: true, - shouldStartOnText: true, - shouldStartOnReasoning: false, - signalRunStart: vi.fn(async () => {}), - signalMessageStart: vi.fn(async () => {}), - signalTextDelta: vi.fn(async () => {}), - signalReasoningDelta: vi.fn(async () => {}), - signalToolStart: vi.fn(async () => {}), - signalExecutionActivity: vi.fn(async () => {}), - }; -} - -function createFollowupRun(): FollowupRun { - return { - prompt: "hello", - summaryLine: "hello", - enqueuedAt: Date.now(), - run: { - agentId: "agent", - agentDir: "/tmp/agent", - sessionId: "session", - sessionKey: "main", - messageProvider: "whatsapp", - sessionFile: "/tmp/session.jsonl", - workspaceDir: "/tmp", - config: {}, - skillsSnapshot: {}, - provider: "anthropic", - model: "claude", - verboseLevel: "off", - elevatedLevel: "off", - bashElevated: { - enabled: false, - allowed: false, - defaultLevel: "off", - }, - timeoutMs: 1_000, - blockReplyBreak: "message_end", - }, - } as unknown as FollowupRun; -} - -function createTestUserTurnRecorder(message: PersistedUserTurnMessage) { - return createUserTurnTranscriptRecorder({ - message, - target: createTestUserTurnTranscriptTarget(), - updateMode: "none", - }); -} - -function createMockReplyOperation(): { - replyOperation: ReplyOperation; - failMock: ReturnType; - freezeAbortMock: ReturnType; - retainFailureUntilCompleteMock: ReturnType; - updateSessionIdMock: ReturnType; -} { - const failMock = vi.fn(); - const freezeAbortMock = vi.fn(); - const retainFailureUntilCompleteMock = vi.fn(); - const updateSessionIdMock = vi.fn(); - return { - failMock, - freezeAbortMock, - retainFailureUntilCompleteMock, - updateSessionIdMock, - replyOperation: { - key: "main", - sessionId: "session", - abortSignal: new AbortController().signal, - resetTriggered: false, - terminalRecovery: false, - acceptedSteeredInboundAudio: false, - phase: "running", - result: null, - startedAtMs: Date.now(), - lastActivityAtMs: Date.now(), - hasOwnedSessionId: vi.fn((sessionId: string) => sessionId === "session"), - recordActivity: vi.fn(), - setPhase: vi.fn(), - markWaitingForDeferredMaintenance: vi.fn(), - markDeferredMaintenanceWaitEnded: vi.fn(), - updateSessionId: updateSessionIdMock, - updateSessionKey: vi.fn(), - attachBackend: vi.fn(), - detachBackend: vi.fn(), - freezeAbort: freezeAbortMock, - retainFailureUntilComplete: retainFailureUntilCompleteMock, - complete: vi.fn(), - completeThen: vi.fn((afterClear: () => void) => afterClear()), - completeWithAfterClearBarrier: vi.fn(), - fail: failMock, - abortByUser: vi.fn(() => true), - abortForRestart: vi.fn(() => true), - markTerminalRecovery: vi.fn(), - markAcceptedSteeredInboundAudio: vi.fn(), - }, - }; -} - -function requireRecord(value: unknown, label: string): Record { - if (typeof value !== "object" || value === null) { - throw new Error(`${label} was not an object`); - } - return value as Record; -} - -function expectRecordFields(record: Record, fields: Record) { - for (const [key, value] of Object.entries(fields)) { - expect(record[key]).toEqual(value); - } -} - -function requireMockCall(mock: unknown, index: number, label: string): unknown[] { - const call = (mock as { mock?: { calls?: unknown[][] } }).mock?.calls?.[index]; - if (!call) { - throw new Error(`missing ${label} call ${index + 1}`); - } - return call; -} - -function expectMockCallArgFields( - mock: unknown, - index: number, - label: string, - fields: Record, -) { - expectRecordFields(requireRecord(requireMockCall(mock, index, label)[0], label), fields); -} - -function expectNoMockCallWithFields(mock: unknown, fields: Record) { - const calls = (mock as { mock?: { calls?: unknown[][] } }).mock?.calls ?? []; - const hasMatchingCall = calls.some((call) => { - const value = call[0]; - if (typeof value !== "object" || value === null) { - return false; - } - const record = value as Record; - return Object.entries(fields).every(([key, expected]) => record[key] === expected); - }); - expect(hasMatchingCall).toBe(false); -} - -function requireMockCallArgWithFields( - mock: unknown, - fields: Record, - label: string, -) { - const calls = (mock as { mock?: { calls?: unknown[][] } }).mock?.calls ?? []; - const found = calls - .map((call) => call[0]) - .find((value) => { - if (typeof value !== "object" || value === null) { - return false; - } - const record = value as Record; - return Object.entries(fields).every(([key, expected]) => record[key] === expected); - }); - if (!found) { - throw new Error(`missing ${label}`); - } - return requireRecord(found, label); -} - -function expectBlockReplyCall( - onBlockReply: unknown, - index: number, - fields: Record, -) { - expectMockCallArgFields(onBlockReply, index, "block reply payload", fields); -} - -function createMinimalRunAgentTurnParams(overrides?: { - followupRun?: FollowupRun; - opts?: GetReplyOptions & ReplyOptionsWithHeartbeatRunScope; - replyOperation?: ReplyOperation; - sessionCtx?: TemplateContext; - typingSignals?: TypingSignaler; -}) { - return { - commandBody: "fix it", - followupRun: overrides?.followupRun ?? createFollowupRun(), - sessionCtx: - overrides?.sessionCtx ?? - ({ - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext), - opts: overrides?.opts ?? ({} satisfies GetReplyOptions), - replyOperation: overrides?.replyOperation, - typingSignals: overrides?.typingSignals ?? createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end" as const, - applyReplyToMode: (payload: ReplyPayload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set>(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off" as const, - }; -} - -const NON_DIRECT_FAILURE_SURFACE_CASES = [ - { label: "Discord group", provider: "discord", chatType: "group" }, - { label: "Discord channel", provider: "discord", chatType: "channel" }, - { label: "Slack channel", provider: "slack", chatType: "channel" }, - { label: "Telegram group", provider: "telegram", chatType: "group" }, - { label: "WhatsApp group", provider: "whatsapp", chatType: "group" }, - { label: "Microsoft Teams channel", provider: "msteams", chatType: "channel" }, -] as const; - -function createNonDirectFailureSessionCtx( - testCase: (typeof NON_DIRECT_FAILURE_SURFACE_CASES)[number], -): TemplateContext { - return { - Provider: testCase.provider, - Surface: testCase.provider, - ChatType: testCase.chatType, - GroupSubject: `${testCase.label} fixture`, - GroupChannel: "#general", - MessageSid: "msg", - } as unknown as TemplateContext; -} - -describe("computeContextAwareReserveTokensFloor", () => { - it("returns 100000 for 1M context windows", () => { - expect(computeContextAwareReserveTokensFloor(1_000_000)).toBe(100_000); - }); - - it("returns 50000 for 200k context windows", () => { - expect(computeContextAwareReserveTokensFloor(200_000)).toBe(50_000); - }); - - it("returns 35000 for 100k context windows", () => { - expect(computeContextAwareReserveTokensFloor(100_000)).toBe(35_000); - }); - - it("returns 20000 for context windows below 100k", () => { - expect(computeContextAwareReserveTokensFloor(99_999)).toBe(20_000); - expect(computeContextAwareReserveTokensFloor(32_768)).toBe(20_000); - expect(computeContextAwareReserveTokensFloor(50_000)).toBe(20_000); - }); - - it("returns 20000 for undefined context window", () => { - expect(computeContextAwareReserveTokensFloor(undefined)).toBe(20_000); - }); - - it("returns 20000 for non-positive context window", () => { - expect(computeContextAwareReserveTokensFloor(0)).toBe(20_000); - expect(computeContextAwareReserveTokensFloor(-1)).toBe(20_000); - }); - - it("returns correct tiers at exact boundaries", () => { - expect(computeContextAwareReserveTokensFloor(100_000)).toBe(35_000); - expect(computeContextAwareReserveTokensFloor(200_000)).toBe(50_000); - expect(computeContextAwareReserveTokensFloor(1_000_000)).toBe(100_000); - expect(computeContextAwareReserveTokensFloor(99_999)).toBe(20_000); - expect(computeContextAwareReserveTokensFloor(199_999)).toBe(35_000); - expect(computeContextAwareReserveTokensFloor(999_999)).toBe(50_000); - }); -}); - -describe("buildEmptyInteractiveReplyPayload", () => { - const baseParams = { - isInteractive: true, - isMessageToolOnly: false, - hasPendingContinuation: false, - hasExplicitSilentReply: false, - hasCommittedDelivery: false, - sessionCtx: { - Provider: "discord", - Surface: "discord", - ChatType: "group", - }, - } as const; - - it("preserves the default silent policy in group conversations", () => { - const payload = buildEmptyInteractiveReplyPayload(baseParams); - - expect(payload?.text).toBe(SILENT_REPLY_TOKEN); - expect(payload?.isError).toBeUndefined(); - }); - - it("surfaces the fallback when group silence is explicitly disallowed", () => { - expect( - buildEmptyInteractiveReplyPayload({ - ...baseParams, - cfg: { agents: { defaults: { silentReply: { group: "disallow" } } } }, - }), - ).toMatchObject({ text: EMPTY_INTERACTIVE_REPLY_TEXT, isError: true }); - }); -}); - -describe("buildContextOverflowRecoveryText", () => { - it("keeps the generic compaction-buffer hint without heartbeat model evidence", () => { - const text = buildContextOverflowRecoveryText({ - cfg: {}, - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("20000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("suggests 100000 reserveTokensFloor for 1M context models", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("qwen3.6-plus", 1_000_000)], - }, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("100000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("suggests 50000 reserveTokensFloor for 200k context models", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("gpt-5.5-200k", 200_000)], - }, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "gpt-5.5-200k", - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("50000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("suggests 35000 reserveTokensFloor for 100k context models", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("gpt-5.5", 100_000)], - }, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "gpt-5.5", - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("35000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("suggests 20000 reserveTokensFloor for small context windows", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - ollama: { - baseUrl: "http://ollama.test", - models: [makeTestModel("qwen3.5-9b-32k:latest", 32_768)], - }, - }, - }, - }, - primaryProvider: "ollama", - primaryModel: "qwen3.5-9b-32k:latest", - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("20000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("uses session contextTokens as fallback when model metadata is unavailable", () => { - const text = buildContextOverflowRecoveryText({ - cfg: {}, - primaryProvider: "openrouter", - primaryModel: "unknown-model", - activeSessionEntry: { - sessionId: "session", - updatedAt: 1, - modelProvider: "openrouter", - model: "unknown-model", - contextTokens: 200_000, - }, - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("50000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("prefers model metadata over session contextTokens", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("qwen3.6-plus", 1_000_000)], - }, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - activeSessionEntry: { - sessionId: "session", - updatedAt: 1, - modelProvider: "openrouter", - model: "qwen3.6-plus", - contextTokens: 32_768, - }, - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("100000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("keeps the preserved-session copy with the existing overflow hint", () => { - const text = buildContextOverflowRecoveryText({ - preserveSessionMapping: true, - cfg: {}, - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - }); - - expect(text).toContain("kept this conversation mapped to the current session"); - expect(text).toContain("reserveTokensFloor"); - expect(text).not.toContain("reset our conversation"); - }); - - it("falls back to session entry model when runtimeProvider is not provided", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - ollama: { - baseUrl: "http://ollama.test", - models: [makeTestModel("qwen3.5-9b-32k:latest", 32_768)], - }, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "unknown-model", - activeSessionEntry: { - sessionId: "session", - updatedAt: 1, - modelProvider: "ollama", - model: "qwen3.5-9b-32k:latest", - contextTokens: 200_000, - }, - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("20000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("prefers session entry model context over session contextTokens numeric value", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - ollama: { - baseUrl: "http://ollama.test", - models: [makeTestModel("qwen3.5-9b-32k:latest", 32_768)], - }, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "unknown-model", - activeSessionEntry: { - sessionId: "session", - updatedAt: 1, - modelProvider: "ollama", - model: "qwen3.5-9b-32k:latest", - contextTokens: 1_000_000, - }, - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("20000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("uses session contextTokens before primary metadata for uncataloged runtime models", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("qwen3.6-plus", 1_000_000)], - }, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - activeSessionEntry: { - sessionId: "session", - updatedAt: 1, - modelProvider: "custom", - model: "uncataloged-32k", - contextTokens: 32_768, - }, - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("20000"); - expect(text).not.toContain("100000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("does not use primary metadata for explicit uncataloged runtime models", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("qwen3.6-plus", 1_000_000)], - }, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - runtimeProvider: "custom", - runtimeModel: "uncataloged-32k", - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("20000"); - expect(text).not.toContain("100000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("does not use stale session contextTokens for explicit uncataloged runtime models", () => { - const text = buildContextOverflowRecoveryText({ - cfg: {}, - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - runtimeProvider: "custom", - runtimeModel: "uncataloged-32k", - activeSessionEntry: { - sessionId: "session", - updatedAt: 1, - modelProvider: "openrouter", - model: "qwen3.6-plus", - contextTokens: 1_000_000, - }, - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("20000"); - expect(text).not.toContain("100000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("caps reserveTokensFloor hint by agent.defaults.contextTokens", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("qwen3.6-plus", 1_000_000)], - }, - }, - }, - agents: { - defaults: { - contextTokens: 100_000, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("35000"); - expect(text).not.toContain("100000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("caps reserveTokensFloor hint by per-agent contextTokens over defaults", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("qwen3.6-plus", 1_000_000)], - }, - }, - }, - agents: { - defaults: { - contextTokens: 200_000, - }, - list: [ - { - id: "capped-agent", - contextTokens: 32_768, - }, - ], - }, - }, - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - agentId: "capped-agent", - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("20000"); - expect(text).not.toContain("50000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("caps the session contextTokens fallback by agent contextTokens", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - agents: { - defaults: { - contextTokens: 200_000, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "unknown-model", - activeSessionEntry: { - sessionId: "session", - updatedAt: 1, - modelProvider: "openrouter", - model: "unknown-model", - contextTokens: 32_768, - }, - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("20000"); - expect(text).not.toContain("50000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("uses runtime model over primary model when both are available", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("qwen3.6-plus", 1_000_000)], - }, - ollama: { - baseUrl: "http://ollama.test", - models: [makeTestModel("qwen3.5-9b-32k:latest", 32_768)], - }, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - runtimeProvider: "ollama", - runtimeModel: "qwen3.5-9b-32k:latest", - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("20000"); - expect(text).not.toContain("100000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("uses runtime model with 200k context when primary is 1M", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("qwen3.6-plus", 1_000_000)], - }, - openai: { - baseUrl: "https://openai.test", - models: [makeTestModel("gpt-5.5-200k", 200_000)], - }, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - runtimeProvider: "openai", - runtimeModel: "gpt-5.5-200k", - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("50000"); - expect(text).not.toContain("100000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("does not use stale heartbeat bleed hints for different explicit runtime refs", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - agents: { - defaults: { - heartbeat: { model: "ollama/qwen3.5-9b-32k:latest" }, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - runtimeProvider: "custom", - runtimeModel: "uncataloged-32k", - activeSessionEntry: { - sessionId: "session", - updatedAt: 1, - modelProvider: "ollama", - model: "qwen3.5-9b-32k:latest", - contextTokens: 32_768, - }, - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).toContain("20000"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("points to heartbeat model bleed when the last runtime model matches configured heartbeat.model", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("qwen3.6-plus", 1_000_000)], - }, - ollama: { - baseUrl: "http://ollama.test", - models: [makeTestModel("qwen3.5-9b-32k:latest", 32_768)], - }, - }, - }, - agents: { - defaults: { - heartbeat: { model: "ollama/qwen3.5-9b-32k:latest" }, - }, - }, - }, - agentId: "agent", - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - activeSessionEntry: { - sessionId: "session", - updatedAt: 1, - modelProvider: "ollama", - model: "qwen3.5-9b-32k:latest", - contextTokens: 32_768, - }, - }); - - expect(text).toContain("ollama/qwen3.5-9b-32k:latest (32k context)"); - expect(text).toContain("openrouter/qwen3.6-plus"); - expect(text).toContain("heartbeat model bleed"); - expect(text).toContain("heartbeat.isolatedSession"); - expect(text).not.toContain("reserveTokensFloor"); - }); - - it("uses the stored session context window as the uncataloged runtime model fallback", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("qwen3.6-plus", 1_000_000)], - }, - }, - }, - agents: { - defaults: { - contextTokens: 100_000, - heartbeat: { model: "ollama/custom-32k" }, - }, - }, - }, - agentId: "agent", - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - activeSessionEntry: { - sessionId: "session", - updatedAt: 1, - modelProvider: "ollama", - model: "custom-32k", - contextTokens: 32_768, - }, - }); - - expect(text).toContain("ollama/custom-32k (32k context)"); - expect(text).not.toContain("ollama/custom-32k (98k context)"); - expect(text).toContain("heartbeat model bleed"); - }); - - it("does not blame heartbeat when the stored session fallback matches the capped primary window", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("qwen3.6-plus", 1_000_000)], - }, - }, - }, - agents: { - defaults: { - contextTokens: 100_000, - heartbeat: { model: "ollama/custom-large" }, - }, - }, - }, - agentId: "agent", - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - activeSessionEntry: { - sessionId: "session", - updatedAt: 1, - modelProvider: "ollama", - model: "custom-large", - contextTokens: 200_000, - }, - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("does not blame heartbeat when the same agent cap constrains both cataloged models", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("qwen3.6-plus", 1_000_000)], - }, - ollama: { - baseUrl: "http://ollama.test", - models: [makeTestModel("custom-large", 1_000_000)], - }, - }, - }, - agents: { - defaults: { - contextTokens: 100_000, - heartbeat: { model: "ollama/custom-large" }, - }, - }, - }, - agentId: "agent", - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - activeSessionEntry: { - sessionId: "session", - updatedAt: 1, - modelProvider: "ollama", - model: "custom-large", - contextTokens: 1_000_000, - }, - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).not.toContain("heartbeat model bleed"); - }); - - it("does not blame heartbeat when the smaller runtime model is not the configured heartbeat model", () => { - const text = buildContextOverflowRecoveryText({ - cfg: { - agents: { - defaults: { - heartbeat: { model: "ollama/qwen3.5-9b-32k:latest" }, - }, - }, - }, - primaryProvider: "openrouter", - primaryModel: "qwen3.6-plus", - activeSessionEntry: { - sessionId: "session", - updatedAt: 1, - modelProvider: "anthropic", - model: "claude-haiku-4-5", - contextTokens: 32_768, - }, - }); - - expect(text).toContain("reserveTokensFloor"); - expect(text).not.toContain("heartbeat model bleed"); - }); -}); - -describe("runAgentTurnWithFallback", () => { - beforeEach(() => { - vi.useRealTimers(); - state.runEmbeddedAgentMock.mockReset(); - state.runCliAgentMock.mockReset(); - state.runWithModelFallbackMock.mockReset(); - state.isCliProviderMock.mockReset(); - state.isCliProviderMock.mockReturnValue(false); - state.isInternalMessageChannelMock.mockReset(); - state.isInternalMessageChannelMock.mockReturnValue(false); - state.createBlockReplyDeliveryHandlerMock.mockReset(); - state.createBlockReplyDeliveryHandlerMock.mockReturnValue(undefined); - state.isCompactionFailureErrorMock.mockReset(); - state.isCompactionFailureErrorMock.mockReturnValue(false); - state.isContextOverflowErrorMock.mockReset(); - state.isContextOverflowErrorMock.mockReturnValue(false); - state.isLikelyContextOverflowErrorMock.mockReset(); - state.isLikelyContextOverflowErrorMock.mockReturnValue(false); - state.updateSessionStoreMock.mockReset(); - state.resolveCurrentTurnImagesMock.mockReset(); - state.resolveCurrentTurnImagesMock.mockImplementation( - async (params: { images?: unknown[]; imageOrder?: unknown[] }) => ({ - images: params.images, - imageOrder: params.imageOrder, - }), - ); - state.runWithModelFallbackMock.mockImplementation(async (params: FallbackRunnerParams) => ({ - result: await params.run("anthropic", "claude"), - provider: "anthropic", - model: "claude", - attempts: [], - })); - }); - - afterEach(() => { - // Fake-timer tests in this describe must not leak into --isolate=false peers. - vi.useRealTimers(); - cliBackendsTesting.resetDepsForTest(); - vi.clearAllMocks(); - }); - - it("passes the reply abort signal to fallback orchestration and candidates", async () => { - const { replyOperation } = createMockReplyOperation(); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "ok" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams(), - replyOperation, - }); - - const fallbackCall = requireRecord( - state.runWithModelFallbackMock.mock.calls[0]?.[0], - "runWithModelFallback params", - ); - const embeddedCall = requireRecord( - state.runEmbeddedAgentMock.mock.calls[0]?.[0], - "runEmbeddedAgent params", - ); - expect(fallbackCall.abortSignal).toBe(replyOperation.abortSignal); - expect(fallbackCall.sessionId).toBe("session"); - expect(embeddedCall.abortSignal).toBe(replyOperation.abortSignal); - }); - - it("revalidates thinking for each main-chat fallback candidate without mutating the run", async () => { - const followupRun = createFollowupRun(); - followupRun.run.provider = "openai"; - followupRun.run.model = "gpt-5.6-sol"; - followupRun.run.thinkLevel = "ultra"; - followupRun.run.config = { - agents: { - defaults: { - models: { - "openai/gpt-5.6-sol": { agentRuntime: { id: "openclaw" } }, - "demo/basic": { agentRuntime: { id: "openclaw" } }, - }, - }, - }, - }; - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - await params.run("openai", "gpt-5.6-sol"); - const result = await params.run("demo", "basic"); - return { result, provider: "demo", model: "basic", attempts: [] }; - }); - state.runEmbeddedAgentMock.mockResolvedValue({ payloads: [{ text: "ok" }], meta: {} }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - }); - - expect(state.runEmbeddedAgentMock.mock.calls.map((call) => call[0]?.thinkLevel)).toEqual([ - "ultra", - "high", - ]); - expect(followupRun.run.thinkLevel).toBe("ultra"); - }); - - it("freezes abort ownership only after model fallback settles", async () => { - const { replyOperation, freezeAbortMock } = createMockReplyOperation(); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - expect(freezeAbortMock).not.toHaveBeenCalled(); - await params.run("anthropic", "claude").catch(() => undefined); - expect(freezeAbortMock).not.toHaveBeenCalled(); - const result = await params.run("openai", "gpt-5.5"); - expect(freezeAbortMock).not.toHaveBeenCalled(); - return { - result, - provider: "openai", - model: "gpt-5.5", - attempts: [], - }; - }); - state.runEmbeddedAgentMock - .mockRejectedValueOnce(new Error("primary failed")) - .mockResolvedValueOnce({ - payloads: [{ text: "ok" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams(), - replyOperation, - }); - - expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(2); - expect(freezeAbortMock).toHaveBeenCalledTimes(1); - }); - - it("suppresses a settled fallback result after an accepted user abort", async () => { - const { replyOperation, freezeAbortMock } = createMockReplyOperation(); - const abortController = new AbortController(); - let operationResult: ReplyOperation["result"] = null; - let releaseFallback: () => void = () => undefined; - let markCandidateSettled: () => void = () => undefined; - const candidateSettled = new Promise((resolve) => { - markCandidateSettled = resolve; - }); - const fallbackRelease = new Promise((resolve) => { - releaseFallback = resolve; - }); - let releaseToolTask: () => void = () => undefined; - const pendingToolTask = new Promise((resolve) => { - releaseToolTask = resolve; - }); - const pendingToolTasks = new Set([pendingToolTask]); - Object.defineProperty(replyOperation, "abortSignal", { - configurable: true, - get: () => abortController.signal, - }); - Object.defineProperty(replyOperation, "result", { - configurable: true, - get: () => operationResult, - }); - replyOperation.abortByUser = vi.fn(() => { - operationResult = { kind: "aborted", code: "aborted_by_user" }; - abortController.abort("user_abort"); - return true; - }); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "late reply" }], - meta: {}, - }); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - const result = await params.run("anthropic", "claude"); - markCandidateSettled(); - await fallbackRelease; - return { - result, - provider: "anthropic", - model: "claude", - attempts: [], - }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const pending = runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams(), - replyOperation, - pendingToolTasks, - }); - await candidateSettled; - expect(replyOperation.abortByUser()).toBe(true); - let settled = false; - void pending.then(() => { - settled = true; - }); - releaseFallback(); - await new Promise((resolve) => { - setImmediate(resolve); - }); - expect(settled).toBe(false); - expect(freezeAbortMock).not.toHaveBeenCalled(); - releaseToolTask(); - - await expect(pending).resolves.toEqual({ - kind: "final", - payload: { text: SILENT_REPLY_TOKEN }, - }); - expect(freezeAbortMock).toHaveBeenCalledTimes(1); - }); - - it("suppresses a settled fallback result after an upstream abort", async () => { - const upstreamAbort = new AbortController(); - const replyOperation = createReplyOperation({ - sessionKey: "agent:main:upstream-settled-fallback", - sessionId: "upstream-settled-fallback", - resetTriggered: false, - upstreamAbortSignal: upstreamAbort.signal, - }); - replyOperation.setPhase("running"); - let releaseFallback: () => void = () => undefined; - let markCandidateSettled: () => void = () => undefined; - const candidateSettled = new Promise((resolve) => { - markCandidateSettled = resolve; - }); - const fallbackRelease = new Promise((resolve) => { - releaseFallback = resolve; - }); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "late reply" }], - meta: {}, - }); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - const result = await params.run("anthropic", "claude"); - markCandidateSettled(); - await fallbackRelease; - return { - result, - provider: "anthropic", - model: "claude", - attempts: [], - }; - }); - - try { - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const pending = runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams(), - replyOperation, - }); - await candidateSettled; - upstreamAbort.abort(new Error("caller cancelled")); - expect(replyOperation.result).toEqual({ kind: "aborted", code: "aborted_by_user" }); - expect(replyOperation.abortSignal.aborted).toBe(true); - releaseFallback(); - - await expect(pending).resolves.toEqual({ - kind: "final", - payload: { text: SILENT_REPLY_TOKEN }, - }); - } finally { - replyOperation.complete(); - } - }); - - it("preserves restart reply classification after an upstream abort settles fallback", async () => { - const upstreamAbort = new AbortController(); - const replyOperation = createReplyOperation({ - sessionKey: "agent:main:upstream-settled-restart", - sessionId: "upstream-settled-restart", - resetTriggered: false, - upstreamAbortSignal: upstreamAbort.signal, - }); - replyOperation.setPhase("running"); - let releaseFallback: () => void = () => undefined; - let markCandidateSettled: () => void = () => undefined; - const candidateSettled = new Promise((resolve) => { - markCandidateSettled = resolve; - }); - const fallbackRelease = new Promise((resolve) => { - releaseFallback = resolve; - }); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "late reply" }], - meta: {}, - }); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - const result = await params.run("anthropic", "claude"); - markCandidateSettled(); - await fallbackRelease; - return { - result, - provider: "anthropic", - model: "claude", - attempts: [], - }; - }); - - try { - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const pending = runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams(), - replyOperation, - }); - await candidateSettled; - upstreamAbort.abort(createAgentRunRestartAbortError()); - releaseFallback(); - - await expect(pending).resolves.toEqual({ - kind: "final", - payload: { - isError: true, - text: "⚠️ Gateway is restarting. Please wait a few seconds and try again.", - }, - }); - } finally { - replyOperation.complete(); - } - }); - - it("passes the hydrated run account to embedded execution", async () => { - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "ok" }], - meta: {}, - }); - const followupRun = createFollowupRun(); - followupRun.run.agentAccountId = "work"; - followupRun.originatingChannel = "slack"; - followupRun.originatingTo = "user:U1"; - followupRun.originatingAccountId = "work"; - followupRun.originatingChatType = "direct"; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - followupRun, - sessionCtx: { - Provider: "cron-event", - }, - }), - ); - - expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { - messageProvider: "slack", - messageTo: "user:U1", - agentAccountId: "work", - chatType: "direct", - }); - }); - - it("signals typing from embedded harness execution phases before assistant text", async () => { - const typingSignals = createMockTypingSignaler(); - const onAgentRunStart = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - params.onExecutionPhase?.({ - phase: "model_call_started", - provider: "openai", - model: "gpt-5.4", - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ - opts: { - onAgentRunStart, - } satisfies GetReplyOptions, - }), - typingSignals, - }); - - expect(result.kind).toBe("success"); - expect(typingSignals.signalExecutionActivity).toHaveBeenCalledOnce(); - expect(typingSignals.signalRunStart).not.toHaveBeenCalled(); - expect(onAgentRunStart).toHaveBeenCalledOnce(); - }); - - it("forwards CLI harness execution phases into typing signals", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("codex-cli", "gpt-5.4"), - provider: "codex-cli", - model: "gpt-5.4", - attempts: [], - })); - state.runCliAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - params.onExecutionPhase?.({ - phase: "process_spawned", - provider: "codex-cli", - model: "gpt-5.4", - backend: "codex", - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - const followupRun = createFollowupRun(); - followupRun.run.provider = "codex-cli"; - followupRun.run.model = "gpt-5.4"; - followupRun.run.clientCaps = ["tool-events", "inline-widgets"]; - const typingSignals = createMockTypingSignaler(); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - followupRun, - typingSignals, - }), - ); - - expect(result.kind).toBe("success"); - expect(typingSignals.signalExecutionActivity).toHaveBeenCalledOnce(); - expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { - provider: "codex-cli", - model: "gpt-5.4", - clientCaps: ["tool-events", "inline-widgets"], - }); - }); - - it("propagates commitment-only bootstrap scope to CLI runs", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("claude-cli", "sonnet-4.6"), - provider: "claude-cli", - model: "sonnet-4.6", - attempts: [], - })); - state.runCliAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "final" }], - meta: {}, - }); - const followupRun = createFollowupRun(); - followupRun.run.provider = "claude-cli"; - followupRun.run.model = "sonnet-4.6"; - const params = createMinimalRunAgentTurnParams({ - followupRun, - opts: { - isHeartbeat: true, - bootstrapContextMode: "lightweight", - [HEARTBEAT_RUN_SCOPE]: "commitment-only", - }, - }); - params.isHeartbeat = true; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback(params); - - expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { - trigger: "heartbeat", - bootstrapContextMode: "lightweight", - bootstrapContextRunKind: "commitment-only", - }); - }); - - it("registers run ownership before asynchronous image preflight", async () => { - const agentEvents = await import("../../infra/agent-events.js"); - const registerAgentRunContext = vi.mocked(agentEvents.registerAgentRunContext); - let resolveImages: (() => void) | undefined; - state.resolveCurrentTurnImagesMock.mockImplementationOnce( - () => - new Promise>((resolve) => { - resolveImages = () => resolve({}); - }), - ); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "ok" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const runPromise = runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(registerAgentRunContext).toHaveBeenCalledWith( - expect.any(String), - expect.objectContaining({ - sessionKey: "main", - sessionId: "session", - }), - ); - expect(state.runWithModelFallbackMock).not.toHaveBeenCalled(); - - resolveImages?.(); - await runPromise; - }); - - it("clears run ownership when image preflight fails", async () => { - const agentEvents = await import("../../infra/agent-events.js"); - const clearAgentRunContext = vi.mocked(agentEvents.clearAgentRunContext); - state.resolveCurrentTurnImagesMock.mockRejectedValueOnce(new Error("invalid image metadata")); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await expect( - runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - opts: { runId: "preflight-failure" }, - }), - ), - ).rejects.toThrow("invalid image metadata"); - - expect(clearAgentRunContext).toHaveBeenCalledWith("preflight-failure", expect.any(String)); - expect(state.runWithModelFallbackMock).not.toHaveBeenCalled(); - }); - - it("passes runtime toolsAllow to embedded agent runs", async () => { - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "ok" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - opts: { - toolsAllow: ["message"], - }, - }), - ); - - expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { - toolsAllow: ["message"], - }); - }); - - it("rechecks queued auto fallback primary probes before running", async () => { - const { markAutoFallbackPrimaryProbe } = await import("../../agents/agent-scope.js"); - const probe = { - provider: "anthropic", - model: "claude-sonnet-4-6", - fallbackProvider: "google", - fallbackModel: "gemini-3-pro", - fallbackAuthProfileId: "google:fallback", - fallbackAuthProfileIdSource: "auto" as const, - }; - markAutoFallbackPrimaryProbe({ - probe, - sessionKey: "main", - now: Date.now(), - }); - const sessionEntry: SessionEntry = { - sessionId: "session", - updatedAt: 1, - providerOverride: "google", - modelOverride: "gemini-3-pro", - modelOverrideSource: "auto", - modelOverrideFallbackOriginProvider: "anthropic", - modelOverrideFallbackOriginModel: "claude-sonnet-4-6", - authProfileOverride: "google:fallback", - authProfileOverrideSource: "auto", - }; - const run = createFollowupRun().run; - run.provider = "anthropic"; - run.model = "claude-sonnet-4-6"; - run.authProfileId = "anthropic:primary"; - run.authProfileIdSource = "auto"; - run.autoFallbackPrimaryProbe = probe; - - expect( - resolveRunAfterAutoFallbackPrimaryProbeRecheck({ - run, - entry: sessionEntry, - sessionKey: "main", - }), - ).toMatchObject({ - provider: "google", - model: "gemini-3.1-pro-preview", - authProfileId: "google:fallback", - authProfileIdSource: "auto", - autoFallbackPrimaryProbe: undefined, - }); - }); - - it("drops stale queued primary probes after a user model switch", async () => { - const probe = { - provider: "anthropic", - model: "claude-sonnet-4-6", - fallbackProvider: "google", - fallbackModel: "gemini-3-pro", - }; - const sessionEntry: SessionEntry = { - sessionId: "session", - updatedAt: 1, - modelOverride: "openai/gpt-5.4", - modelOverrideSource: "user", - authProfileOverride: "openai:work", - authProfileOverrideSource: "user", - }; - const run = createFollowupRun().run; - run.provider = "anthropic"; - run.model = "claude-sonnet-4-6"; - run.autoFallbackPrimaryProbe = probe; - - expect( - resolveRunAfterAutoFallbackPrimaryProbeRecheck({ - run, - entry: sessionEntry, - sessionKey: "main", - }), - ).toMatchObject({ - provider: "openai", - model: "gpt-5.4", - authProfileId: "openai:work", - authProfileIdSource: "user", - modelOverrideSource: "user", - autoFallbackPrimaryProbe: undefined, - }); - }); - - it("propagates rechecked user selections to post-run state", async () => { - const sessionKey = "rechecked-user-selection"; - const sessionEntry: SessionEntry = { - sessionId: "session", - updatedAt: 1, - providerOverride: "openai", - modelOverride: "gpt-5.4", - modelOverrideSource: "user", - authProfileOverride: "openai:work", - authProfileOverrideSource: "user", - }; - const activeSessionStore = { [sessionKey]: sessionEntry }; - const staleAutoEntry: SessionEntry = { - sessionId: "session", - updatedAt: 1, - providerOverride: "google", - modelOverride: "gemini-3-pro", - modelOverrideSource: "auto", - modelOverrideFallbackOriginProvider: "anthropic", - modelOverrideFallbackOriginModel: "claude-sonnet-4-6", - }; - const followupRun = createFollowupRun(); - followupRun.run.provider = "anthropic"; - followupRun.run.model = "claude-sonnet-4-6"; - followupRun.run.autoFallbackPrimaryProbe = { - provider: "anthropic", - model: "claude-sonnet-4-6", - fallbackProvider: "google", - fallbackModel: "gemini-3-pro", - }; - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run(params.provider, params.model), - provider: params.provider, - model: params.model, - attempts: [], - })); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "user model" }], - meta: { - agentMeta: { - provider: "openai", - model: "gpt-5.4", - }, - }, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - sessionKey, - activeSessionStore, - getActiveSessionEntry: () => staleAutoEntry, - }); - - expectRecordFields(followupRun.run as unknown as Record, { - provider: "openai", - model: "gpt-5.4", - authProfileId: "openai:work", - authProfileIdSource: "user", - modelOverrideSource: "user", - }); - expect(followupRun.run.autoFallbackPrimaryProbe).toBeUndefined(); - expectRecordFields(activeSessionStore[sessionKey] as unknown as Record, { - providerOverride: "openai", - modelOverride: "gpt-5.4", - modelOverrideSource: "user", - }); - }); - - it("drops stale queued probe metadata after the auto fallback pin is cleared", () => { - const probe = { - provider: "anthropic", - model: "claude-sonnet-4-6", - fallbackProvider: "google", - fallbackModel: "gemini-3-pro", - }; - const sessionEntry: SessionEntry = { - sessionId: "session", - updatedAt: 1, - authProfileOverride: "google:fallback", - authProfileOverrideSource: "user", - }; - const run = createFollowupRun().run; - run.provider = "anthropic"; - run.model = "claude-sonnet-4-6"; - run.hasSessionModelOverride = true; - run.modelOverrideSource = "auto"; - run.hasAutoFallbackProvenance = true; - run.autoFallbackPrimaryProbe = probe; - - expect( - resolveRunAfterAutoFallbackPrimaryProbeRecheck({ - run, - entry: sessionEntry, - sessionKey: "main", - }), - ).toMatchObject({ - provider: "anthropic", - model: "claude-sonnet-4-6", - autoFallbackPrimaryProbe: undefined, - }); - const rechecked = resolveRunAfterAutoFallbackPrimaryProbeRecheck({ - run, - entry: sessionEntry, - sessionKey: "main", - }); - expect(rechecked.authProfileId).toBeUndefined(); - expect(rechecked.authProfileIdSource).toBeUndefined(); - expect(rechecked.hasSessionModelOverride).toBeUndefined(); - expect(rechecked.modelOverrideSource).toBeUndefined(); - expect(rechecked.hasAutoFallbackProvenance).toBeUndefined(); - }); - - it("keeps fallback auth available when a primary probe falls back", () => { - const probe = { - provider: "anthropic", - model: "claude-sonnet-4-6", - fallbackProvider: "google", - fallbackModel: "gemini-3-pro", - fallbackAuthProfileId: "google:fallback", - fallbackAuthProfileIdSource: "auto" as const, - }; - const followupRun = createFollowupRun(); - followupRun.run.provider = "anthropic"; - followupRun.run.model = "claude-sonnet-4-6"; - followupRun.run.authProfileId = "anthropic:primary"; - followupRun.run.authProfileIdSource = "auto"; - followupRun.run.autoFallbackPrimaryProbe = probe; - expect(resolveFallbackCandidateRun(followupRun.run, "google", "gemini-3-pro")).toMatchObject({ - provider: "google", - model: "gemini-3-pro", - authProfileId: "google:fallback", - authProfileIdSource: "auto", - }); - }); - - it("does not clear an auto-fallback pin for an exhausted preserved result", async () => { - const probe = { - provider: "anthropic", - model: "claude-sonnet-4-6", - fallbackProvider: "google", - fallbackModel: "gemini-3-pro", - fallbackAuthProfileId: "google:fallback", - fallbackAuthProfileIdSource: "auto" as const, - }; - const followupRun = createFollowupRun(); - followupRun.run.provider = probe.provider; - followupRun.run.model = probe.model; - followupRun.run.autoFallbackPrimaryProbe = probe; - const sessionKey = "exhausted-primary-probe"; - const sessionEntry: SessionEntry = { - sessionId: "session", - updatedAt: 1, - providerOverride: probe.fallbackProvider, - modelOverride: probe.fallbackModel, - modelOverrideSource: "auto", - modelOverrideFallbackOriginProvider: probe.provider, - modelOverrideFallbackOriginModel: probe.model, - authProfileOverride: probe.fallbackAuthProfileId, - authProfileOverrideSource: "auto", - }; - const activeSessionStore: Record = { [sessionKey]: sessionEntry }; - const exhaustedResult = { - payloads: [{ text: "Terminal tool summary", isError: true }], - meta: { - error: { - kind: "incomplete_turn", - message: "All fallback candidates ended incomplete", - fallbackSafe: true, - terminalPresentation: true, - }, - }, - }; - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "lifecycle", - data: { - phase: "finishing", - error: "All fallback candidates ended incomplete", - livenessState: "blocked", - providerStarted: true, - replayInvalid: true, - timeoutPhase: "provider", - }, - }); - return exhaustedResult; - }); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - outcome: "exhausted", - result: await params.run(probe.provider, probe.model), - provider: probe.provider, - model: probe.model, - attempts: [ - { provider: probe.provider, model: probe.model, error: "incomplete" }, - { - provider: probe.fallbackProvider, - model: probe.fallbackModel, - error: "incomplete", - }, - ], - })); - const { replyOperation, failMock, retainFailureUntilCompleteMock } = createMockReplyOperation(); - const emitAgentEvent = vi.mocked((await import("../../infra/agent-events.js")).emitAgentEvent); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun, replyOperation }), - sessionKey, - activeSessionStore, - getActiveSessionEntry: () => activeSessionStore[sessionKey], - }); - - expect(result).toMatchObject({ - kind: "success", - fallbackExhausted: true, - fallbackProvider: probe.provider, - fallbackModel: probe.model, - runResult: exhaustedResult, - }); - expect(activeSessionStore[sessionKey]).toMatchObject({ - providerOverride: probe.fallbackProvider, - modelOverride: probe.fallbackModel, - modelOverrideSource: "auto", - modelOverrideFallbackOriginProvider: probe.provider, - modelOverrideFallbackOriginModel: probe.model, - }); - expect(retainFailureUntilCompleteMock).toHaveBeenCalledTimes(1); - expect(failMock).toHaveBeenCalledWith("run_failed", expect.any(Error)); - expect( - emitAgentEvent.mock.calls - .map((call) => call[0]) - .find( - (event) => - event.stream === "lifecycle" && - event.data.phase === "error" && - event.data.fallbackExhaustedFailure === true && - event.data.livenessState === "blocked" && - event.data.providerStarted === true && - event.data.replayInvalid === true && - event.data.timeoutPhase === "provider", - ), - ).toBeDefined(); - }); - - it("reports a completed non-fallbackable error result as a failure terminal", async () => { - const terminalErrorResult = { - payloads: [{ text: "Command may have changed state", isError: true }], - meta: { - replayInvalid: true, - error: { - kind: "incomplete_turn", - message: "raw provider detail should stay private", - fallbackSafe: false, - }, - }, - }; - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "lifecycle", - data: { - phase: "finishing", - error: "Command may have changed state", - replayInvalid: true, - }, - }); - return terminalErrorResult; - }); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - outcome: "completed", - result: await params.run("anthropic", "claude"), - provider: "anthropic", - model: "claude", - attempts: [], - })); - const { replyOperation, failMock, retainFailureUntilCompleteMock } = createMockReplyOperation(); - const emitAgentEvent = vi.mocked((await import("../../infra/agent-events.js")).emitAgentEvent); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - replyOperation, - opts: { runId: "run-non-fallbackable-error" }, - }), - ); - - expect(result).toMatchObject({ kind: "success", runResult: terminalErrorResult }); - expect(retainFailureUntilCompleteMock).toHaveBeenCalledTimes(1); - expect(failMock).toHaveBeenCalledWith("run_failed", expect.any(Error)); - const lifecycleEvents = emitAgentEvent.mock.calls - .map((call) => call[0]) - .filter( - (event) => event.runId === "run-non-fallbackable-error" && event.stream === "lifecycle", - ); - expect(lifecycleEvents).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - phase: "error", - error: "Command may have changed state", - replayInvalid: true, - }), - }), - ]), - ); - expect( - lifecycleEvents.some( - (event) => event.data.phase === "end" || event.data.fallbackExhaustedFailure === true, - ), - ).toBe(false); - expect(JSON.stringify(lifecycleEvents)).not.toContain("raw provider detail"); - }); - - it.each([ - { - label: "exhausted", - outcome: "exhausted" as const, - attempts: [{ error: "missing tool result" }], - isHeartbeat: false, - expectedText: GENERIC_RUN_FAILURE_TEXT, - }, - { - label: "completed", - outcome: "completed" as const, - attempts: [], - isHeartbeat: false, - expectedText: GENERIC_RUN_FAILURE_TEXT, - }, - { - label: "heartbeat", - outcome: "completed" as const, - attempts: [], - isHeartbeat: true, - expectedText: HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT, - }, - ])("surfaces an empty $label terminal result through the normal reply path", async (testCase) => { - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [], - meta: { - error: { - kind: "tool_result_mismatch", - message: "Agent run reached a terminal error before reply delivery.", - }, - }, - }); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - outcome: testCase.outcome, - result: await params.run("anthropic", "claude"), - provider: "anthropic", - model: "claude", - attempts: testCase.attempts, - })); - const { replyOperation, failMock } = createMockReplyOperation(); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ replyOperation }), - isHeartbeat: testCase.isHeartbeat, - }); - - expect(result).toMatchObject({ - kind: "success", - terminalFailurePayload: { - text: testCase.expectedText, - isError: true, - }, - runResult: { - payloads: [], - }, - }); - expect(failMock).toHaveBeenCalledWith("run_failed", expect.any(Error)); - }); - - it("reports exhausted CLI results without a success lifecycle terminal", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - outcome: "exhausted", - result: await params.run("codex-cli", "gpt-5.4"), - provider: "codex-cli", - model: "gpt-5.4", - attempts: [{ provider: "codex-cli", model: "gpt-5.4", error: "incomplete" }], - })); - state.runCliAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "Terminal tool summary", isError: true }], - meta: { - error: { - kind: "incomplete_turn", - message: "CLI turn ended incomplete", - }, - }, - }); - const followupRun = createFollowupRun(); - followupRun.run.provider = "codex-cli"; - followupRun.run.model = "gpt-5.4"; - const { replyOperation, failMock, retainFailureUntilCompleteMock } = createMockReplyOperation(); - const emitAgentEvent = vi.mocked((await import("../../infra/agent-events.js")).emitAgentEvent); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - followupRun, - replyOperation, - opts: { runId: "run-cli-exhausted" }, - }), - ); - - expect(result).toMatchObject({ - kind: "success", - fallbackExhausted: true, - fallbackProvider: "codex-cli", - fallbackModel: "gpt-5.4", - }); - expect(retainFailureUntilCompleteMock).toHaveBeenCalledTimes(1); - expect(failMock).toHaveBeenCalledWith("run_failed", expect.any(Error)); - const lifecycleEvents = emitAgentEvent.mock.calls - .map((call) => call[0]) - .filter((event) => event.runId === "run-cli-exhausted" && event.stream === "lifecycle"); - expect(lifecycleEvents).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - phase: "error", - fallbackExhaustedFailure: true, - }), - }), - ]), - ); - expect(lifecycleEvents.some((event) => event.data.phase === "end")).toBe(false); - }); - - it("preserves a CLI watchdog timeout through the lifecycle backstop", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - try { - return await params.run("codex-cli", "gpt-5.4"); - } catch (cause) { - throw new Error("All model fallback candidates failed", { cause }); - } - }); - state.runCliAgentMock.mockRejectedValueOnce( - new FailoverError("CLI produced no output", { reason: "timeout" }), - ); - const followupRun = createFollowupRun(); - followupRun.run.provider = "codex-cli"; - followupRun.run.model = "gpt-5.4"; - const emitAgentEvent = vi.mocked((await import("../../infra/agent-events.js")).emitAgentEvent); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - followupRun, - opts: { runId: "run-cli-timeout" }, - }), - ); - - expect( - emitAgentEvent.mock.calls - .map((call) => call[0]) - .find( - (event) => - event.runId === "run-cli-timeout" && - event.stream === "lifecycle" && - event.data.phase === "error", - )?.data, - ).toMatchObject({ - stopReason: "timeout", - timeoutPhase: "provider", - fallbackExhaustedFailure: true, - }); - }); - - it("keeps primary auth on same-provider primary probes", async () => { - const probe = { - provider: "openai", - model: "gpt-5.5", - fallbackProvider: "openai", - fallbackModel: "gpt-5.4", - fallbackAuthProfileId: "openai:fallback", - fallbackAuthProfileIdSource: "auto" as const, - }; - const followupRun = createFollowupRun(); - followupRun.run.provider = "openai"; - followupRun.run.model = "gpt-5.5"; - followupRun.run.authProfileId = "openai:primary"; - followupRun.run.authProfileIdSource = "auto"; - followupRun.run.autoFallbackPrimaryProbe = probe; - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - await params.run("openai", "gpt-5.5"); - return { - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [{ provider: "openai", model: "gpt-5.5", error: "rate limit" }], - }; - }); - state.runEmbeddedAgentMock - .mockResolvedValueOnce({ payloads: [], meta: {} }) - .mockResolvedValueOnce({ payloads: [{ text: "fallback" }], meta: {} }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback(createMinimalRunAgentTurnParams({ followupRun })); - - expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "primary run", { - provider: "openai", - model: "gpt-5.5", - authProfileId: "openai:primary", - authProfileIdSource: "auto", - }); - expectMockCallArgFields(state.runEmbeddedAgentMock, 1, "fallback run", { - provider: "openai", - model: "gpt-5.4", - authProfileId: "openai:fallback", - authProfileIdSource: "auto", - }); - }); - - it("does not clear a concurrent user selection after primary probe success", async () => { - const probe = { - provider: "anthropic", - model: "claude-sonnet-4-6", - fallbackProvider: "google", - fallbackModel: "gemini-3-pro", - }; - const sessionKey = "concurrent-user-switch-during-probe"; - const staleAutoEntry: SessionEntry = { - sessionId: "session", - updatedAt: 1, - providerOverride: "google", - modelOverride: "gemini-3-pro", - modelOverrideSource: "auto", - modelOverrideFallbackOriginProvider: "anthropic", - modelOverrideFallbackOriginModel: "claude-sonnet-4-6", - }; - const activeSessionStore = { [sessionKey]: staleAutoEntry }; - const followupRun = createFollowupRun(); - followupRun.run.sessionKey = sessionKey; - followupRun.run.provider = "anthropic"; - followupRun.run.model = "claude-sonnet-4-6"; - followupRun.run.autoFallbackPrimaryProbe = probe; - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - const result = await params.run(params.provider, params.model); - activeSessionStore[sessionKey] = { - sessionId: "session", - updatedAt: 2, - providerOverride: "openai", - modelOverride: "gpt-5.4", - modelOverrideSource: "user", - }; - return { - result, - provider: params.provider, - model: params.model, - attempts: [], - }; - }); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "primary recovered" }], - meta: { - agentMeta: { - provider: "anthropic", - model: "claude-sonnet-4-6", - }, - }, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - sessionKey, - activeSessionStore, - getActiveSessionEntry: () => staleAutoEntry, - }); - - expectRecordFields(activeSessionStore[sessionKey] as unknown as Record, { - providerOverride: "openai", - modelOverride: "gpt-5.4", - modelOverrideSource: "user", - }); - }); - - it("keeps rechecked primary probe runs in sync after live model switches", async () => { - const probe = { - provider: "anthropic", - model: "claude-sonnet-4-6", - fallbackProvider: "openai", - fallbackModel: "gpt-5.5", - fallbackAuthProfileId: "openai:fallback", - fallbackAuthProfileIdSource: "auto" as const, - }; - const sessionEntry: SessionEntry = { - sessionId: "session", - updatedAt: 1, - providerOverride: "openai", - modelOverride: "gpt-5.5", - modelOverrideSource: "auto", - modelOverrideFallbackOriginProvider: "anthropic", - modelOverrideFallbackOriginModel: "claude-sonnet-4-6", - }; - const sessionKey = "live-switch-probe"; - const activeSessionStore = { [sessionKey]: sessionEntry }; - const followupRun = createFollowupRun(); - followupRun.run.sessionKey = sessionKey; - followupRun.run.provider = "anthropic"; - followupRun.run.model = "claude-sonnet-4-6"; - followupRun.run.autoFallbackPrimaryProbe = probe; - const attemptedProviders: Array = []; - state.runWithModelFallbackMock.mockImplementation(async (params: FallbackRunnerParams) => { - attemptedProviders.push(params.provider); - const provider = params.provider ?? "anthropic"; - const model = params.model ?? "claude-sonnet-4-6"; - return { - result: await params.run(provider, model), - provider, - model, - attempts: [], - }; - }); - state.runEmbeddedAgentMock - .mockImplementationOnce(async () => { - throw new LiveSessionModelSwitchError({ - provider: "openai", - model: "gpt-5.4", - authProfileId: "openai:primary", - authProfileIdSource: "auto", - }); - }) - .mockResolvedValueOnce({ - payloads: [{ text: "switched" }], - meta: { - agentMeta: { - provider: "openai", - model: "gpt-5.4", - }, - }, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - sessionKey, - activeSessionStore, - getActiveSessionEntry: () => activeSessionStore[sessionKey], - }); - - expect(result.kind).toBe("success"); - expect(attemptedProviders).toEqual(["anthropic", "openai"]); - expectMockCallArgFields(state.runEmbeddedAgentMock, 1, "embedded run", { - provider: "openai", - model: "gpt-5.4", - authProfileId: "openai:primary", - authProfileIdSource: "auto", - }); - }); - - it("forwards the static extra system prompt to CLI backends", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("codex-cli", "gpt-5.4"), - provider: "codex-cli", - model: "gpt-5.4", - attempts: [], - })); - state.runCliAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "final" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "codex-cli"; - followupRun.run.model = "gpt-5.4"; - followupRun.run.extraSystemPrompt = "dynamic inbound metadata\n\nstable group prompt"; - followupRun.run.extraSystemPromptStatic = "stable group prompt"; - followupRun.run.senderId = "sender-static"; - followupRun.run.senderName = "Sender Static"; - followupRun.run.senderUsername = "sender-static-user"; - followupRun.run.senderE164 = "+15550002222"; - followupRun.run.execOverrides = { host: "node", node: "mac-a" }; - followupRun.run.bashElevated = { - enabled: true, - allowed: true, - defaultLevel: "full", - }; - followupRun.run.groupId = "group-static"; - followupRun.run.groupChannel = "ops"; - followupRun.run.groupSpace = "workspace-static"; - followupRun.run.spawnedBy = "agent:main:telegram:group:parent"; - followupRun.run.runtimePolicySessionKey = "agent:main:telegram:default:direct:sender-static"; - followupRun.originatingChannel = "telegram"; - - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { - modelProvider: "codex-cli", - extraSystemPrompt: "dynamic inbound metadata\n\nstable group prompt", - extraSystemPromptStatic: "stable group prompt", - trigger: "user", - messageChannel: "telegram", - messageProvider: "telegram", - senderId: "sender-static", - senderName: "Sender Static", - senderUsername: "sender-static-user", - senderE164: "+15550002222", - execOverrides: { host: "node", node: "mac-a" }, - bashElevated: { enabled: true, allowed: true, defaultLevel: "full" }, - groupId: "group-static", - groupChannel: "ops", - groupSpace: "workspace-static", - spawnedBy: "agent:main:telegram:group:parent", - runtimePolicySessionKey: "agent:main:telegram:default:direct:sender-static", - }); - }); - - it("passes silent empty-reply policy to CLI backends for message-tool-only turns", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("claude-cli", "claude-sonnet-4-6"), - provider: "claude-cli", - model: "claude-sonnet-4-6", - attempts: [], - })); - state.runCliAgentMock.mockResolvedValueOnce({ - payloads: [{ text: SILENT_REPLY_TOKEN }], - meta: { executionTrace: { fallbackUsed: false } }, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "claude-cli"; - followupRun.run.model = "claude-sonnet-4-6"; - followupRun.run.sourceReplyDeliveryMode = "message_tool_only"; - followupRun.run.allowEmptyAssistantReplyAsSilent = true; - followupRun.originatingChannel = "telegram"; - - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - followupRun, - sessionCtx: { - Provider: "telegram", - MessageSid: "msg", - ChatType: "group", - } as unknown as TemplateContext, - }), - ); - - expect(result.kind).toBe("success"); - expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { - provider: "claude-cli", - model: "claude-sonnet-4-6", - sourceReplyDeliveryMode: "message_tool_only", - allowEmptyAssistantReplyAsSilent: true, - messageChannel: "telegram", - messageProvider: "telegram", - }); - }); - - it("passes prepared CLI user turns to the runtime persistence boundary", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("codex-cli", "gpt-5.4"), - provider: "codex-cli", - model: "gpt-5.4", - attempts: [], - })); - state.runCliAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "final" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "codex-cli"; - followupRun.run.model = "gpt-5.4"; - const preparedUserTurnMessage = { - role: "user", - content: "describe this", - MediaPath: "/tmp/image.png", - MediaPaths: ["/tmp/image.png"], - MediaType: "image/png", - MediaTypes: ["image/png"], - } as never; - followupRun.userTurnTranscriptRecorder = createTestUserTurnRecorder(preparedUserTurnMessage); - const sessionEntry: SessionEntry = { - sessionId: "session", - sessionFile: "/tmp/session.jsonl", - updatedAt: 1, - }; - const activeSessionStore = { main: sessionEntry }; - - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - commandBody: "runtime prompt", - transcriptCommandBody: "display prompt", - activeSessionStore, - storePath: "/tmp/sessions.json", - getActiveSessionEntry: () => activeSessionStore.main, - }); - - expect(result.kind).toBe("success"); - expect(state.runCliAgentMock).toHaveBeenCalledOnce(); - expectMockCallArgFields(state.runCliAgentMock, 0, "CLI runtime", { - sessionKey: "main", - agentId: "agent", - sessionId: "session", - suppressNextUserMessagePersistence: false, - persistAssistantTranscript: true, - storePath: "/tmp/sessions.json", - }); - const call = requireMockCall(state.runCliAgentMock, 0, "CLI runtime"); - const callParams = requireRecord(call[0], "CLI runtime"); - expect(callParams.userTurnTranscriptRecorder).toEqual(expect.any(Object)); - expect(requireRecord(callParams.userTurnTranscriptRecorder, "user turn recorder").message).toBe( - preparedUserTurnMessage, - ); - expect(callParams.onUserMessagePersisted).toEqual(expect.any(Function)); - }); - - it("reuses CLI sessions for room-event turns", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("codex-cli", "gpt-5.4"), - provider: "codex-cli", - model: "gpt-5.4", - attempts: [], - })); - state.runCliAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "ambient" }], - meta: { - agentMeta: { - sessionId: "existing-cli-session", - cliSessionBinding: { - sessionId: "existing-cli-session", - authProfileId: "profile", - }, - }, - }, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.currentInboundEventKind = "room_event"; - followupRun.run.provider = "codex-cli"; - followupRun.run.model = "gpt-5.4"; - const sessionEntry = { - cliSessionBindings: { - "codex-cli": { sessionId: "existing-cli-session" }, - }, - } as unknown as SessionEntry; - const activeSessionStore = { main: sessionEntry }; - - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - activeSessionStore, - getActiveSessionEntry: () => sessionEntry, - }); - - expect(result.kind).toBe("success"); - expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { - currentInboundEventKind: "room_event", - persistAssistantTranscript: false, - cliSessionId: "existing-cli-session", - cliSessionBinding: { - sessionId: "existing-cli-session", - }, - }); - if (result.kind !== "success") { - throw new Error("expected success"); - } - expect(result.runResult.meta?.agentMeta?.sessionId).toBe("existing-cli-session"); - expect(result.runResult.meta?.agentMeta?.cliSessionBinding).toEqual({ - sessionId: "existing-cli-session", - authProfileId: "profile", - }); - }); - - it("keeps the first CLI session created by a room-event turn", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("codex-cli", "gpt-5.4"), - provider: "codex-cli", - model: "gpt-5.4", - attempts: [], - })); - state.runCliAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "ambient" }], - meta: { - agentMeta: { - sessionId: "new-cli-session", - cliSessionBinding: { - sessionId: "new-cli-session", - authProfileId: "profile", - }, - }, - }, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.currentInboundEventKind = "room_event"; - followupRun.run.provider = "codex-cli"; - followupRun.run.model = "gpt-5.4"; - const sessionEntry = {} as unknown as SessionEntry; - - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - getActiveSessionEntry: () => sessionEntry, - }); - - expect(result.kind).toBe("success"); - expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { - currentInboundEventKind: "room_event", - cliSessionId: undefined, - cliSessionBinding: undefined, - }); - if (result.kind !== "success") { - throw new Error("expected success"); - } - expect(result.runResult.meta?.agentMeta?.sessionId).toBe("new-cli-session"); - expect(result.runResult.meta?.agentMeta?.cliSessionBinding).toEqual({ - sessionId: "new-cli-session", - authProfileId: "profile", - }); - }); - - it("drops replacement room-event CLI sessions when reuse fails", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("codex-cli", "gpt-5.4"), - provider: "codex-cli", - model: "gpt-5.4", - attempts: [], - })); - state.runCliAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "ambient" }], - meta: { - agentMeta: { - sessionId: "transient-cli-session", - cliSessionBinding: { - sessionId: "transient-cli-session", - authProfileId: "profile", - }, - clearCliSessionBinding: true, - }, - }, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.currentInboundEventKind = "room_event"; - followupRun.run.provider = "codex-cli"; - followupRun.run.model = "gpt-5.4"; - const sessionEntry = { - cliSessionBindings: { - "codex-cli": { sessionId: "existing-cli-session" }, - }, - } as unknown as SessionEntry; - const activeSessionStore = { main: sessionEntry }; - - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - activeSessionStore, - getActiveSessionEntry: () => sessionEntry, - }); - - expect(result.kind).toBe("success"); - expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { - currentInboundEventKind: "room_event", - cliSessionId: "existing-cli-session", - cliSessionBinding: { - sessionId: "existing-cli-session", - }, - }); - if (result.kind !== "success") { - throw new Error("expected success"); - } - expect(result.runResult.meta?.agentMeta?.sessionId).toBe(""); - expect(result.runResult.meta?.agentMeta?.cliSessionBinding).toBeUndefined(); - expect(result.runResult.meta?.agentMeta?.clearCliSessionBinding).toBeUndefined(); - expect(activeSessionStore.main.cliSessionBindings?.["codex-cli"]).toBeUndefined(); - }); - - it("keeps room-event CLI bindings when synthetic hooks return no CLI binding", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("codex-cli", "gpt-5.4"), - provider: "codex-cli", - model: "gpt-5.4", - attempts: [], - })); - state.runCliAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "handled" }], - meta: { - agentMeta: { - sessionId: "openclaw-session", - provider: "codex-cli", - model: "gpt-5.4", - }, - }, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.currentInboundEventKind = "room_event"; - followupRun.run.provider = "codex-cli"; - followupRun.run.model = "gpt-5.4"; - const sessionEntry = { - cliSessionBindings: { - "codex-cli": { sessionId: "existing-cli-session" }, - }, - } as unknown as SessionEntry; - const activeSessionStore = { main: sessionEntry }; - - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - activeSessionStore, - getActiveSessionEntry: () => sessionEntry, - }); - - expect(result.kind).toBe("success"); - if (result.kind !== "success") { - throw new Error("expected success"); - } - expect(result.runResult.meta?.agentMeta?.sessionId).toBe(""); - expect(result.runResult.meta?.agentMeta?.cliSessionBinding).toBeUndefined(); - expect(activeSessionStore.main.cliSessionBindings?.["codex-cli"]).toEqual({ - sessionId: "existing-cli-session", - }); - }); - - it("clears room-event CLI bindings when an unflushed replacement is dropped", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("codex-cli", "gpt-5.4"), - provider: "codex-cli", - model: "gpt-5.4", - attempts: [], - })); - state.runCliAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "handled" }], - meta: { - agentMeta: { - sessionId: "", - provider: "codex-cli", - model: "gpt-5.4", - clearCliSessionBinding: true, - }, - }, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.currentInboundEventKind = "room_event"; - followupRun.run.provider = "codex-cli"; - followupRun.run.model = "gpt-5.4"; - const sessionEntry = { - cliSessionBindings: { - "codex-cli": { sessionId: "existing-cli-session" }, - }, - } as unknown as SessionEntry; - const activeSessionStore = { main: sessionEntry }; - - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - activeSessionStore, - getActiveSessionEntry: () => sessionEntry, - }); - - expect(result.kind).toBe("success"); - if (result.kind !== "success") { - throw new Error("expected success"); - } - expect(result.runResult.meta?.agentMeta?.sessionId).toBe(""); - expect(result.runResult.meta?.agentMeta?.cliSessionBinding).toBeUndefined(); - expect(result.runResult.meta?.agentMeta?.clearCliSessionBinding).toBeUndefined(); - expect(activeSessionStore.main.cliSessionBindings?.["codex-cli"]).toBeUndefined(); - }); - - it("bridges CLI assistant agent events into onPartialReply for live preview (#76869)", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("claude-cli", "claude-opus-4-6"), - provider: "claude-cli", - model: "claude-opus-4-6", - attempts: [], - })); - state.runCliAgentMock.mockImplementationOnce( - async (params: { runId: string; emitCommentaryText?: boolean }) => { - expect(params.emitCommentaryText).toBe(false); - const realAgentEvents = await vi.importActual( - "../../infra/agent-events.js", - ); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "assistant", - data: { text: "Hello", delta: "Hello" }, - }); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "assistant", - data: { text: "Hello world", delta: " world" }, - }); - return { payloads: [{ text: "Hello world" }], meta: {} }; - }, - ); - - const onPartialReply = vi.fn>( - async (_payload) => undefined, - ); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "claude-cli"; - followupRun.run.model = "claude-opus-4-6"; - - await runAgentTurnWithFallback({ - commandBody: "hi", - followupRun, - sessionCtx: { - Provider: "telegram", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onPartialReply }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - const partialTexts = onPartialReply.mock.calls.map((call) => call[0].text); - expect(partialTexts).toEqual(["Hello", "Hello world"]); - }); - - it("serializes and drains bridged CLI assistant previews before completing (#76869)", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("claude-cli", "claude-opus-4-6"), - provider: "claude-cli", - model: "claude-opus-4-6", - attempts: [], - })); - state.runCliAgentMock.mockImplementationOnce( - async (params: { runId: string; emitCommentaryText?: boolean }) => { - expect(params.emitCommentaryText).toBe(false); - const realAgentEvents = await vi.importActual( - "../../infra/agent-events.js", - ); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "assistant", - data: { text: "Hello", delta: "Hello" }, - }); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "assistant", - data: { text: "Hello world", delta: " world" }, - }); - return { payloads: [{ text: "Hello world" }], meta: {} }; - }, - ); - - let firstPreviewStarted: (() => void) | undefined; - let releaseFirstPreview: (() => void) | undefined; - const firstPreviewPromise = new Promise((resolve) => { - firstPreviewStarted = resolve; - }); - const previewOrder: string[] = []; - const onPartialReply = vi.fn>( - async (payload) => { - previewOrder.push(payload.text ?? ""); - if (payload.text === "Hello") { - firstPreviewStarted?.(); - await new Promise((resolve) => { - releaseFirstPreview = resolve; - }); - previewOrder.push("Hello released"); - } - }, - ); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "claude-cli"; - followupRun.run.model = "claude-opus-4-6"; - - const runPromise = runAgentTurnWithFallback({ - commandBody: "hi", - followupRun, - sessionCtx: { - Provider: "telegram", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onPartialReply }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - await firstPreviewPromise; - await new Promise((resolve) => { - setImmediate(resolve); - }); - expect(previewOrder).toEqual(["Hello"]); - - releaseFirstPreview?.(); - await runPromise; - - expect(previewOrder).toEqual(["Hello", "Hello released", "Hello world"]); - }); - - it("bridges CLI tool agent events into onToolStart for live preview", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("claude-cli", "claude-opus-4-6"), - provider: "claude-cli", - model: "claude-opus-4-6", - attempts: [], - })); - state.runCliAgentMock.mockImplementationOnce( - async (params: { runId: string; emitCommentaryText?: boolean }) => { - expect(params.emitCommentaryText).toBe(false); - const realAgentEvents = await vi.importActual( - "../../infra/agent-events.js", - ); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "tool", - data: { - phase: "start", - name: "Bash", - toolCallId: "toolu_01ABCD", - args: { command: "ls -la" }, - }, - }); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "tool", - data: { - phase: "result", - name: "Bash", - toolCallId: "toolu_01ABCD", - isError: false, - }, - }); - return { payloads: [{ text: "done" }], meta: {} }; - }, - ); - - const onToolStart = vi.fn>(async () => undefined); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "claude-cli"; - followupRun.run.model = "claude-opus-4-6"; - - await runAgentTurnWithFallback({ - commandBody: "hi", - followupRun, - sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, - opts: { onToolStart }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - await new Promise((resolve) => { - setImmediate(resolve); - }); - - expect(onToolStart).toHaveBeenCalledTimes(1); - const call = onToolStart.mock.calls[0]?.[0]; - expect(call?.name).toBe("Bash"); - expect(call?.phase).toBe("start"); - expect(call?.args).toEqual({ command: "ls -la" }); - }); - - it("starts CLI assistant progress before a later tool while typing is slow", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("claude-cli", "claude-opus-4-6"), - provider: "claude-cli", - model: "claude-opus-4-6", - attempts: [], - })); - state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { - const agentEvents = await import("../../infra/agent-events.js"); - agentEvents.emitAgentEvent({ - runId: params.runId, - stream: "assistant", - data: { text: "answer before tool", delta: "answer before tool" }, - }); - agentEvents.emitAgentEvent({ - runId: params.runId, - stream: "assistant", - data: { text: "answer before tool 2", delta: " 2" }, - }); - agentEvents.emitAgentEvent({ - runId: params.runId, - stream: "tool", - data: { - phase: "start", - name: "Bash", - toolCallId: "toolu_order", - args: { command: "echo hi" }, - }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - let releaseTyping: (() => void) | undefined; - const typingPending = new Promise((resolve) => { - releaseTyping = resolve; - }); - const typingSignals = createMockTypingSignaler(); - vi.mocked(typingSignals.signalTextDelta).mockReturnValue(typingPending); - const callbackOrder: string[] = []; - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "claude-cli"; - followupRun.run.model = "claude-opus-4-6"; - const runPromise = runAgentTurnWithFallback({ - commandBody: "hi", - followupRun, - sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, - opts: { - preserveProgressCallbackStartOrder: true, - onPartialReply: (payload) => { - callbackOrder.push(`partial:${payload.text}`); - }, - onToolStart: () => { - callbackOrder.push("tool"); - }, - }, - typingSignals, - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - try { - await vi.waitFor(() => { - expect(callbackOrder).toContain("tool"); - }); - expect(callbackOrder).toEqual([ - "partial:answer before tool", - "partial:answer before tool 2", - "tool", - ]); - } finally { - releaseTyping?.(); - await runPromise; - } - }); - - it("starts CLI tool progress before later assistant text while typing is slow", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("claude-cli", "claude-opus-4-6"), - provider: "claude-cli", - model: "claude-opus-4-6", - attempts: [], - })); - state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { - const agentEvents = await import("../../infra/agent-events.js"); - agentEvents.emitAgentEvent({ - runId: params.runId, - stream: "tool", - data: { - phase: "start", - name: "Bash", - toolCallId: "toolu_inverse_order", - args: { command: "echo hi" }, - }, - }); - agentEvents.emitAgentEvent({ - runId: params.runId, - stream: "tool", - data: { - phase: "update", - name: "Bash", - toolCallId: "toolu_inverse_order", - args: { command: "echo hi" }, - }, - }); - agentEvents.emitAgentEvent({ - runId: params.runId, - stream: "assistant", - data: { text: "answer after tool", delta: "answer after tool" }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - let releaseTyping: (() => void) | undefined; - const typingPending = new Promise((resolve) => { - releaseTyping = resolve; - }); - const typingSignals = createMockTypingSignaler(); - vi.mocked(typingSignals.signalToolStart).mockReturnValue(typingPending); - const callbackOrder: string[] = []; - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "claude-cli"; - followupRun.run.model = "claude-opus-4-6"; - const runPromise = runAgentTurnWithFallback({ - commandBody: "hi", - followupRun, - sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, - opts: { - preserveProgressCallbackStartOrder: true, - onPartialReply: (payload) => { - callbackOrder.push(`partial:${payload.text}`); - }, - onToolStart: (payload) => { - callbackOrder.push(`tool:${payload.phase}`); - }, - }, - typingSignals, - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - try { - await vi.waitFor(() => { - expect(callbackOrder).toContain("partial:answer after tool"); - }); - expect(callbackOrder).toEqual(["tool:start", "tool:update", "partial:answer after tool"]); - } finally { - releaseTyping?.(); - await runPromise; - } - }); - - it("bridges CLI preambles for progress headlines when commentary is disabled", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("claude-cli", "claude-opus-4-6"), - provider: "claude-cli", - model: "claude-opus-4-6", - attempts: [], - })); - state.runCliAgentMock.mockImplementationOnce( - async (params: { runId: string; emitCommentaryText?: boolean }) => { - expect(params.emitCommentaryText).toBe(true); - const agentEvents = await import("../../infra/agent-events.js"); - // Inter-tool commentary surfaces as a stream:"item", kind:"preamble" agent event. - agentEvents.emitAgentEvent({ - runId: params.runId, - stream: "item", - data: { - kind: "preamble", - itemId: "commentary-1", - progressText: "Let me check the files.", - }, - }); - return { payloads: [{ text: "done" }], meta: {} }; - }, - ); - - const onItemEvent = vi.fn>(async () => undefined); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "claude-cli"; - followupRun.run.model = "claude-opus-4-6"; - - await runAgentTurnWithFallback({ - commandBody: "hi", - followupRun, - sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, - opts: { - onItemEvent, - commentaryProgressEnabled: false, - progressPreambleEnabled: true, - }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - await new Promise((resolve) => { - setImmediate(resolve); - }); - - expect(onItemEvent).toHaveBeenCalledTimes(1); - const call = onItemEvent.mock.calls[0]?.[0]; - expect(call?.kind).toBe("preamble"); - expect(call?.progressText).toBe("Let me check the files."); - expect(call?.itemId).toBe("commentary-1"); - }); - - it("does not emit CLI preambles when both progress lanes are disabled", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("claude-cli", "claude-opus-4-6"), - provider: "claude-cli", - model: "claude-opus-4-6", - attempts: [], - })); - state.runCliAgentMock.mockImplementationOnce( - async (params: { runId: string; emitCommentaryText?: boolean }) => { - // With no commentary lane or headline consumer, pre-tool text stays in - // the assistant stream instead of being split into progress events. - expect(params.emitCommentaryText).toBe(false); - return { payloads: [{ text: "done" }], meta: {} }; - }, - ); - - const onItemEvent = vi.fn>(); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "claude-cli"; - followupRun.run.model = "claude-opus-4-6"; - - await runAgentTurnWithFallback({ - commandBody: "hi", - followupRun, - sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, - opts: { - onItemEvent, - commentaryProgressEnabled: false, - progressPreambleEnabled: false, - }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(state.runCliAgentMock).toHaveBeenCalledTimes(1); - expect(onItemEvent).not.toHaveBeenCalled(); - }); - - it("does not bridge CLI tool deltas when silentExpected is set", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("claude-cli", "claude-opus-4-6"), - provider: "claude-cli", - model: "claude-opus-4-6", - attempts: [], - })); - state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { - const realAgentEvents = await vi.importActual( - "../../infra/agent-events.js", - ); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "tool", - data: { - phase: "start", - name: "Bash", - toolCallId: "toolu_silent", - args: { command: "echo silent" }, - }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const onToolStart = vi.fn>(async () => undefined); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "claude-cli"; - followupRun.run.model = "claude-opus-4-6"; - followupRun.run.silentExpected = true; - - await runAgentTurnWithFallback({ - commandBody: "hi", - followupRun, - sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, - opts: { onToolStart }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - await new Promise((resolve) => { - setImmediate(resolve); - }); - - expect(onToolStart).not.toHaveBeenCalled(); - }); - - it("does not bridge CLI assistant deltas when silentExpected is set (#76869)", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("claude-cli", "claude-opus-4-6"), - provider: "claude-cli", - model: "claude-opus-4-6", - attempts: [], - })); - state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { - const realAgentEvents = await vi.importActual( - "../../infra/agent-events.js", - ); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "assistant", - data: { text: "secret heartbeat output", delta: "secret heartbeat output" }, - }); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "assistant", - data: { text: "NO_REPLY do not preview", delta: " do not preview" }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const onPartialReply = vi.fn>( - async (_payload) => undefined, - ); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "claude-cli"; - followupRun.run.model = "claude-opus-4-6"; - followupRun.run.silentExpected = true; - - await runAgentTurnWithFallback({ - commandBody: "hi", - followupRun, - sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, - opts: { onPartialReply }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - await new Promise((resolve) => { - setImmediate(resolve); - }); - - expect(onPartialReply).not.toHaveBeenCalled(); - }); - - it("bridges CLI thinking agent events into onReasoningStream with the reasoning opt-in gate", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("claude-cli", "claude-opus-4-7"), - provider: "claude-cli", - model: "claude-opus-4-7", - attempts: [], - })); - state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { - const realAgentEvents = await vi.importActual( - "../../infra/agent-events.js", - ); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "thinking", - data: { text: "Thinking", delta: "Thinking", isReasoningSnapshot: true }, - }); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "thinking", - data: { text: "Thinking", delta: "", isReasoningSnapshot: true }, - }); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "thinking", - data: { text: "Thinking about it", delta: " about it", isReasoningSnapshot: true }, - }); - return { payloads: [{ text: "Thinking about it" }], meta: {} }; - }); - - const onReasoningStream = vi.fn>( - async (_payload) => undefined, - ); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "claude-cli"; - followupRun.run.model = "claude-opus-4-7"; - - await runAgentTurnWithFallback({ - commandBody: "hi", - followupRun, - sessionCtx: { - Provider: "telegram", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onReasoningStream }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(onReasoningStream.mock.calls.map((call) => call[0])).toEqual([ - { - text: "Thinking", - isReasoningSnapshot: true, - requiresReasoningProgressOptIn: true, - }, - { - text: "Thinking about it", - isReasoningSnapshot: true, - requiresReasoningProgressOptIn: true, - }, - ]); - }); - - it("does not bridge CLI thinking events to onReasoningStream when silentExpected is set", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("claude-cli", "claude-opus-4-7"), - provider: "claude-cli", - model: "claude-opus-4-7", - attempts: [], - })); - state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { - const realAgentEvents = await vi.importActual( - "../../infra/agent-events.js", - ); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "thinking", - data: { text: "heartbeat scratch text", delta: "heartbeat scratch text" }, - }); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "thinking", - data: { text: "NO_REPLY do not preview reasoning", delta: " do not preview reasoning" }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const onReasoningStream = vi.fn>( - async (_payload) => undefined, - ); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "claude-cli"; - followupRun.run.model = "claude-opus-4-7"; - followupRun.run.silentExpected = true; - - await runAgentTurnWithFallback({ - commandBody: "hi", - followupRun, - sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, - opts: { onReasoningStream }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - await new Promise((resolve) => { - setImmediate(resolve); - }); - - expect(onReasoningStream).not.toHaveBeenCalled(); - }); - - it("does not bridge non-Claude CLI assistant events to onReasoningStream", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("codex-cli", "gpt-5.5"), - provider: "codex-cli", - model: "gpt-5.5", - attempts: [], - })); - state.runCliAgentMock.mockImplementationOnce(async (params: { runId: string }) => { - const realAgentEvents = await vi.importActual( - "../../infra/agent-events.js", - ); - realAgentEvents.emitAgentEvent({ - runId: params.runId, - stream: "assistant", - data: { text: "final answer", delta: "final answer" }, - }); - return { payloads: [{ text: "final answer" }], meta: {} }; - }); - - const onReasoningStream = vi.fn>( - async (_payload) => undefined, - ); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "codex-cli"; - followupRun.run.model = "gpt-5.5"; - - await runAgentTurnWithFallback({ - commandBody: "hi", - followupRun, - sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, - opts: { onReasoningStream }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - await new Promise((resolve) => { - setImmediate(resolve); - }); - - expect(onReasoningStream).not.toHaveBeenCalled(); - }); - - it("does not double-fire onReasoningStream from the bridge when the API/native runtime path is active", async () => { - state.isCliProviderMock.mockReturnValue(false); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("anthropic", "claude-sonnet-4-7"), - provider: "anthropic", - model: "claude-sonnet-4-7", - attempts: [], - })); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - const realAgentEvents = await vi.importActual( - "../../infra/agent-events.js", - ); - realAgentEvents.emitAgentEvent({ - runId: "api-run", - stream: "assistant", - data: { text: "assistant text from API run", delta: "assistant text from API run" }, - }); - await params.onAgentEvent?.({ - stream: "assistant", - data: { text: "assistant text from API run", delta: "assistant text from API run" }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const onReasoningStream = vi.fn>( - async (_payload) => undefined, - ); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "anthropic"; - followupRun.run.model = "claude-sonnet-4-7"; - - await runAgentTurnWithFallback({ - commandBody: "hi", - followupRun, - sessionCtx: { Provider: "telegram", MessageSid: "msg" } as unknown as TemplateContext, - opts: { onReasoningStream }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - await new Promise((resolve) => { - setImmediate(resolve); - }); - - expect(onReasoningStream).not.toHaveBeenCalled(); - }); - - it("preserves embedded reasoning stream opt-in markers", async () => { - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onReasoningStream?.({ text: "stream thought" }); - await params.onReasoningStream?.({ - text: "ambient thought", - requiresReasoningProgressOptIn: true, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const onReasoningStream = vi.fn>( - async (_payload) => undefined, - ); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - - await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - opts: { onReasoningStream }, - }), - ); - - expect( - onReasoningStream.mock.calls.map(([payload]) => ({ - text: payload.text, - requiresReasoningProgressOptIn: payload.requiresReasoningProgressOptIn, - })), - ).toEqual([ - { text: "stream thought", requiresReasoningProgressOptIn: undefined }, - { text: "ambient thought", requiresReasoningProgressOptIn: true }, - ]); - }); - - it("resolves CLI messageProvider from the live session surface when no origin channel is set", async () => { - state.isCliProviderMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("codex-cli", "gpt-5.4"), - provider: "codex-cli", - model: "gpt-5.4", - attempts: [], - })); - state.runCliAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "final" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "codex-cli"; - followupRun.run.model = "gpt-5.4"; - followupRun.run.messageProvider = "stale-provider"; - - await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "discord", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { - messageChannel: undefined, - messageProvider: "discord", - }); - }); - - it("does not pass CLI runtime overrides as embedded harness ids for fallback providers", async () => { - cliBackendsTesting.setDepsForTest({ - resolveRuntimeCliBackends: () => [], - resolvePluginSetupCliBackend: ({ backend, config }) => - backend === "claude-cli" && config - ? { - pluginId: "anthropic", - backend: { - id: "claude-cli", - modelProvider: "anthropic", - config: { command: "claude" }, - bundleMcp: false, - }, - } - : undefined, - }); - state.isCliProviderMock.mockImplementation((provider: unknown) => provider === "claude-cli"); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - })); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "fallback" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "anthropic"; - followupRun.run.model = "claude-opus-4-7"; - followupRun.run.config = { - agents: { - defaults: { - agentRuntime: { id: "claude-cli" }, - }, - }, - }; - - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - getActiveSessionEntry: () => - ({ - sessionId: "session", - updatedAt: Date.now(), - agentRuntimeOverride: "claude-cli", - }) as SessionEntry, - }); - - expect(result.kind).toBe("success"); - expect(state.runCliAgentMock).not.toHaveBeenCalled(); - expect(state.runEmbeddedAgentMock).toHaveBeenCalledOnce(); - expect( - requireRecord( - requireMockCall(state.runEmbeddedAgentMock, 0, "embedded run params")[0], - "embedded run params", - ), - ).not.toHaveProperty("agentHarnessId", "claude-cli"); - }); - - it("passes OpenAI session runtime overrides as embedded harness ids", async () => { - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - })); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "openai" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "openai"; - followupRun.run.model = "gpt-5.4"; - - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - getActiveSessionEntry: () => - ({ - sessionId: "session", - updatedAt: Date.now(), - agentRuntimeOverride: "codex", - }) as SessionEntry, - }); - - expect(result.kind).toBe("success"); - expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { - provider: "openai", - model: "gpt-5.4", - agentHarnessId: "codex", - }); - }); - - it("keeps catalog-adopted Codex sessions on Codex during heartbeat model overrides", async () => { - state.isCliProviderMock.mockImplementation((provider: unknown) => provider === "claude-cli"); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("anthropic", "claude-opus-4-6"), - provider: "anthropic", - model: "claude-opus-4-6", - attempts: [], - })); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "heartbeat" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "anthropic"; - followupRun.run.model = "claude-opus-4-6"; - followupRun.run.config = { - agents: { - defaults: { - models: { - "anthropic/claude-opus-4-6": { agentRuntime: { id: "claude-cli" } }, - }, - }, - }, - }; - - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - isHeartbeat: true, - getActiveSessionEntry: () => - ({ - sessionId: "catalog-adopted-session", - updatedAt: Date.now(), - agentHarnessId: "codex", - modelSelectionLocked: true, - pluginExtensions: { - codex: { - supervision: { - sourceThreadId: "019f-codex-thread", - modelLocked: true, - }, - }, - }, - }) as SessionEntry, - }); - - expect(result.kind).toBe("success"); - expect(state.runCliAgentMock).not.toHaveBeenCalled(); - expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { - provider: "anthropic", - model: "claude-opus-4-6", - trigger: "heartbeat", - agentHarnessId: "codex", - agentHarnessRuntimeOverride: "codex", - }); - }); - - it("keeps a locked Codex harness embedded when cliBackends.codex is configured", async () => { - state.isCliProviderMock.mockImplementation((provider: unknown) => provider === "codex"); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - })); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "continued" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "openai"; - followupRun.run.model = "gpt-5.4"; - followupRun.run.config = { - agents: { - defaults: { - cliBackends: { - codex: { command: "codex" }, - }, - }, - }, - }; - - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - getActiveSessionEntry: () => - ({ - sessionId: "catalog-adopted-session", - updatedAt: Date.now(), - agentHarnessId: "codex", - modelSelectionLocked: true, - }) as SessionEntry, - }); - - expect(result.kind).toBe("success"); - expect(state.runCliAgentMock).not.toHaveBeenCalled(); - expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { - provider: "openai", - model: "gpt-5.4", - agentHarnessId: "codex", - agentHarnessRuntimeOverride: "codex", - }); - }); - - it("honors agent session runtime overrides before CLI runtime aliases", async () => { - state.isCliProviderMock.mockImplementation((provider: unknown) => provider === "claude-cli"); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - })); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "agent" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "openai"; - followupRun.run.model = "gpt-5.4"; - followupRun.run.config = { - agents: { - defaults: { - agentRuntime: { id: "claude-cli" }, - }, - }, - }; - - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - getActiveSessionEntry: () => - ({ - sessionId: "session", - updatedAt: Date.now(), - agentRuntimeOverride: "codex", - }) as SessionEntry, - }); - - expect(result.kind).toBe("success"); - expect(state.runCliAgentMock).not.toHaveBeenCalled(); - expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { - provider: "openai", - model: "gpt-5.4", - agentHarnessId: "codex", - }); - }); - - it("forwards media-only tool results without typing text", async () => { - const onToolResult = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onToolResult?.({ mediaUrls: ["/tmp/generated.png"] }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const pendingToolTasks = new Set>(); - const typingSignals = createMockTypingSignaler(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { - onToolResult, - } satisfies GetReplyOptions, - typingSignals, - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks, - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - await Promise.all(pendingToolTasks); - - expect(result.kind).toBe("success"); - expect(typingSignals.signalTextDelta).not.toHaveBeenCalled(); - expect(onToolResult).toHaveBeenCalledTimes(1); - expectMockCallArgFields(onToolResult, 0, "tool result payload", { - mediaUrls: ["/tmp/generated.png"], - }); - expect( - requireRecord( - requireMockCall(onToolResult, 0, "tool result payload")[0], - "tool result payload", - ).text, - ).toBeUndefined(); - }); - - it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( - "surfaces model capacity errors from no-text mid-turn failures in $label chats", - async (testCase) => { - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "thinking", isReasoning: true }], - meta: { - error: { - kind: "server_overloaded", - message: "Selected model is at capacity. Please try a different model.", - }, - }, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: createNonDirectFailureSessionCtx(testCase), - }), - ); - - expect(result.kind).toBe("success"); - if (result.kind === "success") { - expect(result.runResult.payloads).toEqual([ - { - text: "⚠️ Selected model is at capacity. Try a different model, or wait and retry.", - isError: true, - }, - ]); - } - }, - ); - - it("surfaces model capacity errors from pre-reply CLI failures", async () => { - state.runWithModelFallbackMock.mockRejectedValueOnce( - new Error("Selected model is at capacity. Please try a different model."), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "openai"; - followupRun.run.model = "gpt-5.5"; - - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result).toEqual({ - kind: "final", - payload: { - isError: true, - text: "⚠️ Selected model is at capacity. Try a different model, or wait and retry.", - }, - }); - }); - - it("classifies structured harness plan-only terminal results as fallback-eligible", async () => { - const followupRun = createFollowupRun(); - followupRun.run.provider = "openai"; - followupRun.run.model = "gpt-5.4"; - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [], - meta: { - agentHarnessResultClassification: "planning-only", - }, - }); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - const first = (await params.run("openai", "gpt-5.4")) as { - payloads?: Array<{ text?: string; isError?: boolean; isReasoning?: boolean }>; - }; - const classification = await params.classifyResult?.({ - result: first, - provider: "openai", - model: "gpt-5.4", - attempt: 1, - total: 2, - }); - expectRecordFields(requireRecord(classification, "fallback classification"), { - reason: "format", - code: "planning_only_result", - }); - return { - result: { payloads: [{ text: "fallback ok" }], meta: {} }, - provider: "anthropic", - model: "claude", - attempts: [ - { - provider: "openai", - model: "gpt-5.4", - error: "planning-only", - reason: "format", - }, - ], - }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams({ followupRun })); - - expect(result.kind).toBe("success"); - if (result.kind === "success") { - expect(result.runResult.payloads?.[0]?.text).toBe("fallback ok"); - expect(result.fallbackProvider).toBe("anthropic"); - expect(result.fallbackAttempts[0]?.reason).toBe("format"); - } - }); - - it("does not classify silent NO_REPLY terminal results for fallback", async () => { - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - const result = { payloads: [{ text: "NO_REPLY" }], meta: {} }; - expect( - await params.classifyResult?.({ - result, - provider: "openai", - model: "gpt-5.4", - attempt: 1, - total: 2, - }), - ).toBeNull(); - return { - result, - provider: "openai", - model: "gpt-5.4", - attempts: [], - }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("success"); - }); - - it("does not classify empty final payloads after block replies were sent", async () => { - const followupRun = createFollowupRun(); - followupRun.run.provider = "openai"; - followupRun.run.model = "gpt-5.4"; - state.createBlockReplyDeliveryHandlerMock.mockImplementationOnce( - (params: { directlySentBlockKeys?: Set }) => async () => { - params.directlySentBlockKeys?.add("block:1"); - }, - ); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onBlockReply?.({ text: "streamed block" }); - return { payloads: [], meta: {} }; - }); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - const result = (await params.run("openai", "gpt-5.4")) as { - payloads?: Array<{ text?: string; isError?: boolean; isReasoning?: boolean }>; - }; - expect( - await params.classifyResult?.({ - result, - provider: "openai", - model: "gpt-5.4", - attempt: 1, - total: 2, - }), - ).toBeNull(); - return { - result, - provider: "openai", - model: "gpt-5.4", - attempts: [], - }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - followupRun, - opts: { onBlockReply: vi.fn() } satisfies GetReplyOptions, - }), - ); - - expect(result.kind).toBe("success"); - }); - - it("does not classify empty final payloads while block replies are buffered", async () => { - const followupRun = createFollowupRun(); - followupRun.run.provider = "openai"; - followupRun.run.model = "gpt-5.4"; - const blockReplyPipeline = { - enqueue: vi.fn(), - flush: vi.fn(async () => {}), - stop: vi.fn(), - hasBuffered: vi.fn(() => true), - didStream: vi.fn(() => false), - isAborted: vi.fn(() => false), - hasSentPayload: vi.fn(() => false), - getSentMediaUrls: vi.fn(() => []), - }; - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - const result = { payloads: [], meta: {} }; - expect( - await params.classifyResult?.({ - result, - provider: "openai", - model: "gpt-5.4", - attempt: 1, - total: 2, - }), - ).toBeNull(); - return { - result, - provider: "openai", - model: "gpt-5.4", - attempts: [], - }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - blockReplyPipeline, - blockStreamingEnabled: true, - opts: { onBlockReply: vi.fn() } satisfies GetReplyOptions, - }); - - expect(result.kind).toBe("success"); - }); - - it("classifies final GPT-5 terminal-empty results instead of silently succeeding", async () => { - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - const result = { payloads: [], meta: {} }; - const classification = await params.classifyResult?.({ - result, - provider: "openai", - model: "gpt-5.4", - attempt: 1, - total: 1, - }); - expectRecordFields(requireRecord(classification, "fallback classification"), { - reason: "format", - code: "empty_result", - }); - return { - result, - provider: "openai", - model: "gpt-5.4", - attempts: [], - }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("success"); - }); - - it("keeps fallback candidate selection turn-local during result classification", async () => { - const followupRun = createFollowupRun(); - followupRun.run.provider = "anthropic"; - followupRun.run.model = "claude"; - const sessionEntry: SessionEntry = { - sessionId: "session", - updatedAt: Date.now(), - totalTokens: 1, - compactionCount: 0, - }; - const activeSessionStore = { main: sessionEntry }; - state.runEmbeddedAgentMock.mockResolvedValueOnce({ payloads: [], meta: {} }); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - const failedResult = await params.run("openai", "gpt-5.4"); - expect(sessionEntry.providerOverride).toBeUndefined(); - expect(sessionEntry.modelOverride).toBeUndefined(); - const classification = await params.classifyResult?.({ - result: failedResult as { payloads?: [] }, - provider: "openai", - model: "gpt-5.4", - attempt: 1, - total: 2, - }); - expectRecordFields(requireRecord(classification, "fallback classification"), { - code: "empty_result", - }); - return { - result: { payloads: [{ text: "fallback ok" }], meta: {} }, - provider: "anthropic", - model: "claude", - attempts: [], - }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ followupRun }), - activeSessionStore, - getActiveSessionEntry: () => sessionEntry, - }); - - expect(result.kind).toBe("success"); - expect(sessionEntry.providerOverride).toBeUndefined(); - expect(sessionEntry.modelOverride).toBeUndefined(); - }); - - it("strips a glued leading NO_REPLY token from streamed tool results", async () => { - const onToolResult = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onToolResult?.({ text: "NO_REPLYThe user is saying hello" }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const pendingToolTasks = new Set>(); - const typingSignals = createMockTypingSignaler(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { - onToolResult, - } satisfies GetReplyOptions, - typingSignals, - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks, - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - await Promise.all(pendingToolTasks); - - expect(result.kind).toBe("success"); - expect(typingSignals.signalTextDelta).toHaveBeenCalledWith("The user is saying hello"); - expect(onToolResult).toHaveBeenCalledWith({ text: "The user is saying hello" }); - }); - - it("continues delivering later streamed tool results after an earlier delivery failure", async () => { - const delivered: string[] = []; - const onToolResult = vi.fn(async (payload: { text?: string }) => { - if (payload.text === "first") { - throw new Error("simulated delivery failure"); - } - delivered.push(payload.text ?? ""); - }); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - void params.onToolResult?.({ text: "first", mediaUrls: [] }); - void params.onToolResult?.({ text: "second", mediaUrls: [] }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const pendingToolTasks = new Set>(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onToolResult } satisfies GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks, - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - await Promise.all(pendingToolTasks); - - expect(result.kind).toBe("success"); - expect(onToolResult).toHaveBeenCalledTimes(2); - expect(delivered).toEqual(["second"]); - }); - - it("delivers streamed tool results in callback order even when dispatch latency differs", async () => { - const deliveryOrder: string[] = []; - const onToolResult = vi.fn(async (payload: { text?: string }) => { - const delay = payload.text === "first" ? 5 : 1; - await new Promise((resolve) => { - setTimeout(resolve, delay); - }); - deliveryOrder.push(payload.text ?? ""); - }); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - void params.onToolResult?.({ text: "first", mediaUrls: [] }); - void params.onToolResult?.({ text: "second", mediaUrls: [] }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const pendingToolTasks = new Set>(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onToolResult } satisfies GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks, - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - await Promise.all(pendingToolTasks); - - expect(result.kind).toBe("success"); - expect(onToolResult).toHaveBeenCalledTimes(2); - expect(deliveryOrder).toEqual(["first", "second"]); - }); - - it("forwards item lifecycle events to reply options", async () => { - const onItemEvent = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "item", - data: { - itemId: "tool:read-1", - toolCallId: "read-1", - kind: "tool", - title: "read", - name: "read", - phase: "start", - status: "running", - }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const pendingToolTasks = new Set>(); - const typingSignals = createMockTypingSignaler(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { - onItemEvent, - } satisfies GetReplyOptions, - typingSignals, - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks, - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - await Promise.all(pendingToolTasks); - - expect(result.kind).toBe("success"); - expect(onItemEvent).toHaveBeenCalledWith({ - itemId: "tool:read-1", - toolCallId: "read-1", - kind: "tool", - title: "read", - name: "read", - phase: "start", - status: "running", - }); - }); - - it("skips channel item progress when a matching tool event carries the progress", async () => { - const onItemEvent = vi.fn(); - const onToolStart = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "item", - data: { - itemId: "cmd-1", - toolCallId: "cmd-1", - kind: "command", - title: "Command", - name: "bash", - phase: "start", - status: "running", - suppressChannelProgress: true, - }, - }); - await params.onAgentEvent?.({ - stream: "tool", - data: { - itemId: "cmd-1", - toolCallId: "cmd-1", - name: "bash", - phase: "start", - args: { command: "pnpm test" }, - }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ - opts: { - onItemEvent, - onToolStart, - } satisfies GetReplyOptions, - }), - }); - - expect(result.kind).toBe("success"); - expect(onItemEvent).not.toHaveBeenCalled(); - expect(onToolStart).toHaveBeenCalledWith({ - itemId: "cmd-1", - toolCallId: "cmd-1", - name: "bash", - phase: "start", - args: { command: "pnpm test" }, - detailMode: undefined, - }); - }); - - it("preserves suppressed item progress when no tool-start callback is registered", async () => { - const onItemEvent = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "item", - data: { - itemId: "cmd-1", - toolCallId: "cmd-1", - kind: "command", - title: "Command", - name: "bash", - phase: "start", - status: "running", - suppressChannelProgress: true, - }, - }); - await params.onAgentEvent?.({ - stream: "tool", - data: { - itemId: "cmd-1", - toolCallId: "cmd-1", - name: "bash", - phase: "start", - args: { command: "pnpm test" }, - }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ - opts: { - onItemEvent, - } satisfies GetReplyOptions, - }), - }); - - expect(result.kind).toBe("success"); - expect(onItemEvent).toHaveBeenCalledWith({ - itemId: "cmd-1", - toolCallId: "cmd-1", - kind: "command", - title: "Command", - name: "bash", - phase: "start", - status: "running", - }); - }); - - it("hides internal lifecycle events while preserving visible tool progress", async () => { - const onItemEvent = vi.fn(); - const onToolStart = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "tool", - data: { - name: "exec", - phase: "start", - args: { command: "pwd" }, - }, - }); - await params.onAgentEvent?.({ - stream: "item", - data: { - itemId: "tool:exec-1", - kind: "tool", - title: "exec pwd", - name: "exec", - phase: "start", - status: "running", - }, - }); - await params.onAgentEvent?.({ - stream: "tool", - data: { - name: "wait", - phase: "start", - args: { runId: "ordinary_wait" }, - }, - }); - await params.onAgentEvent?.({ - stream: "tool", - data: { - name: "wait", - phase: "start", - args: { runId: "cm_1" }, - hideFromChannelProgress: true, - }, - }); - await params.onAgentEvent?.({ - stream: "item", - data: { - itemId: "tool:wait-1", - kind: "tool", - title: "wait", - name: "wait", - phase: "start", - status: "running", - hideFromChannelProgress: true, - }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ - opts: { onItemEvent, onToolStart } satisfies GetReplyOptions, - }), - }); - - expect(result.kind).toBe("success"); - expect(onToolStart).toHaveBeenCalledTimes(2); - expect(onToolStart).toHaveBeenCalledWith( - expect.objectContaining({ name: "exec", phase: "start" }), - ); - expect(onToolStart).toHaveBeenCalledWith( - expect.objectContaining({ name: "wait", phase: "start" }), - ); - expect(onItemEvent).toHaveBeenCalledTimes(1); - expect(onItemEvent).toHaveBeenCalledWith( - expect.objectContaining({ name: "exec", phase: "start" }), - ); - }); - - it("forwards raw tool progress detail mode to tool-start reply options", async () => { - const onToolStart = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "tool", - data: { - name: "exec", - phase: "start", - args: { command: "pnpm test -- --watch=false" }, - }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ - opts: { - onToolStart, - } satisfies GetReplyOptions, - }), - toolProgressDetail: "raw", - }); - - expect(result.kind).toBe("success"); - expect(onToolStart).toHaveBeenCalledWith({ - itemId: undefined, - toolCallId: undefined, - name: "exec", - phase: "start", - args: { command: "pnpm test -- --watch=false" }, - detailMode: "raw", - }); - }); - - it("fires tool-start progress before slow typing signals resolve for best-effort agent events", async () => { - const onToolStart = vi.fn(async () => {}); - let releaseTyping: (() => void) | undefined; - const typingSignals = createMockTypingSignaler(); - vi.mocked(typingSignals.signalToolStart).mockImplementation( - () => - new Promise((resolve) => { - releaseTyping = resolve; - }), - ); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - void params.onAgentEvent?.({ - stream: "tool", - data: { - name: "exec", - phase: "start", - args: { command: "echo hi" }, - }, - }); - await Promise.resolve(); - await Promise.resolve(); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ - opts: { - onToolStart, - } satisfies GetReplyOptions, - }), - typingSignals, - }); - - try { - expect(result.kind).toBe("success"); - expect(onToolStart).toHaveBeenCalledWith({ - itemId: undefined, - toolCallId: undefined, - name: "exec", - phase: "start", - args: { command: "echo hi" }, - detailMode: undefined, - }); - } finally { - releaseTyping?.(); - await Promise.resolve(); - } - }); - - it("starts presentation callbacks in source order while typing signals are slow", async () => { - const callbackOrder: string[] = []; - let releaseTyping: (() => void) | undefined; - const typingPending = new Promise((resolve) => { - releaseTyping = resolve; - }); - const typingSignals = createMockTypingSignaler(); - vi.mocked(typingSignals.signalMessageStart).mockReturnValue(typingPending); - vi.mocked(typingSignals.signalTextDelta).mockReturnValue(typingPending); - vi.mocked(typingSignals.signalReasoningDelta).mockReturnValue(typingPending); - vi.mocked(typingSignals.signalToolStart).mockReturnValue(typingPending); - - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - void params.onAssistantMessageStart?.(); - void params.onPartialReply?.({ text: "answer before tool" }); - void params.onReasoningStream?.({ text: "reasoning before tool" }); - void params.onReasoningEnd?.(); - void params.onAgentEvent?.({ - stream: "tool", - data: { - name: "exec", - phase: "start", - args: { command: "echo hi" }, - }, - }); - await Promise.resolve(); - await Promise.resolve(); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ - opts: { - preserveProgressCallbackStartOrder: true, - onAssistantMessageStart: () => { - callbackOrder.push("message-start"); - }, - onPartialReply: () => { - callbackOrder.push("partial"); - }, - onReasoningStream: () => { - callbackOrder.push("reasoning"); - }, - onReasoningEnd: () => { - callbackOrder.push("reasoning-end"); - }, - onToolStart: () => { - callbackOrder.push("tool"); - }, - } satisfies GetReplyOptions, - }), - typingSignals, - }); - - try { - expect(result.kind).toBe("success"); - expect(callbackOrder).toEqual([ - "message-start", - "partial", - "reasoning", - "reasoning-end", - "tool", - ]); - } finally { - releaseTyping?.(); - await Promise.resolve(); - } - }); - - it("starts typing when a presentation callback throws inline", async () => { - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await expect(params.onPartialReply?.({ text: "before failure" })).rejects.toThrow( - "presentation failed", - ); - return { payloads: [{ text: "final" }], meta: {} }; - }); - const typingSignals = createMockTypingSignaler(); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ - opts: { - preserveProgressCallbackStartOrder: true, - onPartialReply: () => { - throw new Error("presentation failed"); - }, - }, - }), - typingSignals, - }); - - expect(result.kind).toBe("success"); - expect(typingSignals.signalTextDelta).toHaveBeenCalledWith("before failure"); - }); - - it("leaves Codex app-server telemetry publication to the harness", async () => { - const agentEvents = await import("../../infra/agent-events.js"); - const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "codex_app_server.guardian", - sessionKey: "agent:main:subagent:codex-child", - data: { - phase: "blocked", - message: "command requires approval", - }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { runId: "run-codex" } as GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - expectNoMockCallWithFields(emitAgentEvent, { - runId: "run-codex", - stream: "codex_app_server.guardian", - }); - }); - - it("emits an embedded lifecycle terminal backstop when the runner returns without one", async () => { - const agentEvents = await import("../../infra/agent-events.js"); - const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "lifecycle", - data: { phase: "start", startedAt: 1_000 }, - }); - return { - payloads: [{ text: "Request timed out before a response was generated.", isError: true }], - meta: { aborted: true, livenessState: "blocked", replayInvalid: true }, - }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { runId: "run-timeout" } as GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - const lifecycleEvent = requireRecord( - requireMockCallArgWithFields( - emitAgentEvent, - { runId: "run-timeout", sessionKey: "main", stream: "lifecycle" }, - "agent event", - ), - "agent event", - ); - expectRecordFields(lifecycleEvent, { - runId: "run-timeout", - sessionKey: "main", - stream: "lifecycle", - }); - const lifecycleData = requireRecord(lifecycleEvent.data, "lifecycle data"); - expectRecordFields(lifecycleData, { - phase: "end", - startedAt: 1_000, - aborted: true, - livenessState: "blocked", - replayInvalid: true, - }); - expect(typeof lifecycleData.endedAt).toBe("number"); - }); - - it("uses a rebound lifecycle generation for embedded terminal events", async () => { - const agentEvents = await import("../../infra/agent-events.js"); - const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - params.onExecutionStarted?.({ lifecycleGeneration: "post-restart" }); - await params.onAgentEvent?.({ - stream: "lifecycle", - data: { phase: "start", startedAt: 1_000 }, - }); - throw new Error("rebound failure"); - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { runId: "run-rebound" } as GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - const lifecycleEvents = emitAgentEvent.mock.calls - .map((call) => call[0]) - .filter( - (event) => - event.runId === "run-rebound" && - event.stream === "lifecycle" && - (event.data.phase === "error" || event.data.fallbackExhaustedFailure === true), - ); - expect(lifecycleEvents.length).toBeGreaterThan(0); - expect(lifecycleEvents).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - lifecycleGeneration: "post-restart", - }), - ]), - ); - expect(lifecycleEvents.every((event) => event.lifecycleGeneration === "post-restart")).toBe( - true, - ); - }); - - it("does not duplicate embedded lifecycle terminal events already reported by the runner", async () => { - const agentEvents = await import("../../infra/agent-events.js"); - const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "lifecycle", - data: { phase: "start", startedAt: 1_000 }, - }); - await params.onAgentEvent?.({ - stream: "lifecycle", - data: { phase: "end", endedAt: 1_500 }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { runId: "run-complete" } as GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - expectNoMockCallWithFields(emitAgentEvent, { - runId: "run-complete", - stream: "lifecycle", - }); - }); - - it("preserves GPT ack-turn final prose without reply-side truncation", async () => { - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - })); - state.runEmbeddedAgentMock.mockImplementationOnce(async () => ({ - payloads: [ - { - text: [ - "I updated the prompt overlay and tightened the runtime guard.", - "I also added the ack-turn fast path so short approvals skip the recap.", - "The reply-side output now keeps long prose-heavy GPT confirmations intact.", - "I updated tests for the overlay, retry guard, and reply normalization.", - "Everything is wired together and ready for verification.", - ].join(" "), - }, - ], - meta: {}, - })); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "openai"; - followupRun.run.model = "gpt-5.4"; - const result = await runAgentTurnWithFallback({ - commandBody: "ok do it", - followupRun, - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - if (result.kind === "success") { - expect(result.runResult.payloads?.[0]?.text).toBe( - [ - "I updated the prompt overlay and tightened the runtime guard.", - "I also added the ack-turn fast path so short approvals skip the recap.", - "The reply-side output now keeps long prose-heavy GPT confirmations intact.", - "I updated tests for the overlay, retry guard, and reply normalization.", - "Everything is wired together and ready for verification.", - ].join(" "), - ); - } - }); - - it("does not trim GPT replies when the user asked for depth", async () => { - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - })); - const longDetailedReply = [ - "Here is the detailed breakdown.", - "First, the runner now detects short approval turns and skips the recap path.", - "Second, the reply layer scores long prose-heavy GPT confirmations and trims them only in chat-style turns.", - "Third, code fences and richer structured outputs are left untouched so technical answers stay intact.", - "Finally, the overlay reinforces that this is a live chat and nudges the model toward short natural replies.", - ].join(" "); - state.runEmbeddedAgentMock.mockImplementationOnce(async () => ({ - payloads: [{ text: longDetailedReply }], - meta: {}, - })); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.provider = "openai"; - followupRun.run.model = "gpt-5.4"; - const result = await runAgentTurnWithFallback({ - commandBody: "explain in detail what changed", - followupRun, - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - if (result.kind === "success") { - expect(result.runResult.payloads?.[0]?.text).toBe(longDetailedReply); - } - }); - - it("forwards plan, approval, command output, and patch events", async () => { - const onPlanUpdate = vi.fn(); - const onApprovalEvent = vi.fn(); - const onCommandOutput = vi.fn(); - const onPatchSummary = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "plan", - data: { - phase: "update", - title: "Assistant proposed a plan", - explanation: "Inspect code, patch it, run tests.", - steps: ["Inspect code", "Patch code", "Run tests"], - }, - }); - await params.onAgentEvent?.({ - stream: "approval", - data: { - phase: "requested", - kind: "exec", - status: "pending", - title: "Command approval requested", - approvalId: "approval-1", - }, - }); - await params.onAgentEvent?.({ - stream: "command_output", - data: { - itemId: "command:exec-1", - phase: "delta", - title: "command ls", - toolCallId: "exec-1", - output: "README.md", - }, - }); - await params.onAgentEvent?.({ - stream: "patch", - data: { - itemId: "patch:patch-1", - phase: "end", - title: "apply patch", - toolCallId: "patch-1", - added: ["a.ts"], - modified: ["b.ts"], - deleted: [], - summary: "1 added, 1 modified", - }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const pendingToolTasks = new Set>(); - await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { - onPlanUpdate, - onApprovalEvent, - onCommandOutput, - onPatchSummary, - } satisfies GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks, - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(onPlanUpdate).toHaveBeenCalledWith({ - phase: "update", - title: "Assistant proposed a plan", - explanation: "Inspect code, patch it, run tests.", - steps: ["Inspect code", "Patch code", "Run tests"], - source: undefined, - }); - expect(onApprovalEvent).toHaveBeenCalledWith({ - phase: "requested", - kind: "exec", - status: "pending", - title: "Command approval requested", - itemId: undefined, - toolCallId: undefined, - approvalId: "approval-1", - approvalSlug: undefined, - command: undefined, - host: undefined, - reason: undefined, - scope: undefined, - message: undefined, - }); - expect(onCommandOutput).toHaveBeenCalledWith({ - itemId: "command:exec-1", - phase: "delta", - title: "command ls", - toolCallId: "exec-1", - name: undefined, - output: "README.md", - status: undefined, - exitCode: undefined, - durationMs: undefined, - cwd: undefined, - }); - expect(onPatchSummary).toHaveBeenCalledWith({ - itemId: "patch:patch-1", - phase: "end", - title: "apply patch", - toolCallId: "patch-1", - name: undefined, - added: ["a.ts"], - modified: ["b.ts"], - deleted: [], - summary: "1 added, 1 modified", - }); - }); - - it("forwards Codex command tool results as command output completion", async () => { - const onCommandOutput = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "tool", - data: { - phase: "result", - itemId: "command:exec-1", - toolCallId: "exec-1", - name: "exec", - status: "completed", - result: { - exitCode: 0, - durationMs: 42, - }, - }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onCommandOutput } satisfies GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(onCommandOutput).toHaveBeenCalledWith({ - itemId: "command:exec-1", - phase: "end", - title: undefined, - toolCallId: "exec-1", - name: "exec", - output: undefined, - status: "completed", - exitCode: 0, - durationMs: 42, - cwd: undefined, - }); - }); - - it("marks Codex command tool result errors as failed command output", async () => { - const onCommandOutput = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "tool", - data: { - phase: "result", - itemId: "command:exec-1", - toolCallId: "exec-1", - name: "exec", - isError: true, - result: { - content: [{ type: "text", text: "command failed" }], - }, - }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onCommandOutput } satisfies GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(onCommandOutput).toHaveBeenCalledWith( - expect.objectContaining({ - itemId: "command:exec-1", - phase: "end", - toolCallId: "exec-1", - name: "exec", - status: "failed", - }), - ); - }); - - it("does not synthesize command output from bare exec tool results", async () => { - const onCommandOutput = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "tool", - data: { - phase: "result", - name: "exec", - toolCallId: "exec-1", - isError: false, - }, - }); - await params.onAgentEvent?.({ - stream: "command_output", - data: { - itemId: "command:exec-1", - phase: "end", - title: "command ls", - toolCallId: "exec-1", - name: "exec", - status: "completed", - exitCode: 0, - }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onCommandOutput } satisfies GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(onCommandOutput).toHaveBeenCalledTimes(1); - expect(onCommandOutput).toHaveBeenCalledWith( - expect.objectContaining({ - itemId: "command:exec-1", - phase: "end", - status: "completed", - }), - ); - }); - - it("suppresses progress callbacks after message-tool-only delivery completes", async () => { - let releaseItemEvent: (() => void) | undefined; - const itemEventGate = new Promise((resolve) => { - releaseItemEvent = resolve; - }); - let markItemEventStarted: (() => void) | undefined; - const itemEventStarted = new Promise((resolve) => { - markItemEventStarted = resolve; - }); - const onItemEvent = vi.fn(async () => { - markItemEventStarted?.(); - await itemEventGate; - }); - const onCommandOutput = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "tool", - data: { - phase: "start", - name: "message", - toolCallId: "message-1", - args: { - action: "send", - message: "Visible reply", - }, - }, - }); - const itemEventPromise = params.onAgentEvent?.({ - stream: "item", - data: { - itemId: "tool-message-1", - phase: "end", - kind: "tool", - title: "message", - name: "message", - toolCallId: "message-1", - status: "completed", - }, - }); - await itemEventStarted; - await params.onAgentEvent?.({ - stream: "command_output", - data: { - itemId: "command:exec-1", - phase: "end", - title: "command false", - toolCallId: "exec-1", - name: "exec", - output: "failed command output", - status: "failed", - exitCode: 1, - }, - }); - await params.onAgentEvent?.({ - stream: "assistant", - data: { - phase: "commentary", - itemId: "commentary-1", - text: "This must stay suppressed.", - }, - }); - releaseItemEvent?.(); - await itemEventPromise; - return { payloads: [{ text: "NO_REPLY" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.sourceReplyDeliveryMode = "message_tool_only"; - await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "discord", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { - onItemEvent, - onCommandOutput, - progressPreambleEnabled: true, - } satisfies InternalGetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "on", - }); - - expect(onItemEvent).toHaveBeenCalledWith( - expect.objectContaining({ - name: "message", - phase: "end", - status: "completed", - }), - ); - expect(onItemEvent).toHaveBeenCalledTimes(1); - expect(onCommandOutput).not.toHaveBeenCalled(); - }); - - it("preserves message-tool-only suppression across fallback candidates", async () => { - const onItemEvent = vi.fn(); - const onCommandOutput = vi.fn(); - state.runEmbeddedAgentMock - .mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "tool", - data: { - phase: "start", - name: "message", - toolCallId: "message-1", - args: { action: "send", message: "Visible reply" }, - }, - }); - await params.onAgentEvent?.({ - stream: "item", - data: { - itemId: "tool-message-1", - phase: "end", - kind: "tool", - name: "message", - toolCallId: "message-1", - status: "completed", - }, - }); - return { payloads: [], meta: {} }; - }) - .mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "command_output", - data: { - itemId: "command:exec-1", - phase: "end", - name: "exec", - output: "must stay suppressed", - status: "completed", - exitCode: 0, - }, - }); - return { payloads: [{ text: "NO_REPLY" }], meta: {} }; - }); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - await params.run("anthropic", "primary"); - return { - result: await params.run("openai", "fallback"), - provider: "openai", - model: "fallback", - attempts: [], - }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.sourceReplyDeliveryMode = "message_tool_only"; - await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { Provider: "discord", MessageSid: "msg" } as unknown as TemplateContext, - opts: { onItemEvent, onCommandOutput } satisfies GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "on", - }); - - expect(onItemEvent).toHaveBeenCalledTimes(1); - expect(onCommandOutput).not.toHaveBeenCalled(); - }); - - it("keeps opted-in progress callbacks active after message-tool-only delivery completes", async () => { - const onToolStart = vi.fn(); - const onCommandOutput = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "tool", - data: { - phase: "start", - name: "message", - toolCallId: "message-1", - args: { - action: "send", - message: "Visible reply", - }, - }, - }); - await params.onAgentEvent?.({ - stream: "item", - data: { - itemId: "tool-message-1", - phase: "end", - kind: "tool", - title: "message", - name: "message", - toolCallId: "message-1", - status: "completed", - }, - }); - await params.onAgentEvent?.({ - stream: "tool", - data: { - phase: "start", - name: "bash", - toolCallId: "bash-1", - args: { - command: "sleep 6", - }, - }, - }); - await params.onAgentEvent?.({ - stream: "command_output", - data: { - itemId: "command:bash-1", - phase: "end", - title: "sleep 6", - toolCallId: "bash-1", - name: "bash", - output: "done", - status: "completed", - exitCode: 0, - }, - }); - return { payloads: [{ text: "NO_REPLY" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.sourceReplyDeliveryMode = "message_tool_only"; - await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "discord", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { - allowProgressCallbacksWhenSourceDeliverySuppressed: true, - onToolStart, - onCommandOutput, - } satisfies GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "on", - }); - - expect(onToolStart).toHaveBeenCalledWith( - expect.objectContaining({ - name: "bash", - phase: "start", - args: { command: "sleep 6" }, - detailMode: undefined, - }), - ); - expect(onCommandOutput).toHaveBeenCalledWith( - expect.objectContaining({ - name: "bash", - output: "done", - status: "completed", - }), - ); - }); - - it("keeps progress callbacks active after message-tool-only reads", async () => { - const onItemEvent = vi.fn(); - const onCommandOutput = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "tool", - data: { - phase: "start", - name: "message", - toolCallId: "message-read-1", - args: { - action: "read", - threadId: "thread-1", - }, - }, - }); - await params.onAgentEvent?.({ - stream: "item", - data: { - itemId: "tool-message-1", - phase: "end", - kind: "tool", - title: "message", - name: "message", - toolCallId: "message-read-1", - status: "completed", - }, - }); - await params.onAgentEvent?.({ - stream: "command_output", - data: { - itemId: "command:exec-1", - phase: "end", - title: "command false", - toolCallId: "exec-1", - name: "exec", - output: "failed command output", - status: "failed", - exitCode: 1, - }, - }); - return { payloads: [{ text: "NO_REPLY" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - followupRun.run.sourceReplyDeliveryMode = "message_tool_only"; - await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "discord", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { - onItemEvent, - onCommandOutput, - } satisfies GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "on", - }); - - expect(onItemEvent).toHaveBeenCalledWith( - expect.objectContaining({ - name: "message", - phase: "end", - status: "completed", - }), - ); - expect(onCommandOutput).toHaveBeenCalledWith( - expect.objectContaining({ - output: "failed command output", - status: "failed", - }), - ); - }); - - it("keeps compaction start notices silent by default", async () => { - const onBlockReply = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onBlockReply }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - expect(onBlockReply).not.toHaveBeenCalled(); - }); - - it("keeps compaction callbacks active when notices are silent by default", async () => { - const onBlockReply = vi.fn(); - const onCompactionStart = vi.fn(); - const onCompactionEnd = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); - await params.onAgentEvent?.({ - stream: "compaction", - data: { phase: "end", completed: true }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { - onBlockReply, - onCompactionStart, - onCompactionEnd, - }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - expect(onCompactionStart).toHaveBeenCalledTimes(1); - expect(onCompactionEnd).toHaveBeenCalledTimes(1); - expect(onBlockReply).not.toHaveBeenCalled(); - }); - - it("logs Codex app-server compaction completion while notices stay silent by default", async () => { - const onBlockReply = vi.fn(); - const consoleLog = vi.fn(); - setLoggerOverride({ level: "silent", consoleLevel: "info", consoleStyle: "compact" }); - loggingState.rawConsole = { - log: consoleLog, - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - }; - try { - state.runWithModelFallbackMock.mockImplementationOnce( - async (params: FallbackRunnerParams) => ({ - result: await params.run("openai", "gpt-5.5"), - provider: "openai", - model: "gpt-5.5", - attempts: [{ provider: "anthropic", model: "claude", error: "rate limit" }], - }), - ); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "compaction", - data: { - phase: "start", - backend: "codex-app-server", - threadId: "thread-1", - turnId: "turn-1", - itemId: "compaction-1", - }, - }); - await params.onAgentEvent?.({ - stream: "compaction", - data: { - phase: "end", - completed: true, - backend: "codex-app-server", - threadId: "thread-1", - turnId: "turn-1", - itemId: "compaction-1", - }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ - opts: { onBlockReply }, - }), - }); - - expect(result.kind).toBe("success"); - expect(onBlockReply).not.toHaveBeenCalled(); - expect(consoleLog.mock.calls.map(([line]) => String(line)).join("\n")).toContain( - "codex app-server auto-compaction succeeded for openai/gpt-5.5; refreshed session context", - ); - } finally { - loggingState.rawConsole = null; - setLoggerOverride(null); - resetLogger(); - } - }); - - it("emits a compaction start notice when notifyUser is enabled", async () => { - const onBlockReply = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const followupRun = createFollowupRun(); - followupRun.run.config = { - agents: { - defaults: { - compaction: { - notifyUser: true, - }, - }, - }, - }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onBlockReply }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - expect(onBlockReply).toHaveBeenCalledTimes(1); - expectBlockReplyCall(onBlockReply, 0, { - text: "🧹 Compacting context...", - replyToId: "msg", - replyToCurrent: true, - isCompactionNotice: true, - }); - }); - - it("emits a compaction completion notice when notifyUser is enabled", async () => { - const onBlockReply = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); - await params.onAgentEvent?.({ - stream: "compaction", - data: { phase: "end", completed: true }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const followupRun = createFollowupRun(); - followupRun.run.config = { - agents: { - defaults: { - compaction: { - notifyUser: true, - }, - }, - }, - }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onBlockReply }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - expectBlockReplyCall(onBlockReply, 0, { - text: "🧹 Compacting context...", - replyToId: "msg", - replyToCurrent: true, - isCompactionNotice: true, - }); - expectBlockReplyCall(onBlockReply, 1, { - text: "🧹 Compaction complete", - replyToId: "msg", - replyToCurrent: true, - isCompactionNotice: true, - }); - }); - - it("delivers compaction hook messages alongside notifyUser notices (#90185)", async () => { - const onBlockReply = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ - stream: "compaction", - data: { phase: "start", messages: ["Hook before"] }, - }); - await params.onAgentEvent?.({ - stream: "compaction", - data: { phase: "end", completed: true, messages: ["Hook after"] }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const followupRun = createFollowupRun(); - followupRun.run.config = { - agents: { - defaults: { - compaction: { - notifyUser: true, - }, - }, - }, - }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onBlockReply }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - expect(onBlockReply).toHaveBeenCalledTimes(4); - expectBlockReplyCall(onBlockReply, 0, { - text: "Hook before", - replyToId: "msg", - replyToCurrent: true, - isCompactionNotice: true, - }); - expectBlockReplyCall(onBlockReply, 1, { - text: "🧹 Compacting context...", - replyToId: "msg", - replyToCurrent: true, - isCompactionNotice: true, - }); - expectBlockReplyCall(onBlockReply, 2, { - text: "Hook after", - replyToId: "msg", - replyToCurrent: true, - isCompactionNotice: true, - }); - expectBlockReplyCall(onBlockReply, 3, { - text: "🧹 Compaction complete", - replyToId: "msg", - replyToCurrent: true, - isCompactionNotice: true, - }); - }); - - it("fires both notifyUser notices alongside onCompactionStart / onCompactionEnd callbacks (#87107)", async () => { - const onBlockReply = vi.fn(); - const onCompactionStart = vi.fn(); - const onCompactionEnd = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); - await params.onAgentEvent?.({ - stream: "compaction", - data: { phase: "end", completed: true }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const followupRun = createFollowupRun(); - followupRun.run.config = { - agents: { - defaults: { - compaction: { - notifyUser: true, - }, - }, - }, - }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onBlockReply, onCompactionStart, onCompactionEnd }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - // Internal callbacks (Control UI etc.) and the user-channel notifyUser - // notices are independent audiences; both must fire when opted in. - expect(onCompactionStart).toHaveBeenCalledTimes(1); - expect(onCompactionEnd).toHaveBeenCalledTimes(1); - expect(onBlockReply).toHaveBeenCalledTimes(2); - expectBlockReplyCall(onBlockReply, 0, { - text: "🧹 Compacting context...", - isCompactionNotice: true, - }); - expectBlockReplyCall(onBlockReply, 1, { - text: "🧹 Compaction complete", - isCompactionNotice: true, - }); - }); - - it("emits an incomplete compaction notice when compaction ends without completing", async () => { - const onBlockReply = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); - await params.onAgentEvent?.({ - stream: "compaction", - data: { phase: "end", completed: false }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const followupRun = createFollowupRun(); - followupRun.run.config = { - agents: { - defaults: { - compaction: { - notifyUser: true, - }, - }, - }, - }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { onBlockReply }, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - expectBlockReplyCall(onBlockReply, 0, { - text: "🧹 Compacting context...", - isCompactionNotice: true, - }); - expectBlockReplyCall(onBlockReply, 1, { - text: "🧹 Compaction incomplete", - isCompactionNotice: true, - }); - }); - - it("uses the compaction notice fallback when no block-reply dispatcher is wired", async () => { - const onCompactionNoticePayload = vi.fn(); - state.runEmbeddedAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => { - await params.onAgentEvent?.({ stream: "compaction", data: { phase: "start" } }); - await params.onAgentEvent?.({ - stream: "compaction", - data: { phase: "end", completed: true }, - }); - return { payloads: [{ text: "final" }], meta: {} }; - }); - - const followupRun = createFollowupRun(); - followupRun.run.config = { - agents: { - defaults: { - compaction: { - notifyUser: true, - }, - }, - }, - }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - onCompactionNoticePayload, - }); - - expect(result.kind).toBe("success"); - expect(onCompactionNoticePayload).toHaveBeenCalledTimes(2); - expectBlockReplyCall(onCompactionNoticePayload, 0, { - text: "🧹 Compacting context...", - replyToId: "msg", - replyToCurrent: true, - isCompactionNotice: true, - }); - expectBlockReplyCall(onCompactionNoticePayload, 1, { - text: "🧹 Compaction complete", - replyToId: "msg", - replyToCurrent: true, - isCompactionNotice: true, - }); - }); - - it("surfaces billing guidance for mixed-cause fallback exhaustion", async () => { - state.runWithModelFallbackMock.mockRejectedValueOnce( - Object.assign( - new Error( - "All models failed (2): anthropic/claude: 429 (rate_limit) | openai/gpt-5.4: 402 (billing)", - ), - { - name: "FallbackSummaryError", - attempts: [ - { provider: "anthropic", model: "claude", error: "429", reason: "rate_limit" }, - { provider: "openai", model: "gpt-5.4", error: "402", reason: "billing" }, - ], - soonestCooldownExpiry: Date.now() + 60_000, - }, - ), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe("billing"); - expect(result.payload.text).not.toContain("All models failed"); - expect(result.payload.text).not.toContain("402 (billing)"); - expect(result.payload.text).not.toContain("Rate-limited"); - } - }); - - it("surfaces Codex usage-limit reset details for pure fallback exhaustion", async () => { - const codexMessage = - "You've reached your Codex subscription usage limit. Next reset in 42 minutes (2026-05-04T21:34:00.000Z). Run /codex account for current usage details."; - state.runWithModelFallbackMock.mockRejectedValueOnce( - Object.assign(new Error(`All models failed (1): openai/gpt-5.5: ${codexMessage}`), { - name: "FallbackSummaryError", - attempts: [ - { - provider: "openai", - model: "gpt-5.5", - error: codexMessage, - reason: "rate_limit", - }, - ], - soonestCooldownExpiry: null, - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "telegram", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe(`⚠️ ${codexMessage}`); - expect(result.payload.text).not.toContain("All models failed"); - expectRecordFields(requireRecord(getReplyPayloadMetadata(result.payload), "reply metadata"), { - deliverDespiteSourceReplySuppression: true, - }); - } - }); - - it("surfaces direct Codex usage-limit errors when fallback does not wrap one attempt", async () => { - const codexMessage = - "You've reached your Codex subscription usage limit. Codex did not return a reset time for this limit. Run /codex account for current usage details."; - state.runWithModelFallbackMock.mockRejectedValueOnce(new Error(codexMessage)); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "telegram", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe(`⚠️ ${codexMessage}`); - expectRecordFields(requireRecord(getReplyPayloadMetadata(result.payload), "reply metadata"), { - deliverDespiteSourceReplySuppression: true, - }); - } - }); - - it("surfaces billing guidance for pure billing cooldown fallback exhaustion", async () => { - state.runWithModelFallbackMock.mockRejectedValueOnce( - Object.assign( - new Error( - "All models failed (2): anthropic/claude-opus-4-6: Provider anthropic has billing issue (skipping all models) (billing) | anthropic/claude-sonnet-4-6: Provider anthropic has billing issue (skipping all models) (billing)", - ), - { - name: "FallbackSummaryError", - attempts: [ - { - provider: "anthropic", - model: "claude-opus-4-6", - error: "Provider anthropic has billing issue (skipping all models)", - reason: "billing", - }, - { - provider: "anthropic", - model: "claude-sonnet-4-6", - error: "Provider anthropic has billing issue (skipping all models)", - reason: "billing", - }, - ], - soonestCooldownExpiry: Date.now() + 60_000, - }, - ), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe("billing"); - } - }); - - it("surfaces restart text when fallback exhaustion wraps a drain error, keeping fail bookkeeping", async () => { - const { replyOperation, failMock } = createMockReplyOperation(); - state.runWithModelFallbackMock.mockRejectedValueOnce( - Object.assign(new Error("fallback exhausted"), { - name: "FallbackSummaryError", - attempts: [ - { - provider: "anthropic", - model: "claude", - error: new GatewayDrainingError(), - }, - ], - soonestCooldownExpiry: null, - cause: new GatewayDrainingError(), - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - replyOperation, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - "⚠️ Gateway is restarting. Please wait a few seconds and try again.", - ); - } - const failCall = requireMockCall(failMock, 0, "reply operation fail"); - expect(failCall[0]).toBe("gateway_draining"); - expect(failCall[1]).toBeInstanceOf(GatewayDrainingError); - }); - - it("surfaces restart text when fallback exhaustion wraps a cleared lane error, keeping fail bookkeeping", async () => { - const { replyOperation, failMock } = createMockReplyOperation(); - state.runWithModelFallbackMock.mockRejectedValueOnce( - Object.assign(new Error("fallback exhausted"), { - name: "FallbackSummaryError", - attempts: [ - { - provider: "anthropic", - model: "claude", - error: new CommandLaneClearedError("session:main"), - }, - ], - soonestCooldownExpiry: null, - cause: new CommandLaneClearedError("session:main"), - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - replyOperation, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - "⚠️ Gateway is restarting. Please wait a few seconds and try again.", - ); - } - const failCall = requireMockCall(failMock, 0, "reply operation fail"); - expect(failCall[0]).toBe("command_lane_cleared"); - expect(failCall[1]).toBeInstanceOf(CommandLaneClearedError); - }); - - it("stays silent (NO_REPLY) when the reply operation was aborted for restart", async () => { - const agentEvents = await import("../../infra/agent-events.js"); - const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); - const { replyOperation, failMock } = createMockReplyOperation(); - Object.defineProperty(replyOperation, "result", { - value: { kind: "aborted", code: "aborted_for_restart" } as const, - configurable: true, - }); - state.runEmbeddedAgentMock.mockRejectedValueOnce( - Object.assign(new Error("aborted"), { name: "AbortError" }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - replyOperation, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - isRestartRecoveryArmed: () => true, - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe(SILENT_REPLY_TOKEN); - } - expect(failMock).not.toHaveBeenCalled(); - expect( - emitAgentEvent.mock.calls.some( - ([event]) => - event.stream === "lifecycle" && - event.data.phase === "end" && - event.data.aborted === true && - event.data.stopReason === "restart", - ), - ).toBe(true); - }); - - it("preserves restart ownership when an aborted embedded runner resolves normally", async () => { - const agentEvents = await import("../../infra/agent-events.js"); - const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); - const { replyOperation } = createMockReplyOperation(); - Object.defineProperty(replyOperation, "result", { - value: { kind: "aborted", code: "aborted_for_restart" } as const, - configurable: true, - }); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - replyOperation, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - isRestartRecoveryArmed: () => true, - }); - - expect(result).toEqual({ - kind: "final", - payload: expect.objectContaining({ - text: SILENT_REPLY_TOKEN, - }), - }); - expect( - emitAgentEvent.mock.calls.some( - ([event]) => - event.stream === "lifecycle" && - event.data.phase === "end" && - event.data.aborted === true && - event.data.stopReason === "restart", - ), - ).toBe(true); - }); - - it("uses compact generic copy for raw external chat errors when verbose is off", async () => { - const agentEvents = await import("../../infra/agent-events.js"); - const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent); - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error("INVALID_ARGUMENT: some other failure"), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: { runId: "run-provider-failure" } as GetReplyOptions, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe(GENERIC_RUN_FAILURE_TEXT); - } - const terminalFailureEvent = emitAgentEvent.mock.calls - .map((call) => call[0]) - .find((event) => { - if (!event || typeof event !== "object") { - return false; - } - const data = (event as { data?: Record }).data; - return ( - (event as { runId?: unknown }).runId === "run-provider-failure" && - (event as { stream?: unknown }).stream === "lifecycle" && - data?.phase === "error" && - data.fallbackExhaustedFailure === true - ); - }); - expect(terminalFailureEvent).toBeDefined(); - }); - - it("surfaces CLI max-turn recovery context at normal verbosity", async () => { - const recoveryText = - "Claude CLI stopped after reaching the maximum number of turns (limit: 1). " + - "OpenClaw run: run-max-turns. OpenClaw session: session-1. Claude session: claude-session-1. " + - "Tool actions may already have run; verify their effects before retrying. " + - "Retry with a higher --max-turns value or a narrower task."; - const maxTurns = new FailoverError(recoveryText, { - reason: "unknown", - code: "cli_max_turns", - provider: "claude-cli", - model: "sonnet", - }); - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new AggregateError( - [maxTurns, new Error("fork successor persistence failed")], - "CLI turn failed and its fork successor could not be persisted", - ), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.isError).toBe(true); - expect(result.payload.text).toBe(recoveryText); - expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); - } - }); - - it("uses heartbeat failure copy for raw external errors during heartbeat runs", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error('Command lane "main" task timed out after 120000ms'), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams(), - isHeartbeat: true, - }); - - expect(result.kind).toBe("final"); - if (result.kind !== "final") { - throw new Error("expected final reply"); - } - expect(result.payload.text).toBe(HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT); - expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); - expect(result.payload.text).not.toContain("/new"); - }); - - it.each([ - { - rejection: new Error("CLI exceeded timeout (300s) and was terminated."), - modeLabel: "overall CLI turn budget" as const, - routingSubstring: undefined as string | undefined, - }, - { - rejection: new Error("CLI produced no output for 120s and was terminated."), - modeLabel: "no-output stall" as const, - routingSubstring: undefined, - }, - { - rejection: new Error( - "All models failed (2): anthropic/claude-opus-4-7: CLI exceeded timeout (300s) and was terminated. | anthropic/foo: bar", - ), - modeLabel: "overall CLI turn budget" as const, - routingSubstring: "(routing anthropic/claude-opus-4-7)", - }, - { - rejection: new Error("codex-cli/gpt-5.5: CLI exceeded timeout (60s) and was terminated."), - modeLabel: "overall CLI turn budget" as const, - routingSubstring: "(routing codex-cli/gpt-5.5)", - }, - ])( - "surfaces CLI subprocess timeout copy instead of generic failure when verbose is off ($modeLabel)", - async ({ rejection, modeLabel, routingSubstring }) => { - state.runWithModelFallbackMock.mockRejectedValueOnce(rejection); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams(), - }); - - expect(result.kind).toBe("final"); - if (result.kind !== "final") { - throw new Error("expected final reply"); - } - expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); - expect(result.payload.text).toContain("CLI subprocess"); - expect(result.payload.text).not.toContain("Claude CLI"); - expect(result.payload.text).toContain(modeLabel); - expect(result.payload.text).toContain("gateway may still be healthy"); - expect(result.payload.text).toContain("cliBackends."); - if (routingSubstring) { - expect(result.payload.text).toContain(routingSubstring); - } - }, - ); - - it.each([ - { - rejection: new Error("codex app-server client closed before turn completed"), - expected: "connection closed", - }, - { - rejection: new Error("codex app-server turn idle timed out waiting for turn/completed"), - expected: "did not replay the turn automatically", - }, - ])( - "surfaces Codex app-server bridge failures instead of generic copy", - async ({ rejection, expected }) => { - state.runWithModelFallbackMock.mockRejectedValueOnce(rejection); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams(), - }); - - expect(result.kind).toBe("final"); - if (result.kind !== "final") { - throw new Error("expected final reply"); - } - expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); - expect(result.payload.text).toContain("Codex app-server"); - expect(result.payload.text).toContain(expected); - }, - ); - - it("forwards sanitized generic errors on external chat channels when verbose is on", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error("INVALID_ARGUMENT: some other failure"), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "on", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - "⚠️ Agent failed before reply: INVALID_ARGUMENT: some other failure. Please try again, or use /new to start a fresh session.", - ); - } - }); - - it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( - "keeps raw runner failure boilerplate out of $label chats", - async (testCase) => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error("openai/gpt-5.5 ended with an incomplete terminal response"), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: createNonDirectFailureSessionCtx(testCase), - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe(SILENT_REPLY_TOKEN); - } - }, - ); - - it.each(["group", "channel"] as const)( - "surfaces raw runner failure copy in Discord %s chats when silentReply.group is set to disallow", - async (chatType) => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error("openai/gpt-5.5 ended with an incomplete terminal response"), - ); - - const followupRun = createFollowupRun(); - followupRun.run.config = { - agents: { - defaults: { - silentReply: { group: "disallow" }, - }, - }, - }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - followupRun, - sessionCtx: { - Provider: "discord", - Surface: "discord", - ChatType: chatType, - GroupSubject: "agent group", - GroupChannel: "#general", - MessageSid: "msg", - } as unknown as TemplateContext, - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); - expect(result.payload.text).toBe(GENERIC_RUN_FAILURE_TEXT); - } - }, - ); - - it("surfaces raw runner failure copy when per-surface silentReply.group is set to disallow", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error("openai/gpt-5.5 ended with an incomplete terminal response"), - ); - - const followupRun = createFollowupRun(); - followupRun.run.config = { - agents: { - defaults: { - silentReply: { group: "allow" }, - }, - }, - surfaces: { - discord: { - silentReply: { group: "disallow" }, - }, - }, - }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - followupRun, - sessionCtx: { - Provider: "discord", - Surface: "discord", - ChatType: "group", - GroupSubject: "agent group", - GroupChannel: "#general", - MessageSid: "msg", - } as unknown as TemplateContext, - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe(GENERIC_RUN_FAILURE_TEXT); - } - }); - - it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( - "keeps default silent behavior in $label chats when silentReply policy is unset", - async (testCase) => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error("openai/gpt-5.5 ended with an incomplete terminal response"), - ); - - const followupRun = createFollowupRun(); - followupRun.run.config = {}; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - followupRun, - sessionCtx: createNonDirectFailureSessionCtx(testCase), - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe(SILENT_REPLY_TOKEN); - } - }, - ); - - it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( - "keeps classified non-transient failures visible in $label chats", - async (testCase) => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error('No API key found for provider "openai"'), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: createNonDirectFailureSessionCtx(testCase), - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); - expect(result.payload.text).toContain('Missing API key for provider "openai"'); - } - }, - ); - - it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( - "surfaces provider authentication failures in $label chats", - async (testCase) => { - const rawError = - "unexpected status 401 Unauthorized: Missing bearer or basic authentication in header, url: https://api.openai.com/v1/responses"; - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new FailoverError("LLM request unauthorized.", { - reason: "auth", - provider: "openai", - model: "gpt-5.5", - status: 401, - rawError, - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: createNonDirectFailureSessionCtx(testCase), - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.isError).toBe(true); - expect(result.payload.text).toBe(PROVIDER_AUTHENTICATION_ERROR_USER_MESSAGE); - expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); - expect(result.payload.text).not.toContain(rawError); - } - }, - ); - - it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( - "surfaces rate-limit fallback copy in $label chats", - async (testCase) => { - state.runEmbeddedAgentMock.mockRejectedValueOnce(new Error("429 rate limit exceeded")); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: createNonDirectFailureSessionCtx(testCase), - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.isError).toBe(true); - expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); - expect(result.payload.text).toContain("rate-limited"); - } - }, - ); - - it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( - "surfaces typed periodic rate-limit details in $label chats", - async (testCase) => { - const periodicLimitMessage = "You've hit your weekly limit · resets 6pm (UTC)"; - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new FailoverError(periodicLimitMessage, { - reason: "rate_limit", - provider: "anthropic", - model: "claude-opus-4-1", - rawError: periodicLimitMessage, - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: createNonDirectFailureSessionCtx(testCase), - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.isError).toBe(true); - expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); - expect(result.payload.text).toContain("weekly limit"); - expect(result.payload.text).toContain("resets 6pm"); - expect(result.payload.text).not.toContain("few minutes"); - } - }, - ); - - it("surfaces typed periodic rate-limit details through known failure payloads in group chats", () => { - const periodicLimitMessage = "You've hit your weekly limit · resets 6pm (UTC)"; - const payload = buildKnownAgentRunFailureReplyPayload({ - err: new FailoverError(periodicLimitMessage, { - reason: "rate_limit", - provider: "anthropic", - model: "claude-opus-4-1", - rawError: periodicLimitMessage, - }), - sessionCtx: createNonDirectFailureSessionCtx(NON_DIRECT_FAILURE_SURFACE_CASES[0]), - resolvedVerboseLevel: "off", - }); - - expect(payload).toBeDefined(); - expect(payload?.isError).toBe(true); - expect(payload?.text).not.toBe(SILENT_REPLY_TOKEN); - expect(payload?.text).toContain("weekly limit"); - expect(payload?.text).toContain("resets 6pm"); - expect(payload?.text).not.toContain("few minutes"); - }); - - it.each(NON_DIRECT_FAILURE_SURFACE_CASES)( - "surfaces overloaded fallback copy in $label chats", - async (testCase) => { - state.runEmbeddedAgentMock.mockRejectedValueOnce(new Error("model is overloaded")); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: createNonDirectFailureSessionCtx(testCase), - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.isError).toBe(true); - expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); - expect(result.payload.text).toContain("overloaded"); - } - }, - ); - - it("surfaces typed overloaded failures without rate-limit cooldown copy", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new FailoverError("529 Please try again", { - reason: "overloaded", - provider: "anthropic", - model: "claude-opus-4-1", - status: 529, - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: createNonDirectFailureSessionCtx(NON_DIRECT_FAILURE_SURFACE_CASES[0]), - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.isError).toBe(true); - expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); - expect(result.payload.text).toContain("overloaded"); - expect(result.payload.text).not.toContain("rate-limited"); - expect(result.payload.text).not.toContain("few minutes"); - } - }); - - it("surfaces rate-limit fallback copy in Discord group chats when silentReply.group is disallow", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce(new Error("429 rate limit exceeded")); - - const followupRun = createFollowupRun(); - followupRun.run.config = { - agents: { - defaults: { - silentReply: { group: "disallow" }, - }, - }, - }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - followupRun, - sessionCtx: { - Provider: "discord", - Surface: "discord", - ChatType: "group", - GroupSubject: "agent group", - GroupChannel: "#general", - MessageSid: "msg", - } as unknown as TemplateContext, - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.isError).toBe(true); - expect(result.payload.text).not.toBe(SILENT_REPLY_TOKEN); - expect(result.payload.text).toContain("rate-limited"); - } - }); - - it("uses compact generic copy for raw runner failures in normal Discord direct chats", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error("openai/gpt-5.5 ended with an incomplete terminal response"), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: { - Provider: "discord", - Surface: "discord", - ChatType: "direct", - MessageSid: "msg", - } as unknown as TemplateContext, - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe(GENERIC_RUN_FAILURE_TEXT); - } - }); - - it("keeps raw runner failure guidance visible in verbose Discord direct chats", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error("openai/gpt-5.5 ended with an incomplete terminal response"), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ - sessionCtx: { - Provider: "discord", - Surface: "discord", - ChatType: "direct", - MessageSid: "msg", - } as unknown as TemplateContext, - }), - resolvedVerboseLevel: "on", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toContain("Agent failed before reply"); - expect(result.payload.text).toContain("incomplete terminal response"); - } - }); - - it("surfaces provider quota guidance for generic HTTP 429 failures before reply", async () => { - const error = new Error( - "Something went wrong while processing your request. Please try again.", - ); - Object.assign(error, { status: 429 }); - state.runEmbeddedAgentMock.mockRejectedValueOnce(error); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: { - Provider: "discord", - Surface: "discord", - ChatType: "direct", - MessageSid: "msg", - } as unknown as TemplateContext, - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe(PROVIDER_RATE_LIMIT_OR_QUOTA_ERROR_USER_MESSAGE); - expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); - } - }); - - it("surfaces provider internal errors without session reset guidance before reply", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new FailoverError( - "The AI service returned an internal error. Please try again in a moment.", - { - reason: "server_error", - provider: "fyapis", - model: "gpt-5.5", - status: 500, - }, - ), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: { - Provider: "telegram", - Surface: "telegram", - ChatType: "direct", - MessageSid: "msg", - } as unknown as TemplateContext, - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe(PROVIDER_INTERNAL_ERROR_USER_MESSAGE); - expect(result.payload.text).not.toContain("/new"); - expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); - } - }); - - it("surfaces billing guidance for Volcengine Coding Plan subscription failures before reply", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error( - 'HTTP 400 Bad Request: {"error":{"code":"InvalidSubscription","message":"Your account does not have a valid CodingPlan subscription, or your subscription has expired."}}', - ), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: { - Provider: "discord", - Surface: "discord", - ChatType: "direct", - MessageSid: "msg", - } as unknown as TemplateContext, - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe("billing"); - expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); - } - }); - - it("preserves neutral billing guidance for OAuth failover errors", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new FailoverError(formatBillingErrorMessage("Anthropic", "claude-sonnet-4-5", "oauth"), { - reason: "billing", - provider: "Anthropic", - model: "claude-sonnet-4-5", - authMode: "oauth", - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toContain("check your account for subscription or usage limits"); - expect(result.payload.text).not.toContain("API key"); - expect(result.payload.text).not.toContain("top up"); - } - }); - - it("preserves neutral billing guidance after fallback exhaustion", async () => { - state.runWithModelFallbackMock.mockRejectedValueOnce( - Object.assign(new Error("All models failed (1): openai/gpt-5.5: billing"), { - name: "FallbackSummaryError", - attempts: [ - { - provider: "openai", - model: "gpt-5.5", - error: "billing", - reason: "billing", - authMode: "oauth", - }, - ], - soonestCooldownExpiry: null, - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toContain("check your account for subscription or usage limits"); - expect(result.payload.text).not.toContain("API key"); - expect(result.payload.text).not.toContain("top up"); - } - }); - - it("formats raw Codex API payloads before forwarding verbose external errors", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error( - 'Codex error: {"type":"error","error":{"type":"server_error","message":"Something exploded"},"sequence_number":2}', - ), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "on", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - "⚠️ Agent failed before reply: LLM error server_error: Something exploded. Please try again, or use /new to start a fresh session.", - ); - } - }); - - it("preserves the active session when embedded overflow recovery fails", async () => { - state.isContextOverflowErrorMock.mockReturnValue(true); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [], - meta: { - error: { - message: "400 The prompt is too long: 203557, model maximum context length: 196607", - }, - }, - }); - - const activeSessionEntry = { sessionId: "session", updatedAt: 1 } as SessionEntry; - const activeSessionStore = { "agent:main:main": activeSessionEntry }; - const { replyOperation, failMock, updateSessionIdMock } = createMockReplyOperation(); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ - sessionCtx: { - Provider: "webchat", - MessageSid: "msg", - } as unknown as TemplateContext, - }), - replyOperation, - sessionKey: "agent:main:main", - getActiveSessionEntry: () => activeSessionEntry, - activeSessionStore, - storePath: "/tmp/sessions.json", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toContain("kept this conversation mapped to the current session"); - expect(result.payload.text).toContain("reserveTokensFloor"); - expectRecordFields(requireRecord(getReplyPayloadMetadata(result.payload), "reply metadata"), { - deliverDespiteSourceReplySuppression: true, - }); - } - expect(failMock).toHaveBeenCalledWith( - "run_failed", - expect.objectContaining({ - message: "400 The prompt is too long: 203557, model maximum context length: 196607", - }), - ); - expect(activeSessionStore["agent:main:main"]?.sessionId).toBe("session"); - expect(updateSessionIdMock).not.toHaveBeenCalled(); - expect(state.updateSessionStoreMock).not.toHaveBeenCalled(); - }); - - it("preserves the active session when compaction failure is thrown before reply", async () => { - state.isCompactionFailureErrorMock.mockReturnValue(true); - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error("Auto-compaction failed: nothing to compact"), - ); - - const activeSessionEntry = { sessionId: "session", updatedAt: 1 } as SessionEntry; - const activeSessionStore = { "agent:main:main": activeSessionEntry }; - const { replyOperation, failMock, updateSessionIdMock } = createMockReplyOperation(); - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - ...createMinimalRunAgentTurnParams({ - sessionCtx: { - Provider: "webchat", - MessageSid: "msg", - } as unknown as TemplateContext, - }), - replyOperation, - sessionKey: "agent:main:main", - getActiveSessionEntry: () => activeSessionEntry, - activeSessionStore, - storePath: "/tmp/sessions.json", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toContain("kept this conversation mapped to the current session"); - expect(result.payload.text).toContain("reserveTokensFloor"); - expectRecordFields(requireRecord(getReplyPayloadMetadata(result.payload), "reply metadata"), { - deliverDespiteSourceReplySuppression: true, - }); - } - expect(failMock).toHaveBeenCalledWith( - "run_failed", - expect.objectContaining({ message: "Auto-compaction failed: nothing to compact" }), - ); - expect(activeSessionStore["agent:main:main"]?.sessionId).toBe("session"); - expect(updateSessionIdMock).not.toHaveBeenCalled(); - expect(state.updateSessionStoreMock).not.toHaveBeenCalled(); - }); - - it.each([ - { - reason: "server_error" as const, - message: "upstream provider failed briefly", - }, - { - reason: "timeout" as const, - message: "provider request timed out without status token", - }, - ])( - "retries once for structured FailoverError $reason without leading HTTP status text", - async ({ reason, message }) => { - vi.useFakeTimers(); - state.runEmbeddedAgentMock - .mockRejectedValueOnce( - new FailoverError(message, { - reason, - provider: "openai", - model: "gpt-5.5", - }), - ) - .mockResolvedValueOnce({ - payloads: [{ text: "recovered after transient failover" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const resultPromise = runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - await vi.advanceTimersByTimeAsync(2_500); - const result = await resultPromise; - - expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(2); - expect(result.kind).toBe("success"); - if (result.kind === "success") { - expect(result.runResult.payloads?.[0]?.text).toBe("recovered after transient failover"); - } - }, - ); - - it("uses structured FailoverError context_overflow over non-overflow message text", async () => { - state.isLikelyContextOverflowErrorMock.mockReturnValue(false); - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new FailoverError("provider rejected the request payload", { - reason: "context_overflow", - provider: "anthropic", - model: "claude", - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: { - Provider: "telegram", - Surface: "telegram", - ChatType: "direct", - MessageSid: "msg", - } as unknown as TemplateContext, - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - "⚠️ Context overflow — prompt too large for this model. Try a shorter message or a larger-context model.", - ); - expect(result.payload.text).not.toBe(GENERIC_RUN_FAILURE_TEXT); - expect(result.payload.text).not.toContain("provider rejected the request payload"); - } - }); - - it("uses the throwing fallback candidate model for compaction failure hints", async () => { - state.isCompactionFailureErrorMock.mockReturnValue(true); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - await params.run("custom", "uncataloged-32k"); - throw new Error("expected fallback candidate to throw"); - }); - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error("Auto-compaction failed: nothing to compact"), - ); - - const followupRun = createFollowupRun(); - followupRun.run.provider = "openrouter"; - followupRun.run.model = "qwen3.6-plus"; - followupRun.run.config = { - models: { - providers: { - openrouter: { - baseUrl: "https://openrouter.test", - models: [makeTestModel("qwen3.6-plus", 1_000_000)], - }, - }, - }, - }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams({ followupRun })); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toContain("reserveTokensFloor"); - expect(result.payload.text).toContain("20000"); - expect(result.payload.text).not.toContain("100000"); - } - }); - - it("surfaces gateway reauth guidance for known OAuth refresh failures", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error( - "OAuth token refresh failed for openai: refresh_token_reused. Please try again or re-authenticate.", - ), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - "⚠️ Model login expired on the gateway for openai. Send `/login codex` from a private chat or Web UI session to pair a new Codex login, or re-auth with `openclaw models auth login --provider openai` in a terminal, then try again.", - ); - } - }); - - it("surfaces gateway reauth guidance from typed OAuth refresh failures", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new OAuthRefreshFailureError({ - provider: "openai", - profileId: "openai:user@example.com", - message: "invalid_grant", - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - "⚠️ Model login expired on the gateway for openai. Send `/login codex` from a private chat or Web UI session to pair a new Codex login, or re-auth with `openclaw models auth login --provider openai --profile-id 'openai:user@example.com'` in a terminal, then try again.", - ); - } - }); - - it("preserves OAuth profile guidance through failover wrappers", async () => { - const refreshError = new OAuthRefreshFailureError({ - provider: "openai", - profileId: "openai:user@example.com", - message: "invalid_grant", - }); - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new FailoverError("OpenAI OAuth failed", { - reason: "auth", - provider: "openai", - model: "gpt-5.5", - profileId: "openai:user@example.com", - authProfileFailure: { allInCooldown: false }, - status: 401, - cause: refreshError, - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toContain("--profile-id 'openai:user@example.com'"); - } - }); - - it("preserves OAuth profile guidance through fallback summaries", async () => { - const refreshError = new OAuthRefreshFailureError({ - provider: "openai", - profileId: "openai:user@example.com", - message: "invalid_grant", - }); - const failoverError = new FailoverError("OpenAI OAuth failed", { - reason: "auth", - provider: "openai", - model: "gpt-5.5", - profileId: "openai:user@example.com", - authProfileFailure: { allInCooldown: false }, - status: 401, - cause: refreshError, - }); - const summaryError = new Error("All models failed", { cause: failoverError }); - summaryError.name = "FallbackSummaryError"; - Object.assign(summaryError, { - attempts: [ - { - provider: "openai", - model: "gpt-5.5", - error: "OpenAI OAuth failed", - reason: "auth", - }, - ], - soonestCooldownExpiry: null, - }); - state.runEmbeddedAgentMock.mockRejectedValueOnce(summaryError); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toContain("--profile-id 'openai:user@example.com'"); - } - }); - - it("omits OAuth profile ids from group reauth guidance", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new OAuthRefreshFailureError({ - provider: "openai", - profileId: "openai:user@example.com", - message: "invalid_grant", - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback( - createMinimalRunAgentTurnParams({ - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - ChatType: "group", - } as unknown as TemplateContext, - }), - ); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toContain( - "openclaw models auth login --provider openai` in a terminal", - ); - expect(result.payload.text).not.toContain("user@example.com"); - } - }); - - it("keeps non-OpenAI OAuth refresh failures on provider-specific terminal guidance", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new OAuthRefreshFailureError({ - provider: "anthropic", - message: "invalid_grant", - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - "⚠️ Model login expired on the gateway for anthropic. Re-auth with `openclaw models auth login --provider anthropic` in a terminal, then try again.", - ); - expect(result.payload.text).not.toContain("/login codex"); - } - }); - - it("surfaces claude-cli re-auth hint over generic provider auth copy for 401 OAuth expiry", async () => { - // When the claude subprocess emits a 401 "Failed to authenticate" because - // its OAuth token has expired, the error is wrapped as a FailoverError with - // reason:"auth" and status:401. Without the ordering fix, this would be - // caught by classifyProviderRequestError before reaching classifyOAuthRefreshFailure, - // producing the generic "re-authenticate this provider" copy instead of the - // targeted claude-cli re-auth command. - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new FailoverError( - "Provider claude-cli failed: Failed to authenticate. API Error: 401 Invalid authentication credentials", - { - reason: "auth", - provider: "claude-cli", - model: "claude-sonnet-4-20250514", - status: 401, - }, - ), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - "⚠️ Model login expired on the gateway for claude-cli. Re-auth with `claude auth login && openclaw models auth login --provider anthropic --method cli` in a terminal, then try again.", - ); - } - }); - - it("surfaces claude-cli re-auth hint from structured provider metadata when the message omits claude-cli", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new FailoverError( - "Failed to authenticate. API Error: 401 Invalid authentication credentials", - { - reason: "auth", - provider: "claude-cli", - model: "claude-sonnet-4-20250514", - status: 401, - }, - ), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - "⚠️ Model login expired on the gateway for claude-cli. Re-auth with `claude auth login && openclaw models auth login --provider anthropic --method cli` in a terminal, then try again.", - ); - } - }); - - it("surfaces direct provider auth guidance for missing API keys", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error( - 'No API key found for provider "openai". You are authenticated with OpenAI Codex OAuth; OpenAI agent model runs use openai/gpt-* through the Codex runtime. Set OPENAI_API_KEY only for direct OpenAI API-key surfaces. | No API key found for provider "openai". You are authenticated with OpenAI Codex OAuth; OpenAI agent model runs use openai/gpt-* through the Codex runtime. Set OPENAI_API_KEY only for direct OpenAI API-key surfaces.', - ), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - "⚠️ Missing API key for OpenAI on the gateway. Use `openai/gpt-5.6-sol` with the OpenAI OAuth profile, or set `OPENAI_API_KEY` for direct OpenAI API-key runs.", - ); - } - }); - - it("surfaces typed missing API-key auth guidance without parsing the message", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new MissingProviderAuthError("openai", { - mode: "api-key", - source: "env: OPENAI_API_KEY", - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - '⚠️ Missing API key for provider "openai". Run `openclaw doctor --fix` to repair stale OpenAI model/session routes, restart the gateway if doctor asks, then try again. If doctor has nothing to repair or the error persists, re-auth with `openclaw models auth login --provider openai` or run `openclaw configure`.', - ); - } - }); - - it("formats auth-profile failover copy from typed FailoverError metadata", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new FailoverError("Auth profile failover exhausted for provider openai", { - reason: "auth", - provider: "openai", - status: 401, - authProfileFailure: { allInCooldown: true }, - cause: new Error("invalid_grant"), - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toContain("Couldn't sign in to openai."); - expect(result.payload.text).toContain("openclaw configure"); - expect(result.payload.text).toContain("(invalid_grant)"); - expect(result.payload.text).not.toContain("Auth profile failover exhausted"); - } - }); - - it("does not suggest re-authentication for typed format failures", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new FailoverError("Format failover exhausted for provider openai", { - reason: "format", - provider: "openai", - authProfileFailure: { allInCooldown: true }, - cause: new Error("messages must alternate roles"), - }), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toContain("Couldn't reach openai"); - expect(result.payload.text).toContain("messages must alternate roles"); - expect(result.payload.text).not.toContain("models auth login"); - expect(result.payload.text).not.toContain("openclaw configure"); - } - }); - - it("points stale openai missing-key failures at doctor repair with re-auth fallback", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error('No API key found for provider "openai".'), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - '⚠️ Missing API key for provider "openai". Run `openclaw doctor --fix` to repair stale OpenAI model/session routes, restart the gateway if doctor asks, then try again. If doctor has nothing to repair or the error persists, re-auth with `openclaw models auth login --provider openai` or run `openclaw configure`.', - ); - } - }); - - it("falls back to a generic provider message for unsafe missing-key provider ids", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error('No API key found for provider "openai`\nrm -rf /".'), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - "⚠️ Missing API key for the selected provider on the gateway. Configure provider auth, then try again.", - ); - } - }); - - it("falls back to a generic reauth command when the provider in the OAuth error is unsafe", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error( - "OAuth token refresh failed for openai`\nrm -rf /: invalid_grant. Please try again or re-authenticate.", - ), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe( - "⚠️ Model login expired on the gateway. Re-auth with `openclaw models auth login` in a terminal, then try again.", - ); - } - }); - - it("returns a session reset hint for Bedrock tool mismatch errors on external chat channels", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error( - "The number of toolResult blocks at messages.186.content exceeds the number of toolUse blocks of previous turn.", - ), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe(PROVIDER_CONVERSATION_STATE_ERROR_USER_MESSAGE); - } - }); - - it("returns a provider conversation-state error for OpenAI missing custom tool output errors on external chat channels", async () => { - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error("Custom tool call output is missing for call id: call_live_123."), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "slack", - ChannelId: "channel-1", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe(PROVIDER_CONVERSATION_STATE_ERROR_USER_MESSAGE); - } - }); - - it("does not auto-reset role-ordering provider conversation-state errors", async () => { - const resetSessionAfterRoleOrderingConflict = vi.fn(async () => true); - state.runEmbeddedAgentMock.mockRejectedValueOnce(new Error("400 Incorrect role information")); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "telegram", - ChatId: "chat-1", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(resetSessionAfterRoleOrderingConflict).not.toHaveBeenCalled(); - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toBe(PROVIDER_CONVERSATION_STATE_ERROR_USER_MESSAGE); - } - }); - - it("keeps raw generic errors on internal control surfaces", async () => { - state.isInternalMessageChannelMock.mockReturnValue(true); - state.runEmbeddedAgentMock.mockRejectedValueOnce( - new Error("INVALID_ARGUMENT: some other failure"), - ); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "chat", - Surface: "chat", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - if (result.kind === "final") { - expect(result.payload.text).toContain("Agent failed before reply"); - expect(result.payload.text).toContain("INVALID_ARGUMENT: some other failure"); - expect(result.payload.text).toContain("Logs: openclaw logs --follow"); - } - }); - - it("restarts the active prompt when a live model switch is requested", async () => { - let fallbackInvocation = 0; - state.runWithModelFallbackMock.mockImplementation( - async (params: { run: (provider: string, model: string) => Promise }) => { - const isInitialInvocation = fallbackInvocation++ === 0; - const provider = isInitialInvocation ? "anthropic" : "openai"; - const model = isInitialInvocation ? "claude" : "gpt-5.4"; - return { - result: await params.run(provider, model), - provider, - model, - attempts: [], - }; - }, - ); - state.runEmbeddedAgentMock - .mockImplementationOnce(async () => { - throw new LiveSessionModelSwitchError({ - provider: "openai", - model: "gpt-5.4", - agentRuntimeOverride: "codex", - }); - }) - .mockImplementationOnce(async () => { - return { - payloads: [{ text: "switched" }], - meta: { - agentMeta: { - sessionId: "session", - provider: "openai", - model: "gpt-5.4", - }, - }, - }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(2); - expect(followupRun.run.provider).toBe("openai"); - expect(followupRun.run.model).toBe("gpt-5.4"); - expect(state.runEmbeddedAgentMock.mock.calls[1]?.[0]).toEqual( - expect.objectContaining({ agentHarnessRuntimeOverride: "codex" }), - ); - }); - - it("breaks out of the retry loop when LiveSessionModelSwitchError is thrown repeatedly (#58348)", async () => { - // Simulate a scenario where the persisted session selection keeps conflicting - // with the fallback model, causing LiveSessionModelSwitchError on every attempt. - // The outer loop must be bounded to prevent a session death loop. - let switchCallCount = 0; - state.runWithModelFallbackMock.mockImplementation( - async (params: { run: (provider: string, model: string) => Promise }) => { - switchCallCount++; - return { - result: await params.run("anthropic", "claude"), - provider: "anthropic", - model: "claude", - attempts: [], - }; - }, - ); - state.runEmbeddedAgentMock.mockImplementation(async () => { - throw new LiveSessionModelSwitchError({ - provider: "openai", - model: "gpt-5.4", - }); - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - // After two retries the loop must break instead of continuing - // forever. The result should be a final error, not an infinite hang. - expect(result.kind).toBe("final"); - // One initial attempt plus two retries. - expect(switchCallCount).toBe(3); - }); - - it("propagates auth profile state on bounded live model switch retries (#58348)", async () => { - let invocation = 0; - state.runWithModelFallbackMock.mockImplementation( - async (params: { run: (provider: string, model: string) => Promise }) => { - invocation++; - if (invocation <= 2) { - return { - result: await params.run("anthropic", "claude"), - provider: "anthropic", - model: "claude", - attempts: [], - }; - } - // Third invocation succeeds with the switched model - return { - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - }; - }, - ); - state.runEmbeddedAgentMock - .mockImplementationOnce(async () => { - throw new LiveSessionModelSwitchError({ - provider: "openai", - model: "gpt-5.4", - authProfileId: "profile-b", - authProfileIdSource: "user", - }); - }) - .mockImplementationOnce(async () => { - throw new LiveSessionModelSwitchError({ - provider: "openai", - model: "gpt-5.4", - authProfileId: "profile-c", - authProfileIdSource: "auto", - }); - }) - .mockImplementationOnce(async () => { - return { - payloads: [{ text: "finally ok" }], - meta: { - agentMeta: { - sessionId: "session", - provider: "openai", - model: "gpt-5.4", - }, - }, - }; - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const followupRun = createFollowupRun(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => undefined, - resolvedVerboseLevel: "off", - }); - - // Two switches (within the limit of 2) then success on third attempt - expect(result.kind).toBe("success"); - expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(3); - expect(followupRun.run.provider).toBe("openai"); - expect(followupRun.run.model).toBe("gpt-5.4"); - expect(followupRun.run.authProfileId).toBe("profile-c"); - expect(followupRun.run.authProfileIdSource).toBe("auto"); - }); - - it("does not roll back newer override changes after a failed fallback candidate", async () => { - state.runWithModelFallbackMock.mockImplementation( - async (params: { run: (provider: string, model: string) => Promise }) => { - await expect(params.run("openai", "gpt-5.4")).rejects.toThrow("fallback failed"); - throw new Error("fallback failed"); - }, - ); - const sessionEntry: SessionEntry = { - sessionId: "session", - updatedAt: Date.now(), - providerOverride: "anthropic", - modelOverride: "claude", - authProfileOverride: "anthropic:default", - authProfileOverrideSource: "user", - }; - const sessionStore = { main: sessionEntry }; - state.runEmbeddedAgentMock.mockImplementationOnce(async () => { - sessionEntry.providerOverride = "zai"; - sessionEntry.modelOverride = "glm-5"; - sessionEntry.authProfileOverride = "zai:work"; - sessionEntry.authProfileOverrideSource = "user"; - throw new Error("fallback failed"); - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun: createFollowupRun(), - sessionCtx: { - Provider: "whatsapp", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => sessionEntry, - activeSessionStore: sessionStore, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("final"); - expect(sessionEntry.providerOverride).toBe("zai"); - expect(sessionEntry.modelOverride).toBe("glm-5"); - expect(sessionEntry.authProfileOverride).toBe("zai:work"); - expect(sessionEntry.authProfileOverrideSource).toBe("user"); - expect(sessionStore.main.providerOverride).toBe("zai"); - expect(sessionStore.main.modelOverride).toBe("glm-5"); - }); - - it("keeps cross-provider fallback selection turn-local", async () => { - state.runWithModelFallbackMock.mockImplementation( - async (params: { run: (provider: string, model: string) => Promise }) => ({ - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - }), - ); - state.runEmbeddedAgentMock.mockResolvedValue({ - payloads: [{ text: "ok" }], - meta: {}, - }); - - const followupRun = createFollowupRun(); - followupRun.run.provider = "anthropic"; - followupRun.run.model = "claude-opus"; - followupRun.run.authProfileId = "anthropic:openclaw"; - followupRun.run.authProfileIdSource = "user"; - - const sessionEntry: SessionEntry = { - sessionId: "session", - updatedAt: Date.now(), - totalTokens: 1, - compactionCount: 0, - }; - const sessionStore = { main: sessionEntry }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "telegram", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => sessionEntry, - activeSessionStore: sessionStore, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(1); - expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded run params", { - provider: "openai", - model: "gpt-5.4", - authProfileId: undefined, - authProfileIdSource: undefined, - }); - expect(sessionEntry.providerOverride).toBeUndefined(); - expect(sessionEntry.modelOverride).toBeUndefined(); - expect(sessionEntry.modelOverrideSource).toBeUndefined(); - expect(sessionEntry.authProfileOverride).toBeUndefined(); - expect(sessionEntry.authProfileOverrideSource).toBeUndefined(); - expect(sessionStore.main.authProfileOverride).toBeUndefined(); - }); - - it("does not persist fallback selection for legacy user overrides without modelOverrideSource", async () => { - // Regression: older persisted sessions can have a user-selected override - // (modelOverride set) but no modelOverrideSource field, because the field - // was added later. These legacy entries must still be protected from - // fallback overwrite, matching the backward-compat treatment in - // session-reset-service. - state.runWithModelFallbackMock.mockImplementation( - async (params: { run: (provider: string, model: string) => Promise }) => ({ - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - }), - ); - state.runEmbeddedAgentMock.mockResolvedValue({ - payloads: [{ text: "ok" }], - meta: {}, - }); - - const followupRun = createFollowupRun(); - followupRun.run.provider = "bailian"; - followupRun.run.model = "qwen3.6-plus"; - - const sessionEntry: SessionEntry = { - sessionId: "session", - updatedAt: Date.now(), - totalTokens: 1, - compactionCount: 0, - // Legacy entry: override is set but the source field is missing. - providerOverride: "anthropic", - modelOverride: "claude-opus-4-6", - // modelOverrideSource intentionally absent - }; - const sessionStore = { main: sessionEntry }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "telegram", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => sessionEntry, - activeSessionStore: sessionStore, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - // Legacy user override must survive the fallback unchanged. - expect(sessionEntry.providerOverride).toBe("anthropic"); - expect(sessionEntry.modelOverride).toBe("claude-opus-4-6"); - expect(sessionEntry.modelOverrideSource).toBeUndefined(); - }); - - it("does not replace a recovered auto override during fallback", async () => { - state.runWithModelFallbackMock.mockImplementation( - async (params: { run: (provider: string, model: string) => Promise }) => ({ - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - }), - ); - state.runEmbeddedAgentMock.mockResolvedValue({ - payloads: [{ text: "ok" }], - meta: {}, - }); - - const followupRun = createFollowupRun(); - followupRun.run.provider = "anthropic"; - followupRun.run.model = "claude-opus-4-6"; - - const sessionEntry: SessionEntry = { - sessionId: "session", - updatedAt: Date.now(), - totalTokens: 1, - compactionCount: 0, - providerOverride: "bailian", - modelOverride: "qwen3.6-plus", - modelOverrideFallbackOriginProvider: "minimax", - modelOverrideFallbackOriginModel: "MiniMax-M2.7", - // modelOverrideSource intentionally absent - }; - const sessionStore = { main: sessionEntry }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "telegram", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => sessionEntry, - activeSessionStore: sessionStore, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - expect(sessionEntry.providerOverride).toBe("bailian"); - expect(sessionEntry.modelOverride).toBe("qwen3.6-plus"); - expect(sessionEntry.modelOverrideSource).toBeUndefined(); - expect(sessionEntry.modelOverrideFallbackOriginProvider).toBe("minimax"); - expect(sessionEntry.modelOverrideFallbackOriginModel).toBe("MiniMax-M2.7"); - }); - - it("does not persist fallback selection when modelOverrideSource is user", async () => { - // Regression: fallback persistence overwrote user-initiated /models - // selections. When the user explicitly picked a model, the fallback - // should NOT clobber it even when the primary model fails. - state.runWithModelFallbackMock.mockImplementation( - async (params: { run: (provider: string, model: string) => Promise }) => ({ - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - }), - ); - state.runEmbeddedAgentMock.mockResolvedValue({ - payloads: [{ text: "ok" }], - meta: {}, - }); - - const followupRun = createFollowupRun(); - followupRun.run.provider = "anthropic"; - followupRun.run.model = "claude-opus-4-6"; - - const sessionEntry: SessionEntry = { - sessionId: "session", - updatedAt: Date.now(), - totalTokens: 1, - compactionCount: 0, - // User explicitly selected this model via /models - providerOverride: "anthropic", - modelOverride: "claude-opus-4-6", - modelOverrideSource: "user", - }; - const sessionStore = { main: sessionEntry }; - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - const result = await runAgentTurnWithFallback({ - commandBody: "hello", - followupRun, - sessionCtx: { - Provider: "telegram", - MessageSid: "msg", - } as unknown as TemplateContext, - opts: {}, - typingSignals: createMockTypingSignaler(), - blockReplyPipeline: null, - blockStreamingEnabled: false, - resolvedBlockStreamingBreak: "message_end", - applyReplyToMode: (payload) => payload, - shouldEmitToolResult: () => true, - shouldEmitToolOutput: () => false, - pendingToolTasks: new Set(), - resetSessionAfterRoleOrderingConflict: async () => false, - isHeartbeat: false, - sessionKey: "main", - getActiveSessionEntry: () => sessionEntry, - activeSessionStore: sessionStore, - resolvedVerboseLevel: "off", - }); - - expect(result.kind).toBe("success"); - // The user's /models selection must survive the fallback. - expect(sessionEntry.providerOverride).toBe("anthropic"); - expect(sessionEntry.modelOverride).toBe("claude-opus-4-6"); - expect(sessionEntry.modelOverrideSource).toBe("user"); - }); - - it("latches assistant error stub suppression across main reply fallback candidates", async () => { - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - await params.run("anthropic", "claude-opus-4-7").catch(() => undefined); - await params.run("anthropic", "claude-opus-4-6").catch(() => undefined); - return { - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - }; - }); - state.runEmbeddedAgentMock.mockImplementationOnce( - async (args: { - onAssistantErrorMessagePersisted?: (message: { - role: "assistant"; - content: string; - stopReason: "error"; - }) => void; - }) => { - args.onAssistantErrorMessagePersisted?.({ - role: "assistant", - content: "[assistant turn failed before producing content]", - stopReason: "error", - }); - throw new Error("upstream 500"); - }, - ); - state.runEmbeddedAgentMock.mockRejectedValueOnce(new Error("upstream 500")); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "ok" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(3); - expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "primary candidate", { - suppressAssistantErrorPersistence: false, - }); - expectMockCallArgFields(state.runEmbeddedAgentMock, 1, "first fallback candidate", { - suppressAssistantErrorPersistence: true, - }); - expectMockCallArgFields(state.runEmbeddedAgentMock, 2, "second fallback candidate", { - suppressAssistantErrorPersistence: true, - }); - }); - - it("does not suppress the first embedded assistant error after a CLI fallback failure", async () => { - state.isCliProviderMock.mockImplementation((provider: unknown) => provider === "anthropic"); - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - await params.run("anthropic", "claude-opus-4-7").catch(() => undefined); - return { - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - }; - }); - state.runCliAgentMock.mockRejectedValueOnce(new Error("cli failed")); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "ok" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(state.runCliAgentMock).toHaveBeenCalledOnce(); - expect(state.runEmbeddedAgentMock).toHaveBeenCalledOnce(); - expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "embedded fallback candidate", { - suppressAssistantErrorPersistence: false, - }); - }); - - it("latches queued user message persistence across main reply fallback candidates", async () => { - state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { - await params.run("anthropic", "claude-opus-4-7").catch(() => undefined); - return { - result: await params.run("openai", "gpt-5.4"), - provider: "openai", - model: "gpt-5.4", - attempts: [], - }; - }); - state.runEmbeddedAgentMock.mockImplementationOnce( - async (args: { - onUserMessagePersisted?: (m: { - role: "user"; - content: Array<{ type: "text"; text: string }>; - }) => void; - }) => { - args.onUserMessagePersisted?.({ - role: "user", - content: [{ type: "text", text: "queued" }], - }); - throw new Error("upstream 500"); - }, - ); - state.runEmbeddedAgentMock.mockResolvedValueOnce({ - payloads: [{ text: "ok" }], - meta: {}, - }); - - const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); - await runAgentTurnWithFallback(createMinimalRunAgentTurnParams()); - - expect(state.runEmbeddedAgentMock).toHaveBeenCalledTimes(2); - expectMockCallArgFields(state.runEmbeddedAgentMock, 0, "primary candidate", { - suppressNextUserMessagePersistence: false, - }); - expectMockCallArgFields(state.runEmbeddedAgentMock, 1, "fallback candidate", { - suppressNextUserMessagePersistence: true, - }); - }); -}); -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/auto-reply/reply/agent-runner-execution.ts b/src/auto-reply/reply/agent-runner-execution.ts index c662fb0c9056..4d89a42cd562 100644 --- a/src/auto-reply/reply/agent-runner-execution.ts +++ b/src/auto-reply/reply/agent-runner-execution.ts @@ -5,10 +5,6 @@ import { normalizeOptionalString, } from "@openclaw/normalization-core/string-coerce"; import { hasOutboundReplyContent } from "openclaw/plugin-sdk/reply-payload"; -import { - clearAutoFallbackPrimaryProbeSelection, - entryMatchesAutoFallbackPrimaryProbe, -} from "../../agents/agent-scope.js"; import { resolveBootstrapWarningSignaturesSeen } from "../../agents/bootstrap-budget.js"; import { formatRateLimitOrOverloadedErrorCopy, @@ -19,7 +15,6 @@ import { runEmbeddedAgent } from "../../agents/embedded-agent.js"; import { LiveSessionModelSwitchError } from "../../agents/live-model-switch-error.js"; import { createAgentPatchedSessionModelRunGuard } from "../../agents/session-model-auto-revert.js"; import type { SessionEntry } from "../../config/sessions.js"; -import { updateSessionEntry } from "../../config/sessions/session-accessor.js"; import { logVerbose } from "../../globals.js"; import { captureAgentRunLifecycleGeneration, @@ -29,10 +24,12 @@ import { import { isDiagnosticsEnabled } from "../../infra/diagnostic-events.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { logSessionTurnCreated } from "../../logging/diagnostic.js"; -import { shouldPreserveUserFacingSessionStateForInputProvenance } from "../../sessions/input-provenance.js"; import { isInternalMessageChannel } from "../../utils/message-channel.js"; import type { ReplyPayload } from "../types.js"; -import { resolveRunAfterAutoFallbackPrimaryProbeRecheck } from "./agent-runner-auto-fallback.js"; +import { + clearRecoveredAutoFallbackPrimaryProbeSelection, + resolveRunAfterAutoFallbackPrimaryProbeRecheck, +} from "./agent-runner-auto-fallback.js"; import { handleAgentExecutionError } from "./agent-runner-error-handler.js"; import type { AgentRunLoopResult, @@ -83,9 +80,6 @@ async function runAgentTurnWithFallbackInternal( ...runnableRun, config: runtimeConfig, }; - const preserveUserFacingSessionState = shouldPreserveUserFacingSessionStateForInputProvenance( - effectiveRun.inputProvenance, - ); let liveModelSwitchRuntimeEntry: | Pick | undefined; @@ -228,65 +222,15 @@ async function runAgentTurnWithFallbackInternal( const clearRecoveredAutoFallbackPrimaryProbe = async (paramsForClear: { provider: string; model: string; - }): Promise => { - if (preserveUserFacingSessionState) { - return; - } - const probe = effectiveRun.autoFallbackPrimaryProbe; - if (!probe) { - return; - } - if (paramsForClear.provider !== probe.provider || paramsForClear.model !== probe.model) { - return; - } - if (!params.sessionKey || !params.activeSessionStore) { - return; - } - const activeSessionEntry = - params.activeSessionStore[params.sessionKey] ?? params.getActiveSessionEntry(); - if (!activeSessionEntry) { - return; - } - if (!entryMatchesAutoFallbackPrimaryProbe(activeSessionEntry, probe)) { - return; - } - clearAutoFallbackPrimaryProbeSelection(activeSessionEntry); - params.activeSessionStore[params.sessionKey] = activeSessionEntry; - if (!params.storePath) { - return; - } - await updateSessionEntry( - { storePath: params.storePath, sessionKey: params.sessionKey }, - (persistedEntry) => { - if (!entryMatchesAutoFallbackPrimaryProbe(persistedEntry, probe)) { - return null; - } - const shouldClearAuthProfile = - persistedEntry.authProfileOverrideSource === "auto" || - (persistedEntry.authProfileOverrideSource === undefined && - persistedEntry.authProfileOverrideCompactionCount !== undefined); - clearAutoFallbackPrimaryProbeSelection(persistedEntry); - return { - providerOverride: undefined, - modelOverride: undefined, - modelOverrideSource: undefined, - modelOverrideFallbackOriginProvider: undefined, - modelOverrideFallbackOriginModel: undefined, - ...(shouldClearAuthProfile - ? { - authProfileOverride: undefined, - authProfileOverrideSource: undefined, - authProfileOverrideCompactionCount: undefined, - } - : {}), - fallbackNoticeSelectedModel: undefined, - fallbackNoticeActiveModel: undefined, - fallbackNoticeReason: undefined, - updatedAt: persistedEntry.updatedAt, - }; - }, - ); - }; + }): Promise => + clearRecoveredAutoFallbackPrimaryProbeSelection({ + run: effectiveRun, + ...paramsForClear, + sessionKey: params.sessionKey, + activeSessionStore: params.activeSessionStore, + getActiveSessionEntry: params.getActiveSessionEntry, + storePath: params.storePath, + }); while (true) { try { diff --git a/src/auto-reply/reply/agent-runner-failure-reply.test.ts b/src/auto-reply/reply/agent-runner-failure-reply.test.ts new file mode 100644 index 000000000000..f3700d09b186 --- /dev/null +++ b/src/auto-reply/reply/agent-runner-failure-reply.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { SILENT_REPLY_TOKEN } from "../tokens.js"; +import { buildEmptyInteractiveReplyPayload } from "./agent-runner-failure-reply.js"; + +const EMPTY_INTERACTIVE_REPLY_TEXT = + "I finished the turn, but it did not produce a visible reply. Please try again, or start a new session if this keeps happening."; + +describe("buildEmptyInteractiveReplyPayload", () => { + const baseParams = { + isInteractive: true, + isMessageToolOnly: false, + hasPendingContinuation: false, + hasExplicitSilentReply: false, + hasCommittedDelivery: false, + sessionCtx: { + Provider: "discord", + Surface: "discord", + ChatType: "group", + }, + } as const; + + it("preserves the default silent policy in group conversations", () => { + const payload = buildEmptyInteractiveReplyPayload(baseParams); + + expect(payload?.text).toBe(SILENT_REPLY_TOKEN); + expect(payload?.isError).toBeUndefined(); + }); + + it("surfaces the fallback when group silence is explicitly disallowed", () => { + expect( + buildEmptyInteractiveReplyPayload({ + ...baseParams, + cfg: { agents: { defaults: { silentReply: { group: "disallow" } } } }, + }), + ).toMatchObject({ text: EMPTY_INTERACTIVE_REPLY_TEXT, isError: true }); + }); +}); diff --git a/src/auto-reply/reply/agent-runner-fallback-candidate.ts b/src/auto-reply/reply/agent-runner-fallback-candidate.ts new file mode 100644 index 000000000000..80dd4858316e --- /dev/null +++ b/src/auto-reply/reply/agent-runner-fallback-candidate.ts @@ -0,0 +1,236 @@ +import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; +import { markAutoFallbackPrimaryProbe } from "../../agents/agent-scope.js"; +import { mergeEmbeddedAgentRunResultForModelFallbackExhaustion } from "../../agents/embedded-agent-runner/result-fallback-classifier.js"; +import type { FastModeAutoProgressState } from "../../agents/fast-mode.js"; +import { ensureSelectedAgentHarnessPlugin } from "../../agents/harness/runtime-plugin.js"; +import { runWithModelFallback } from "../../agents/model-fallback.js"; +import { resolveCliRuntimeExecutionProvider } from "../../agents/model-runtime-aliases.js"; +import { isCliProvider } from "../../agents/model-selection.js"; +import { buildAgentRuntimeOutcomePlan } from "../../agents/runtime-plan/build.js"; +import { resolveSessionRuntimeOverrideForProvider } from "../../agents/session-runtime-compat.js"; +import { resolveCandidateThinkingLevel } from "../../agents/thinking-runtime.js"; +import { resolveHeartbeatRunScope } from "../../infra/heartbeat-run-scope.js"; +import { CommandLane } from "../../process/lanes.js"; +import type { AgentLifecycleTerminalBackstop } from "./agent-lifecycle-terminal.js"; +import { resolveFallbackCandidateRun, resolveRunAuthProfile } from "./agent-runner-auth-profile.js"; +import { runCliFallbackCandidate } from "./agent-runner-cli-candidate.js"; +import { runEmbeddedFallbackCandidate } from "./agent-runner-embedded-candidate.js"; +import type { MessageToolDeliveryState } from "./agent-runner-event-handler.js"; +import type { EmbeddedAgentRunResult } from "./agent-runner-execution.types.js"; +import type { AgentFallbackCycleParams } from "./agent-runner-fallback-cycle.types.js"; +import { emitModelFallbackStepLifecycle } from "./agent-runner-model-fallback-lifecycle.js"; +import { + resolveModelFallbackOptions, + resolveRunFastModeForFallbackCandidate, +} from "./agent-runner-utils.js"; + +/** Runs the provider/model fallback candidates while preserving cross-candidate delivery state. */ +export async function runAgentFallbackCandidates(params: AgentFallbackCycleParams) { + const turn = params.turn; + const preserveProgressCallbackStartOrder = turn.opts?.preserveProgressCallbackStartOrder === true; + const sourceRepliesAreToolOnly = + turn.followupRun.run.sourceReplyDeliveryMode === "message_tool_only"; + const outcomePlan = buildAgentRuntimeOutcomePlan(); + const runLane = CommandLane.Main; + let queuedUserMessagePersistedAcrossFallback = false; + let assistantErrorPersistedAcrossFallback = false; + const messageToolDeliveryState: MessageToolDeliveryState = { + toolCallIds: new Set(), + completed: false, + }; + const userTurnTranscriptRecorder = + turn.followupRun.userTurnTranscriptRecorder ?? turn.opts?.userTurnTranscriptRecorder; + const fastModeStartedAtMs = Date.now(); + const fastModeAutoProgressState: FastModeAutoProgressState = { + offAnnounced: false, + resetAnnounced: false, + }; + const bootstrapContextRunKind = + resolveHeartbeatRunScope(turn.opts) === "commitment-only" + ? ("commitment-only" as const) + : turn.opts?.isHeartbeat + ? ("heartbeat" as const) + : ("default" as const); + + params.timing.logMilestoneIfSlow({ + runId: params.runId, + sessionId: turn.followupRun.run.sessionId, + sessionKey: turn.sessionKey, + milestone: "before_model_fallback", + }); + return params.timing.measure("model_fallback", () => + runWithModelFallback({ + ...resolveModelFallbackOptions(params.effectiveRun, params.runtimeConfig), + runId: params.runId, + sessionId: turn.followupRun.run.sessionId, + lane: runLane, + abortSignal: params.runAbortSignal, + resolveAgentHarnessRuntimeOverride: (provider) => + resolveSessionRuntimeOverrideForProvider({ + provider, + entry: params.liveModelSwitchRuntimeEntry ?? turn.getActiveSessionEntry(), + cfg: params.runtimeConfig, + }), + prepareAgentHarnessRuntime: async ({ provider, model, agentHarnessRuntimeOverride }) => { + await params.timing.measure("fallback_prepare_harness", () => + ensureSelectedAgentHarnessPlugin({ + config: params.runtimeConfig, + provider, + modelId: model, + agentId: turn.followupRun.run.agentId, + sessionKey: turn.followupRun.run.runtimePolicySessionKey ?? turn.sessionKey, + agentHarnessRuntimeOverride, + workspaceDir: turn.followupRun.run.workspaceDir, + }), + ); + }, + onFallbackStep: (step) => { + emitModelFallbackStepLifecycle({ runId: params.runId, sessionKey: turn.sessionKey, step }); + }, + classifyResult: ({ result, provider, model }) => + outcomePlan.classifyRunResult({ + result, + provider, + model, + hasDirectlySentBlockReply: params.directlySentBlockKeys.size > 0, + hasBlockReplyPipelineOutput: Boolean( + turn.blockReplyPipeline?.hasBuffered() || turn.blockReplyPipeline?.didStream(), + ), + }), + mergeExhaustedResult: mergeEmbeddedAgentRunResultForModelFallbackExhaustion, + run: async (provider, model, runOptions) => { + params.state.attemptedRuntimeProvider = provider; + params.state.attemptedRuntimeModel = model; + const candidateRun = resolveFallbackCandidateRun(params.effectiveRun, provider, model); + const candidateThinkLevel = resolveCandidateThinkingLevel({ + cfg: params.runtimeConfig, + provider, + modelId: model, + level: turn.followupRun.run.thinkLevel, + agentId: turn.followupRun.run.agentId, + sessionKey: turn.followupRun.run.runtimePolicySessionKey ?? turn.sessionKey, + sessionEntry: turn.getActiveSessionEntry(), + }); + const candidateFastMode = resolveRunFastModeForFallbackCandidate({ + run: candidateRun, + config: params.runtimeConfig, + provider, + model, + sessionEntry: turn.getActiveSessionEntry(), + }); + const activeProbe = params.effectiveRun.autoFallbackPrimaryProbe; + if (activeProbe && provider === activeProbe.provider && model === activeProbe.model) { + markAutoFallbackPrimaryProbe({ probe: activeProbe, sessionKey: turn.sessionKey }); + } + turn.opts?.onModelSelected?.({ provider, model, thinkLevel: candidateThinkLevel }); + const runtime = params.timing.measureSync("fallback_resolve_runtime", () => { + const activeEntry = params.liveModelSwitchRuntimeEntry ?? turn.getActiveSessionEntry(); + const sessionRuntimeOverride = resolveSessionRuntimeOverrideForProvider({ + provider, + entry: activeEntry, + cfg: params.runtimeConfig, + }); + const locksPersistedHarness = + activeEntry?.modelSelectionLocked === true && + normalizeLowercaseStringOrEmpty(activeEntry.agentHarnessId) === sessionRuntimeOverride; + const selectedAuthProfile = resolveRunAuthProfile(candidateRun, provider, { + config: params.runtimeConfig, + }); + const pinnedCliRuntime = + !locksPersistedHarness && + sessionRuntimeOverride && + isCliProvider(sessionRuntimeOverride, params.runtimeConfig) + ? sessionRuntimeOverride + : undefined; + const cliExecutionProvider = + pinnedCliRuntime ?? + (sessionRuntimeOverride + ? provider + : (resolveCliRuntimeExecutionProvider({ + provider, + cfg: params.runtimeConfig, + agentId: turn.followupRun.run.agentId, + modelId: model, + authProfileId: selectedAuthProfile.authProfileId, + }) ?? provider)); + return { + sessionRuntimeOverride, + cliExecutionProvider, + useCliExecution: + pinnedCliRuntime !== undefined || + (!sessionRuntimeOverride && + isCliProvider(cliExecutionProvider, params.runtimeConfig)), + }; + }); + const common = { + turn, + candidateRun, + runtimeConfig: params.runtimeConfig, + provider, + model, + candidateThinkLevel, + candidateFastMode, + runId: params.runId, + runAbortSignal: params.runAbortSignal, + isFinalFallbackAttempt: runOptions?.isFinalFallbackAttempt, + suppressQueuedUserPersistenceForCandidate: + (turn.followupRun.run.suppressNextUserMessagePersistence ?? false) || + queuedUserMessagePersistedAcrossFallback, + userTurnTranscriptRecorder, + notifyUserMessagePersisted: () => { + queuedUserMessagePersistedAcrossFallback = true; + }, + fastModeStartedAtMs, + fastModeAutoProgressState, + bootstrapContextRunKind, + bootstrapPromptWarningSignaturesSeen: params.state.bootstrapPromptWarningSignaturesSeen, + currentTurnImages: params.currentTurnImages, + signalExecutionPhaseForTyping: params.signalExecutionPhaseForTyping, + notifyAgentRunStart: params.notifyAgentRunStart, + preserveProgressCallbackStartOrder, + presentation: params.presentation, + timing: params.timing, + onLifecycleBackstop: (backstop: AgentLifecycleTerminalBackstop) => { + params.state.pendingLifecycleTerminal = { provider, model, backstop }; + }, + }; + if (runtime.useCliExecution) { + const candidate = await runCliFallbackCandidate({ + ...common, + cliExecutionProvider: runtime.cliExecutionProvider, + lifecycleGeneration: params.state.lifecycleGeneration, + runLane, + }); + params.state.bootstrapPromptWarningSignaturesSeen = + candidate.bootstrapPromptWarningSignaturesSeen; + return candidate.result; + } + const candidate = await runEmbeddedFallbackCandidate({ + ...common, + effectiveRun: params.effectiveRun, + sessionRuntimeOverride: runtime.sessionRuntimeOverride, + getLifecycleGeneration: () => params.state.lifecycleGeneration, + onLifecycleGeneration: (generation) => { + params.state.lifecycleGeneration = generation; + }, + allowTransientCooldownProbe: runOptions?.allowTransientCooldownProbe, + suppressAssistantErrorPersistenceForCandidate: assistantErrorPersistedAcrossFallback, + onAssistantErrorMessagePersisted: () => { + assistantErrorPersistedAcrossFallback = true; + }, + notifyUserAboutCompaction: params.notifyUserAboutCompaction, + sourceRepliesAreToolOnly, + messageToolDeliveryState, + onCompactionCount: (count) => { + params.state.autoCompactionCount += count; + }, + }); + params.state.bootstrapPromptWarningSignaturesSeen = + candidate.bootstrapPromptWarningSignaturesSeen; + return candidate.result; + }, + }), + ); +} + +export type AgentFallbackCandidatesResult = Awaited>; diff --git a/src/auto-reply/reply/agent-runner-fallback-cycle.ts b/src/auto-reply/reply/agent-runner-fallback-cycle.ts index a008dc5bae02..0d5b44e208d9 100644 --- a/src/auto-reply/reply/agent-runner-fallback-cycle.ts +++ b/src/auto-reply/reply/agent-runner-fallback-cycle.ts @@ -1,460 +1,22 @@ -import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; -import { markAutoFallbackPrimaryProbe } from "../../agents/agent-scope.js"; -import { isContextOverflowError } from "../../agents/embedded-agent-helpers.js"; -import { mergeEmbeddedAgentRunResultForModelFallbackExhaustion } from "../../agents/embedded-agent-runner/result-fallback-classifier.js"; -import type { RunEmbeddedAgentParams } from "../../agents/embedded-agent-runner/run/params.js"; -import type { FastModeAutoProgressState } from "../../agents/fast-mode.js"; -import { ensureSelectedAgentHarnessPlugin } from "../../agents/harness/runtime-plugin.js"; -import { runWithModelFallback } from "../../agents/model-fallback.js"; -import { resolveCliRuntimeExecutionProvider } from "../../agents/model-runtime-aliases.js"; -import { isCliProvider } from "../../agents/model-selection.js"; -import { - createAgentRunRestartAbortError, - isAgentRunRestartAbortReason, -} from "../../agents/run-termination.js"; -import { buildAgentRuntimeOutcomePlan } from "../../agents/runtime-plan/build.js"; -import { resolveSessionRuntimeOverrideForProvider } from "../../agents/session-runtime-compat.js"; -import { resolveCandidateThinkingLevel } from "../../agents/thinking-runtime.js"; -import type { SessionEntry } from "../../config/sessions.js"; -import type { OpenClawConfig } from "../../config/types.openclaw.js"; -import { logVerbose } from "../../globals.js"; -import { emitAgentEvent } from "../../infra/agent-events.js"; -import { formatErrorMessage } from "../../infra/errors.js"; -import { resolveHeartbeatRunScope } from "../../infra/heartbeat-run-scope.js"; -import { CommandLane } from "../../process/lanes.js"; -import { defaultRuntime } from "../../runtime.js"; -import { SILENT_REPLY_TOKEN } from "../tokens.js"; -import { - resolveAgentLifecycleTerminalMetadata, - type AgentLifecycleTerminalBackstop, -} from "./agent-lifecycle-terminal.js"; -import { resolveFallbackCandidateRun, resolveRunAuthProfile } from "./agent-runner-auth-profile.js"; -import { runCliFallbackCandidate } from "./agent-runner-cli-candidate.js"; -import { buildContextOverflowRecoveryText } from "./agent-runner-context-recovery.js"; -import { runEmbeddedFallbackCandidate } from "./agent-runner-embedded-candidate.js"; -import type { MessageToolDeliveryState } from "./agent-runner-event-handler.js"; +import { runAgentFallbackCandidates } from "./agent-runner-fallback-candidate.js"; import type { - AgentRunLoopResult, - AgentTurnParams, - EmbeddedAgentRunResult, - RuntimeFallbackAttempt, -} from "./agent-runner-execution.types.js"; -import { markAgentRunFailureReplyPayload } from "./agent-runner-failure-reply.js"; -import { emitModelFallbackStepLifecycle } from "./agent-runner-model-fallback-lifecycle.js"; -import type { createAgentTurnPresentation } from "./agent-runner-presentation.js"; -import type { AgentTurnTimingTracker } from "./agent-runner-turn-timing.js"; -import { - resolveModelFallbackOptions, - resolveRunFastModeForFallbackCandidate, -} from "./agent-runner-utils.js"; -import { drainPendingToolTasks } from "./pending-tool-task-drain.js"; -import { - classifyProviderRequestError, - PROVIDER_CONVERSATION_STATE_ERROR_USER_MESSAGE, -} from "./provider-request-error-classifier.js"; -import type { FollowupRun } from "./queue.js"; -import { - isReplyOperationRestartAbort, - isReplyOperationUserAbort, -} from "./reply-operation-abort.js"; + AgentFallbackCycleParams, + AgentFallbackCycleResult, +} from "./agent-runner-fallback-cycle.types.js"; +import { settleAgentFallbackCycle } from "./agent-runner-fallback-settlement.js"; -type Presentation = ReturnType; +export type { AgentFallbackCycleState } from "./agent-runner-fallback-cycle.types.js"; -export type AgentFallbackCycleState = { - lifecycleGeneration: string; - autoCompactionCount: number; - attemptedRuntimeProvider: string; - attemptedRuntimeModel: string; - bootstrapPromptWarningSignaturesSeen: string[]; - pendingLifecycleTerminal?: { - provider: string; - model: string; - backstop: AgentLifecycleTerminalBackstop; - }; -}; - -type CompletedFallbackCycle = { - kind: "completed"; - runResult: EmbeddedAgentRunResult; - fallbackProvider: string; - fallbackModel: string; - fallbackExhausted: boolean; - fallbackAttempts: RuntimeFallbackAttempt[]; - terminalRunFailed: boolean; -}; - -type ModelPatch = { - captureFallbackFailure: (attempts: RuntimeFallbackAttempt[]) => boolean | undefined; - captureFailure: (error: unknown) => void; -}; - -export async function executeAgentFallbackCycle(params: { - turn: AgentTurnParams; - effectiveRun: FollowupRun["run"]; - runtimeConfig: OpenClawConfig; - liveModelSwitchRuntimeEntry?: Pick< - SessionEntry, - "agentHarnessId" | "agentRuntimeOverride" | "modelSelectionLocked" - >; - runId: string; - runAbortSignal?: AbortSignal; - currentTurnImages: Awaited< - ReturnType - >; - state: AgentFallbackCycleState; - presentation: Presentation; - directlySentBlockKeys: Set; - notifyAgentRunStart: () => void; - signalExecutionPhaseForTyping: NonNullable; - notifyUserAboutCompaction: boolean; - timing: AgentTurnTimingTracker; - modelPatch: ModelPatch; - shouldSurfaceToControlUi: boolean; - commitTerminalOutcome: () => void; - clearRecoveredAutoFallbackPrimaryProbe: (candidate: { - provider: string; - model: string; - }) => Promise; -}): Promise> { - const turn = params.turn; - const preserveProgressCallbackStartOrder = turn.opts?.preserveProgressCallbackStartOrder === true; - const sourceRepliesAreToolOnly = - turn.followupRun.run.sourceReplyDeliveryMode === "message_tool_only"; - const outcomePlan = buildAgentRuntimeOutcomePlan(); - const runLane = CommandLane.Main; - let queuedUserMessagePersistedAcrossFallback = false; - let assistantErrorPersistedAcrossFallback = false; - const messageToolDeliveryState: MessageToolDeliveryState = { - toolCallIds: new Set(), - completed: false, - }; - const userTurnTranscriptRecorder = - turn.followupRun.userTurnTranscriptRecorder ?? turn.opts?.userTurnTranscriptRecorder; - const fastModeStartedAtMs = Date.now(); - const fastModeAutoProgressState: FastModeAutoProgressState = { - offAnnounced: false, - resetAnnounced: false, - }; - const bootstrapContextRunKind = - resolveHeartbeatRunScope(turn.opts) === "commitment-only" - ? ("commitment-only" as const) - : turn.opts?.isHeartbeat - ? ("heartbeat" as const) - : ("default" as const); - params.timing.logMilestoneIfSlow({ - runId: params.runId, - sessionId: turn.followupRun.run.sessionId, - sessionKey: turn.sessionKey, - milestone: "before_model_fallback", - }); - const fallbackResult = await params.timing.measure("model_fallback", () => - runWithModelFallback({ - ...resolveModelFallbackOptions(params.effectiveRun, params.runtimeConfig), - runId: params.runId, - sessionId: turn.followupRun.run.sessionId, - lane: runLane, - abortSignal: params.runAbortSignal, - resolveAgentHarnessRuntimeOverride: (provider) => - resolveSessionRuntimeOverrideForProvider({ - provider, - entry: params.liveModelSwitchRuntimeEntry ?? turn.getActiveSessionEntry(), - cfg: params.runtimeConfig, - }), - prepareAgentHarnessRuntime: async ({ provider, model, agentHarnessRuntimeOverride }) => { - await params.timing.measure("fallback_prepare_harness", () => - ensureSelectedAgentHarnessPlugin({ - config: params.runtimeConfig, - provider, - modelId: model, - agentId: turn.followupRun.run.agentId, - sessionKey: turn.followupRun.run.runtimePolicySessionKey ?? turn.sessionKey, - agentHarnessRuntimeOverride, - workspaceDir: turn.followupRun.run.workspaceDir, - }), - ); - }, - onFallbackStep: (step) => { - emitModelFallbackStepLifecycle({ runId: params.runId, sessionKey: turn.sessionKey, step }); - }, - classifyResult: ({ result, provider, model }) => - outcomePlan.classifyRunResult({ - result, - provider, - model, - hasDirectlySentBlockReply: params.directlySentBlockKeys.size > 0, - hasBlockReplyPipelineOutput: Boolean( - turn.blockReplyPipeline?.hasBuffered() || turn.blockReplyPipeline?.didStream(), - ), - }), - mergeExhaustedResult: mergeEmbeddedAgentRunResultForModelFallbackExhaustion, - run: async (provider, model, runOptions) => { - params.state.attemptedRuntimeProvider = provider; - params.state.attemptedRuntimeModel = model; - const candidateRun = resolveFallbackCandidateRun(params.effectiveRun, provider, model); - const candidateThinkLevel = resolveCandidateThinkingLevel({ - cfg: params.runtimeConfig, - provider, - modelId: model, - level: turn.followupRun.run.thinkLevel, - agentId: turn.followupRun.run.agentId, - sessionKey: turn.followupRun.run.runtimePolicySessionKey ?? turn.sessionKey, - sessionEntry: turn.getActiveSessionEntry(), - }); - const candidateFastMode = resolveRunFastModeForFallbackCandidate({ - run: candidateRun, - config: params.runtimeConfig, - provider, - model, - sessionEntry: turn.getActiveSessionEntry(), - }); - const activeProbe = params.effectiveRun.autoFallbackPrimaryProbe; - if (activeProbe && provider === activeProbe.provider && model === activeProbe.model) { - markAutoFallbackPrimaryProbe({ probe: activeProbe, sessionKey: turn.sessionKey }); - } - turn.opts?.onModelSelected?.({ provider, model, thinkLevel: candidateThinkLevel }); - const runtime = params.timing.measureSync("fallback_resolve_runtime", () => { - const activeEntry = params.liveModelSwitchRuntimeEntry ?? turn.getActiveSessionEntry(); - const sessionRuntimeOverride = resolveSessionRuntimeOverrideForProvider({ - provider, - entry: activeEntry, - cfg: params.runtimeConfig, - }); - const locksPersistedHarness = - activeEntry?.modelSelectionLocked === true && - normalizeLowercaseStringOrEmpty(activeEntry.agentHarnessId) === sessionRuntimeOverride; - const selectedAuthProfile = resolveRunAuthProfile(candidateRun, provider, { - config: params.runtimeConfig, - }); - const pinnedCliRuntime = - !locksPersistedHarness && - sessionRuntimeOverride && - isCliProvider(sessionRuntimeOverride, params.runtimeConfig) - ? sessionRuntimeOverride - : undefined; - const cliExecutionProvider = - pinnedCliRuntime ?? - (sessionRuntimeOverride - ? provider - : (resolveCliRuntimeExecutionProvider({ - provider, - cfg: params.runtimeConfig, - agentId: turn.followupRun.run.agentId, - modelId: model, - authProfileId: selectedAuthProfile.authProfileId, - }) ?? provider)); - return { - sessionRuntimeOverride, - cliExecutionProvider, - useCliExecution: - pinnedCliRuntime !== undefined || - (!sessionRuntimeOverride && - isCliProvider(cliExecutionProvider, params.runtimeConfig)), - }; - }); - const common = { - turn, - candidateRun, - runtimeConfig: params.runtimeConfig, - provider, - model, - candidateThinkLevel, - candidateFastMode, - runId: params.runId, - runAbortSignal: params.runAbortSignal, - isFinalFallbackAttempt: runOptions?.isFinalFallbackAttempt, - suppressQueuedUserPersistenceForCandidate: - (turn.followupRun.run.suppressNextUserMessagePersistence ?? false) || - queuedUserMessagePersistedAcrossFallback, - userTurnTranscriptRecorder, - notifyUserMessagePersisted: () => { - queuedUserMessagePersistedAcrossFallback = true; - }, - fastModeStartedAtMs, - fastModeAutoProgressState, - bootstrapContextRunKind, - bootstrapPromptWarningSignaturesSeen: params.state.bootstrapPromptWarningSignaturesSeen, - currentTurnImages: params.currentTurnImages, - signalExecutionPhaseForTyping: params.signalExecutionPhaseForTyping, - notifyAgentRunStart: params.notifyAgentRunStart, - preserveProgressCallbackStartOrder, - presentation: params.presentation, - timing: params.timing, - onLifecycleBackstop: (backstop: AgentLifecycleTerminalBackstop) => { - params.state.pendingLifecycleTerminal = { provider, model, backstop }; - }, - }; - if (runtime.useCliExecution) { - const candidate = await runCliFallbackCandidate({ - ...common, - cliExecutionProvider: runtime.cliExecutionProvider, - lifecycleGeneration: params.state.lifecycleGeneration, - runLane, - }); - params.state.bootstrapPromptWarningSignaturesSeen = - candidate.bootstrapPromptWarningSignaturesSeen; - return candidate.result; - } - const candidate = await runEmbeddedFallbackCandidate({ - ...common, - effectiveRun: params.effectiveRun, - sessionRuntimeOverride: runtime.sessionRuntimeOverride, - getLifecycleGeneration: () => params.state.lifecycleGeneration, - onLifecycleGeneration: (generation) => { - params.state.lifecycleGeneration = generation; - }, - allowTransientCooldownProbe: runOptions?.allowTransientCooldownProbe, - suppressAssistantErrorPersistenceForCandidate: assistantErrorPersistedAcrossFallback, - onAssistantErrorMessagePersisted: () => { - assistantErrorPersistedAcrossFallback = true; - }, - notifyUserAboutCompaction: params.notifyUserAboutCompaction, - sourceRepliesAreToolOnly, - messageToolDeliveryState, - onCompactionCount: (count) => { - params.state.autoCompactionCount += count; - }, - }); - params.state.bootstrapPromptWarningSignaturesSeen = - candidate.bootstrapPromptWarningSignaturesSeen; - return candidate.result; - }, - }), - ); +/** Runs one fallback chain, then settles its terminal lifecycle state. */ +export async function executeAgentFallbackCycle( + params: AgentFallbackCycleParams, +): Promise { + const fallbackResult = await runAgentFallbackCandidates(params); params.timing.logIfSlow({ runId: params.runId, - sessionId: turn.followupRun.run.sessionId, - sessionKey: turn.sessionKey, + sessionId: params.turn.followupRun.run.sessionId, + sessionKey: params.turn.sessionKey, outcome: "completed", }); - const runResult = fallbackResult.result; - const fallbackProvider = fallbackResult.provider; - const fallbackModel = fallbackResult.model; - const fallbackExhausted = fallbackResult.outcome === "exhausted"; - const settledLifecycleTerminal = - params.state.pendingLifecycleTerminal?.provider === fallbackProvider && - params.state.pendingLifecycleTerminal.model === fallbackModel - ? params.state.pendingLifecycleTerminal.backstop - : undefined; - params.state.pendingLifecycleTerminal = undefined; - if (isReplyOperationRestartAbort(turn.replyOperation)) { - settledLifecycleTerminal?.emit("end", runResult); - throw isAgentRunRestartAbortReason(params.runAbortSignal?.reason) - ? params.runAbortSignal?.reason - : createAgentRunRestartAbortError(); - } - if (isReplyOperationUserAbort(turn.replyOperation)) { - settledLifecycleTerminal?.emit("end", runResult); - await drainPendingToolTasks({ tasks: turn.pendingToolTasks, onTimeout: logVerbose }); - return { kind: "final", payload: { text: SILENT_REPLY_TOKEN } }; - } - params.commitTerminalOutcome(); - const fallbackAttempts = Array.isArray(fallbackResult.attempts) - ? fallbackResult.attempts.map((attempt) => ({ - provider: attempt.provider, - model: attempt.model, - error: attempt.error, - reason: attempt.reason || undefined, - status: typeof attempt.status === "number" ? attempt.status : undefined, - code: attempt.code || undefined, - })) - : []; - if (!fallbackExhausted) { - await params.clearRecoveredAutoFallbackPrimaryProbe({ - provider: fallbackProvider, - model: fallbackModel, - }); - } - const embeddedError = runResult.meta?.error; - const deferredLifecycleError = settledLifecycleTerminal?.getDeferredError(); - const userFacingErrorPayload = runResult.payloads?.find( - (payload) => payload.isError === true && typeof payload.text === "string", - )?.text; - const terminalErrorMessage = - deferredLifecycleError ?? - userFacingErrorPayload ?? - (embeddedError ? "Agent run failed" : undefined); - const emitSettledLifecycleError = (error: Error, extraData?: Record) => { - if (settledLifecycleTerminal) { - settledLifecycleTerminal.emit("error", error, extraData); - return; - } - emitAgentEvent({ - runId: params.runId, - lifecycleGeneration: params.state.lifecycleGeneration, - ...(turn.sessionKey ? { sessionKey: turn.sessionKey } : {}), - stream: "lifecycle", - data: { phase: "error", error: error.message, endedAt: Date.now(), ...extraData }, - }); - }; - if (embeddedError && isContextOverflowError(embeddedError.message)) { - emitSettledLifecycleError(new Error(terminalErrorMessage ?? "Agent run failed")); - defaultRuntime.error( - `Auto-compaction failed (${embeddedError.message}). Preserving existing session mapping for ${turn.sessionKey ?? turn.followupRun.run.sessionId}.`, - ); - turn.replyOperation?.fail("run_failed", embeddedError); - return { - kind: "final", - payload: markAgentRunFailureReplyPayload({ - text: buildContextOverflowRecoveryText({ - preserveSessionMapping: true, - cfg: params.runtimeConfig, - agentId: turn.followupRun.run.agentId, - primaryProvider: turn.followupRun.run.provider, - primaryModel: turn.followupRun.run.model, - runtimeProvider: params.state.attemptedRuntimeProvider, - runtimeModel: params.state.attemptedRuntimeModel, - activeSessionEntry: turn.getActiveSessionEntry(), - }), - }), - }; - } - if (embeddedError?.kind === "role_ordering") { - emitSettledLifecycleError(new Error(terminalErrorMessage ?? "Agent run failed")); - const providerRequestError = classifyProviderRequestError(embeddedError); - turn.replyOperation?.fail("run_failed", embeddedError); - const embeddedErrorText = formatErrorMessage(embeddedError).replace(/\.\s*$/, ""); - return { - kind: "final", - payload: markAgentRunFailureReplyPayload({ - text: params.shouldSurfaceToControlUi - ? `⚠️ Agent failed before reply: ${embeddedErrorText}.\nLogs: openclaw logs --follow` - : (providerRequestError?.userMessage ?? PROVIDER_CONVERSATION_STATE_ERROR_USER_MESSAGE), - }), - }; - } - const terminalMetadata = resolveAgentLifecycleTerminalMetadata(runResult.meta); - let terminalRunFailed = false; - if (fallbackExhausted) { - const exhaustionError = new Error( - terminalErrorMessage ?? "All model fallback candidates failed", - ); - terminalRunFailed = true; - if (params.modelPatch.captureFallbackFailure(fallbackAttempts) === undefined) { - params.modelPatch.captureFailure(embeddedError ?? exhaustionError); - } - emitSettledLifecycleError(exhaustionError, { - ...terminalMetadata, - fallbackExhaustedFailure: true, - }); - turn.replyOperation?.retainFailureUntilComplete(); - turn.replyOperation?.fail("run_failed", exhaustionError); - } else if (deferredLifecycleError || embeddedError) { - const terminalError = new Error(terminalErrorMessage ?? "Agent run failed"); - terminalRunFailed = true; - params.modelPatch.captureFailure(embeddedError ?? terminalError); - emitSettledLifecycleError(terminalError, terminalMetadata); - turn.replyOperation?.retainFailureUntilComplete(); - turn.replyOperation?.fail("run_failed", terminalError); - } else { - settledLifecycleTerminal?.emit("end", runResult); - } - return { - kind: "completed", - runResult, - fallbackProvider, - fallbackModel, - fallbackExhausted, - fallbackAttempts, - terminalRunFailed, - }; + return settleAgentFallbackCycle({ cycle: params, fallbackResult }); } diff --git a/src/auto-reply/reply/agent-runner-fallback-cycle.types.ts b/src/auto-reply/reply/agent-runner-fallback-cycle.types.ts new file mode 100644 index 000000000000..26a8ce643796 --- /dev/null +++ b/src/auto-reply/reply/agent-runner-fallback-cycle.types.ts @@ -0,0 +1,74 @@ +import type { RunEmbeddedAgentParams } from "../../agents/embedded-agent-runner/run/params.js"; +import type { SessionEntry } from "../../config/sessions.js"; +import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import type { AgentLifecycleTerminalBackstop } from "./agent-lifecycle-terminal.js"; +import type { + AgentRunLoopResult, + AgentTurnParams, + EmbeddedAgentRunResult, + RuntimeFallbackAttempt, +} from "./agent-runner-execution.types.js"; +import type { createAgentTurnPresentation } from "./agent-runner-presentation.js"; +import type { AgentTurnTimingTracker } from "./agent-runner-turn-timing.js"; +import type { FollowupRun } from "./queue.js"; + +export type AgentFallbackCycleState = { + lifecycleGeneration: string; + autoCompactionCount: number; + attemptedRuntimeProvider: string; + attemptedRuntimeModel: string; + bootstrapPromptWarningSignaturesSeen: string[]; + pendingLifecycleTerminal?: { + provider: string; + model: string; + backstop: AgentLifecycleTerminalBackstop; + }; +}; + +type CompletedFallbackCycle = { + kind: "completed"; + runResult: EmbeddedAgentRunResult; + fallbackProvider: string; + fallbackModel: string; + fallbackExhausted: boolean; + fallbackAttempts: RuntimeFallbackAttempt[]; + terminalRunFailed: boolean; +}; + +export type AgentFallbackCycleResult = + | CompletedFallbackCycle + | Extract; + +type AgentFallbackModelPatch = { + captureFallbackFailure: (attempts: RuntimeFallbackAttempt[]) => boolean | undefined; + captureFailure: (error: unknown) => void; +}; + +export type AgentFallbackCycleParams = { + turn: AgentTurnParams; + effectiveRun: FollowupRun["run"]; + runtimeConfig: OpenClawConfig; + liveModelSwitchRuntimeEntry?: Pick< + SessionEntry, + "agentHarnessId" | "agentRuntimeOverride" | "modelSelectionLocked" + >; + runId: string; + runAbortSignal?: AbortSignal; + currentTurnImages: Awaited< + ReturnType + >; + state: AgentFallbackCycleState; + presentation: ReturnType; + directlySentBlockKeys: Set; + notifyAgentRunStart: () => void; + signalExecutionPhaseForTyping: NonNullable; + notifyUserAboutCompaction: boolean; + timing: AgentTurnTimingTracker; + modelPatch: AgentFallbackModelPatch; + shouldSurfaceToControlUi: boolean; + commitTerminalOutcome: () => void; + clearRecoveredAutoFallbackPrimaryProbe: (candidate: { + provider: string; + model: string; + }) => Promise; +}; diff --git a/src/auto-reply/reply/agent-runner-fallback-settlement.ts b/src/auto-reply/reply/agent-runner-fallback-settlement.ts new file mode 100644 index 000000000000..5db6e5a99ff0 --- /dev/null +++ b/src/auto-reply/reply/agent-runner-fallback-settlement.ts @@ -0,0 +1,167 @@ +import { isContextOverflowError } from "../../agents/embedded-agent-helpers.js"; +import { + createAgentRunRestartAbortError, + isAgentRunRestartAbortReason, +} from "../../agents/run-termination.js"; +import { logVerbose } from "../../globals.js"; +import { emitAgentEvent } from "../../infra/agent-events.js"; +import { formatErrorMessage } from "../../infra/errors.js"; +import { defaultRuntime } from "../../runtime.js"; +import { SILENT_REPLY_TOKEN } from "../tokens.js"; +import { resolveAgentLifecycleTerminalMetadata } from "./agent-lifecycle-terminal.js"; +import { buildContextOverflowRecoveryText } from "./agent-runner-context-recovery.js"; +import { markAgentRunFailureReplyPayload } from "./agent-runner-failure-reply.js"; +import type { AgentFallbackCandidatesResult } from "./agent-runner-fallback-candidate.js"; +import type { + AgentFallbackCycleParams, + AgentFallbackCycleResult, +} from "./agent-runner-fallback-cycle.types.js"; +import { drainPendingToolTasks } from "./pending-tool-task-drain.js"; +import { + classifyProviderRequestError, + PROVIDER_CONVERSATION_STATE_ERROR_USER_MESSAGE, +} from "./provider-request-error-classifier.js"; +import { + isReplyOperationRestartAbort, + isReplyOperationUserAbort, +} from "./reply-operation-abort.js"; + +/** Settles abort, lifecycle, and terminal failure state after fallback execution. */ +export async function settleAgentFallbackCycle(params: { + cycle: AgentFallbackCycleParams; + fallbackResult: AgentFallbackCandidatesResult; +}): Promise { + const { cycle, fallbackResult } = params; + const turn = cycle.turn; + const runResult = fallbackResult.result; + const fallbackProvider = fallbackResult.provider; + const fallbackModel = fallbackResult.model; + const fallbackExhausted = fallbackResult.outcome === "exhausted"; + const settledLifecycleTerminal = + cycle.state.pendingLifecycleTerminal?.provider === fallbackProvider && + cycle.state.pendingLifecycleTerminal.model === fallbackModel + ? cycle.state.pendingLifecycleTerminal.backstop + : undefined; + cycle.state.pendingLifecycleTerminal = undefined; + if (isReplyOperationRestartAbort(turn.replyOperation)) { + settledLifecycleTerminal?.emit("end", runResult); + throw isAgentRunRestartAbortReason(cycle.runAbortSignal?.reason) + ? cycle.runAbortSignal?.reason + : createAgentRunRestartAbortError(); + } + if (isReplyOperationUserAbort(turn.replyOperation)) { + settledLifecycleTerminal?.emit("end", runResult); + await drainPendingToolTasks({ tasks: turn.pendingToolTasks, onTimeout: logVerbose }); + return { kind: "final", payload: { text: SILENT_REPLY_TOKEN } }; + } + cycle.commitTerminalOutcome(); + const fallbackAttempts = Array.isArray(fallbackResult.attempts) + ? fallbackResult.attempts.map((attempt) => ({ + provider: attempt.provider, + model: attempt.model, + error: attempt.error, + reason: attempt.reason || undefined, + status: typeof attempt.status === "number" ? attempt.status : undefined, + code: attempt.code || undefined, + })) + : []; + if (!fallbackExhausted) { + await cycle.clearRecoveredAutoFallbackPrimaryProbe({ + provider: fallbackProvider, + model: fallbackModel, + }); + } + const embeddedError = runResult.meta?.error; + const deferredLifecycleError = settledLifecycleTerminal?.getDeferredError(); + const userFacingErrorPayload = runResult.payloads?.find( + (payload) => payload.isError === true && typeof payload.text === "string", + )?.text; + const terminalErrorMessage = + deferredLifecycleError ?? + userFacingErrorPayload ?? + (embeddedError ? "Agent run failed" : undefined); + const emitSettledLifecycleError = (error: Error, extraData?: Record) => { + if (settledLifecycleTerminal) { + settledLifecycleTerminal.emit("error", error, extraData); + return; + } + emitAgentEvent({ + runId: cycle.runId, + lifecycleGeneration: cycle.state.lifecycleGeneration, + ...(turn.sessionKey ? { sessionKey: turn.sessionKey } : {}), + stream: "lifecycle", + data: { phase: "error", error: error.message, endedAt: Date.now(), ...extraData }, + }); + }; + if (embeddedError && isContextOverflowError(embeddedError.message)) { + emitSettledLifecycleError(new Error(terminalErrorMessage ?? "Agent run failed")); + defaultRuntime.error( + `Auto-compaction failed (${embeddedError.message}). Preserving existing session mapping for ${turn.sessionKey ?? turn.followupRun.run.sessionId}.`, + ); + turn.replyOperation?.fail("run_failed", embeddedError); + return { + kind: "final", + payload: markAgentRunFailureReplyPayload({ + text: buildContextOverflowRecoveryText({ + preserveSessionMapping: true, + cfg: cycle.runtimeConfig, + agentId: turn.followupRun.run.agentId, + primaryProvider: turn.followupRun.run.provider, + primaryModel: turn.followupRun.run.model, + runtimeProvider: cycle.state.attemptedRuntimeProvider, + runtimeModel: cycle.state.attemptedRuntimeModel, + activeSessionEntry: turn.getActiveSessionEntry(), + }), + }), + }; + } + if (embeddedError?.kind === "role_ordering") { + emitSettledLifecycleError(new Error(terminalErrorMessage ?? "Agent run failed")); + const providerRequestError = classifyProviderRequestError(embeddedError); + turn.replyOperation?.fail("run_failed", embeddedError); + const embeddedErrorText = formatErrorMessage(embeddedError).replace(/\.\s*$/, ""); + return { + kind: "final", + payload: markAgentRunFailureReplyPayload({ + text: cycle.shouldSurfaceToControlUi + ? `⚠️ Agent failed before reply: ${embeddedErrorText}.\nLogs: openclaw logs --follow` + : (providerRequestError?.userMessage ?? PROVIDER_CONVERSATION_STATE_ERROR_USER_MESSAGE), + }), + }; + } + const terminalMetadata = resolveAgentLifecycleTerminalMetadata(runResult.meta); + let terminalRunFailed = false; + if (fallbackExhausted) { + const exhaustionError = new Error( + terminalErrorMessage ?? "All model fallback candidates failed", + ); + terminalRunFailed = true; + if (cycle.modelPatch.captureFallbackFailure(fallbackAttempts) === undefined) { + cycle.modelPatch.captureFailure(embeddedError ?? exhaustionError); + } + emitSettledLifecycleError(exhaustionError, { + ...terminalMetadata, + fallbackExhaustedFailure: true, + }); + turn.replyOperation?.retainFailureUntilComplete(); + turn.replyOperation?.fail("run_failed", exhaustionError); + } else if (deferredLifecycleError || embeddedError) { + const terminalError = new Error(terminalErrorMessage ?? "Agent run failed"); + terminalRunFailed = true; + cycle.modelPatch.captureFailure(embeddedError ?? terminalError); + emitSettledLifecycleError(terminalError, terminalMetadata); + turn.replyOperation?.retainFailureUntilComplete(); + turn.replyOperation?.fail("run_failed", terminalError); + } else { + settledLifecycleTerminal?.emit("end", runResult); + } + return { + kind: "completed", + runResult, + fallbackProvider, + fallbackModel, + fallbackExhausted, + fallbackAttempts, + terminalRunFailed, + }; +} diff --git a/src/auto-reply/reply/agent-runner-runtime-selection.test.ts b/src/auto-reply/reply/agent-runner-runtime-selection.test.ts new file mode 100644 index 000000000000..2cf93cebf55d --- /dev/null +++ b/src/auto-reply/reply/agent-runner-runtime-selection.test.ts @@ -0,0 +1,104 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { testing as cliBackendsTesting } from "../../agents/cli-backends.js"; +import { resolveSessionRuntimeOverrideForProvider } from "../../agents/session-runtime-compat.js"; +import type { SessionEntry } from "../../config/sessions.js"; + +describe("resolveSessionRuntimeOverrideForProvider", () => { + it("honors an explicit OpenClaw override for OpenAI", () => { + expect( + resolveSessionRuntimeOverrideForProvider({ + provider: "openai", + entry: { agentRuntimeOverride: "openclaw" } as SessionEntry, + }), + ).toBe("openclaw"); + }); + + afterEach(() => { + cliBackendsTesting.resetDepsForTest(); + }); + + it("ignores unsupported session runtime pins", () => { + expect( + resolveSessionRuntimeOverrideForProvider({ + provider: "openai", + entry: { agentRuntimeOverride: "unsupported-runtime" }, + }), + ).toBeUndefined(); + }); + + it.each([ + { provider: "openai", expected: "codex" }, + { provider: "codex", expected: "codex" }, + { provider: "anthropic", expected: undefined }, + ])("resolves Codex runtime compatibility for $provider", ({ provider, expected }) => { + expect( + resolveSessionRuntimeOverrideForProvider({ + provider, + entry: { agentRuntimeOverride: "codex" }, + }), + ).toBe(expected); + }); + + it("does not treat an observed harness as a future-turn override", () => { + expect( + resolveSessionRuntimeOverrideForProvider({ + provider: "anthropic", + entry: { agentHarnessId: "codex" }, + }), + ).toBeUndefined(); + }); + + it("keeps a locked harness pin ahead of a conflicting runtime override", () => { + expect( + resolveSessionRuntimeOverrideForProvider({ + provider: "anthropic", + entry: { + agentHarnessId: "codex", + agentRuntimeOverride: "claude-cli", + modelSelectionLocked: true, + }, + }), + ).toBe("codex"); + }); + it("keeps CLI runtime pins only when the runtime serves the selected provider", () => { + cliBackendsTesting.setDepsForTest({ + resolveRuntimeCliBackends: () => [], + resolvePluginSetupCliBackend: ({ backend, config }) => + backend === "claude-cli" && config + ? { + pluginId: "anthropic", + backend: { + id: "claude-cli", + modelProvider: "anthropic", + config: { command: "claude" }, + bundleMcp: false, + }, + } + : undefined, + }); + const cfg = { + agents: { + defaults: { + cliBackends: { + "claude-cli": { command: "claude" }, + }, + }, + }, + }; + + expect( + resolveSessionRuntimeOverrideForProvider({ + provider: "anthropic", + entry: { agentRuntimeOverride: "claude-cli" }, + cfg, + }), + ).toBe("claude-cli"); + expect( + resolveSessionRuntimeOverrideForProvider({ + provider: "openai", + entry: { agentRuntimeOverride: "claude-cli" }, + cfg, + }), + ).toBeUndefined(); + }); +});