feat(auto-reply): record message-tool mute outcomes (#124830)

* feat(auto-reply): record message-tool mute outcomes

* fix(auto-reply): require source delivery evidence

* test(infra): preserve historical agent schema fixtures

src/auto-reply/reply/agent-runner-execution-contract.test.ts

@@ -72,11 +72,14 @@ describe("executeAgentTurn contract", () => {
       },
     };
 
-    const result = await executeAgentTurn(
-      createMinimalRunAgentTurnParams({ replyOperation: unsettledOperation }),
-    );
+    const params = createMinimalRunAgentTurnParams({ replyOperation: unsettledOperation });
+    params.followupRun.run.sourceReplyDeliveryMode = "message_tool_only";
+    const result = await executeAgentTurn(params);
 
     expect(result.outcome).toEqual({ kind: "aborted", reason: "restart" });
     expect(complete).toHaveBeenCalledOnce();
+    expect(state.recordMessageToolRunOutcomeMock).toHaveBeenCalledWith(
+      expect.objectContaining({ outcome: "mute", runStatus: "aborted" }),
+    );
   });
 });

src/auto-reply/reply/agent-runner-execution-message-tools.test.ts

@@ -6,6 +6,7 @@ import {
   getExecuteAgentTurnForTest,
   createMockTypingSignaler,
   createFollowupRun,
+  createMinimalRunAgentTurnParams,
 } from "./agent-runner-execution.test-support.js";
 import type {
   FallbackRunnerParams,
@@ -79,7 +80,11 @@ describe("executeAgentTurn: message tool progress", () => {
       });
       releaseItemEvent?.();
       await itemEventPromise;
-      return { payloads: [{ text: "NO_REPLY" }], meta: {} };
+      return {
+        payloads: [{ text: "NO_REPLY" }],
+        didDeliverSourceReplyViaMessageTool: true,
+        meta: {},
+      };
     });
 
     const executeAgentTurn = await getExecuteAgentTurnForTest();
@@ -121,6 +126,41 @@ describe("executeAgentTurn: message tool progress", () => {
     );
     expect(onItemEvent).toHaveBeenCalledTimes(1);
     expect(onCommandOutput).not.toHaveBeenCalled();
+    expect(state.recordMessageToolRunOutcomeMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        outcome: "tool_delivered",
+        runStatus: "completed",
+      }),
+    );
+  });
+
+  it("records mute when a message-tool-only run completes without a send", async () => {
+    state.runEmbeddedAgentMock.mockResolvedValueOnce({ payloads: [], meta: {} });
+    const followupRun = createFollowupRun();
+    followupRun.run.sourceReplyDeliveryMode = "message_tool_only";
+
+    const executeAgentTurn = await getExecuteAgentTurnForTest();
+    await executeAgentTurn(createMinimalRunAgentTurnParams({ followupRun }));
+
+    expect(state.recordMessageToolRunOutcomeMock).toHaveBeenCalledWith(
+      expect.objectContaining({ outcome: "mute", runStatus: "completed" }),
+    );
+  });
+
+  it("classifies a failed message-tool-only run separately from omission", async () => {
+    state.runEmbeddedAgentMock.mockResolvedValueOnce({
+      payloads: [],
+      meta: { error: { message: "provider crashed" } },
+    });
+    const followupRun = createFollowupRun();
+    followupRun.run.sourceReplyDeliveryMode = "message_tool_only";
+
+    const executeAgentTurn = await getExecuteAgentTurnForTest();
+    await executeAgentTurn(createMinimalRunAgentTurnParams({ followupRun }));
+
+    expect(state.recordMessageToolRunOutcomeMock).toHaveBeenCalledWith(
+      expect.objectContaining({ outcome: "mute", runStatus: "errored" }),
+    );
   });
 
   it("preserves message-tool-only suppression across fallback candidates", async () => {

src/auto-reply/reply/agent-runner-execution.test-support.ts

@@ -63,6 +63,7 @@ const state = vi.hoisted(() => ({
   updateSessionStoreMock: vi.fn(),
   resolveCurrentTurnImagesMock: vi.fn(),
   peekSessionMcpRuntimeMock: vi.fn(),
+  recordMessageToolRunOutcomeMock: vi.fn(),
   productionBuildEmbeddedRunExecutionParams: undefined as
     | typeof buildEmbeddedRunExecutionParams
     | undefined,
@@ -222,6 +223,10 @@ vi.mock("../../infra/agent-run-registry.js", async () => {
   };
 });
 
+vi.mock("../../infra/message-tool-run-outcome-store.js", () => ({
+  recordMessageToolRunOutcome: (params: unknown) => state.recordMessageToolRunOutcomeMock(params),
+}));
+
 vi.mock("../../runtime.js", () => ({
   defaultRuntime: {
     error: vi.fn(),
@@ -645,6 +650,7 @@ export function setupAgentRunnerExecutionTestState() {
     state.updateSessionStoreMock.mockReset();
     state.resolveCurrentTurnImagesMock.mockReset();
     state.peekSessionMcpRuntimeMock.mockReset();
+    state.recordMessageToolRunOutcomeMock.mockReset();
     state.productionBuildEmbeddedRunExecutionParams = undefined;
     state.peekSessionMcpRuntimeMock.mockReturnValue(undefined);
     state.resolveCurrentTurnImagesMock.mockImplementation(

src/auto-reply/reply/agent-runner-execution.ts

@@ -13,6 +13,7 @@ import {
   classifyFailoverReason,
   isContextOverflowError,
 } from "../../agents/embedded-agent-helpers.js";
+import { hasCompletedSourceReplyDeliveryEvidence } from "../../agents/embedded-agent-runner/delivery-evidence.js";
 import type { EmbeddedAgentExecutionPhase } from "../../agents/embedded-agent-runner/execution-phase.js";
 import type { RunEmbeddedAgentParams } from "../../agents/embedded-agent-runner/run/params.js";
 import { runEmbeddedAgent } from "../../agents/embedded-agent.js";
@@ -31,7 +32,9 @@ import { clearAgentRunContext, registerAgentRunContext } from "../../infra/agent
 import { emitAgentRunStatusEvent } from "../../infra/agent-run-status-events.js";
 import { isDiagnosticsEnabled } from "../../infra/diagnostic-events.js";
 import { formatErrorMessage } from "../../infra/errors.js";
+import { recordMessageToolRunOutcome } from "../../infra/message-tool-run-outcome-store.js";
 import { logSessionTurnCreated } from "../../logging/diagnostic.js";
+import { createSubsystemLogger } from "../../logging/subsystem.js";
 import { isInternalMessageChannel } from "../../utils/message-channel.js";
 import type { ReplyPayload } from "../types.js";
 import {
@@ -80,6 +83,8 @@ type InternalFollowupRun = FollowupRun & {
   mediaImageLayout?: CurrentTurnImages["mediaImageLayout"];
 };
 
+const messageToolOutcomeLog = createSubsystemLogger("auto-reply/message-tool-outcome");
+
 function resolveRunStartupPhase(
   phase: EmbeddedAgentExecutionPhase,
 ): ChatRunStartupPhase | undefined {
@@ -531,7 +536,7 @@ async function executeAgentTurnInternal(
 }
 
 /** Runs the agent turn with provider/model fallback, retry, and closed settlement. */
-export async function executeAgentTurn(params: AgentTurnParams): Promise<AgentTurnExecutionResult> {
+async function executeAgentTurnOutcome(params: AgentTurnParams): Promise<AgentTurnExecutionResult> {
   const runId = params.opts?.runId ?? crypto.randomUUID();
   const executionParams =
     params.opts?.runId === runId ? params : { ...params, opts: { ...params.opts, runId } };
@@ -645,3 +650,70 @@ export async function executeAgentTurn(params: AgentTurnParams): Promise<AgentTu
     throw error;
   }
 }
+
+function recordMessageToolOnlyRunOutcome(
+  params: AgentTurnParams,
+  result: AgentTurnExecutionResult | undefined,
+): void {
+  const sourceReplyDeliveryMode =
+    params.followupRun.run.sourceReplyDeliveryMode ?? params.opts?.sourceReplyDeliveryMode;
+  if (sourceReplyDeliveryMode !== "message_tool_only") {
+    return;
+  }
+  const sessionKey = params.sessionKey ?? params.followupRun.run.sessionKey;
+  if (!sessionKey) {
+    messageToolOutcomeLog.warn("message-tool-only run outcome missing session key", {
+      runId: result?.runId ?? params.opts?.runId,
+      agentId: params.followupRun.run.agentId,
+    });
+    return;
+  }
+  const outcome = result?.outcome;
+  const resolved =
+    outcome?.kind === "settled" || outcome?.kind === "rejected" ? outcome.resolved : undefined;
+  const provider = resolved?.provider ?? params.followupRun.run.provider;
+  const model = resolved?.model ?? params.followupRun.run.model;
+  const runStatus: "completed" | "errored" | "aborted" =
+    outcome?.kind === "aborted" || (outcome?.kind === "settled" && outcome.abortReason)
+      ? "aborted"
+      : !outcome || outcome.kind === "rejected" || outcome.status === "failed"
+        ? "errored"
+        : "completed";
+  const toolDelivered =
+    outcome?.kind === "settled" && hasCompletedSourceReplyDeliveryEvidence(outcome.result);
+  const values = {
+    runId: result?.runId ?? params.opts?.runId ?? "unknown",
+    sessionKey,
+    agentId: params.followupRun.run.agentId,
+    provider,
+    model,
+    outcome: toolDelivered ? ("tool_delivered" as const) : ("mute" as const),
+    runStatus,
+    occurredAt: Date.now(),
+    storePath: params.storePath,
+  };
+  try {
+    recordMessageToolRunOutcome(values);
+    messageToolOutcomeLog.info("recorded message-tool-only run outcome", values);
+  } catch (error) {
+    messageToolOutcomeLog.warn("failed to record message-tool-only run outcome", {
+      ...values,
+      error: formatErrorMessage(error),
+    });
+  }
+}
+
+/** Runs the agent turn and records its message-tool-only visible-outcome fact once. */
+export async function executeAgentTurn(params: AgentTurnParams): Promise<AgentTurnExecutionResult> {
+  const runId = params.opts?.runId ?? crypto.randomUUID();
+  const executionParams =
+    params.opts?.runId === runId ? params : { ...params, opts: { ...params.opts, runId } };
+  try {
+    const result = await executeAgentTurnOutcome(executionParams);
+    recordMessageToolOnlyRunOutcome(executionParams, result);
+    return result;
+  } catch (error) {
+    recordMessageToolOnlyRunOutcome(executionParams, undefined);
+    throw error;
+  }
+}

src/infra/message-tool-run-outcome-store.test.ts

@@ -0,0 +1,95 @@
+import { afterEach, describe, expect, it } from "vitest";
+import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
+import {
+  closeOpenClawAgentDatabasesForTest,
+  openOpenClawAgentDatabase,
+} from "../state/openclaw-agent-db.js";
+import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
+import { recordMessageToolRunOutcome } from "./message-tool-run-outcome-store.js";
+
+const tempDirs = useAutoCleanupTempDirTracker(afterEach);
+
+function createEnv(): NodeJS.ProcessEnv {
+  return { OPENCLAW_STATE_DIR: tempDirs.make("openclaw-message-tool-outcome-") };
+}
+
+afterEach(() => {
+  closeOpenClawAgentDatabasesForTest();
+  closeOpenClawStateDatabaseForTest();
+});
+
+describe("message-tool run outcome store", () => {
+  it("lazily records typed completion facts in an existing same-version database", () => {
+    const env = createEnv();
+    const database = openOpenClawAgentDatabase({ agentId: "main", env });
+    database.db.exec("DROP TABLE message_tool_run_outcomes;");
+    closeOpenClawAgentDatabasesForTest();
+
+    for (const [runId, outcome, runStatus] of [
+      ["run-delivered", "tool_delivered", "completed"],
+      ["run-mute", "mute", "completed"],
+      ["run-error", "mute", "errored"],
+    ] as const) {
+      recordMessageToolRunOutcome({
+        runId,
+        sessionKey: "agent:main:main",
+        agentId: "main",
+        provider: "openai",
+        model: "gpt-5.6-luna",
+        outcome,
+        runStatus,
+        occurredAt: 100,
+        env,
+      });
+    }
+
+    expect(
+      openOpenClawAgentDatabase({ agentId: "main", env })
+        .db.prepare("SELECT run_id, outcome, run_status FROM message_tool_run_outcomes ORDER BY id")
+        .all(),
+    ).toEqual([
+      { run_id: "run-delivered", outcome: "tool_delivered", run_status: "completed" },
+      { run_id: "run-mute", outcome: "mute", run_status: "completed" },
+      { run_id: "run-error", outcome: "mute", run_status: "errored" },
+    ]);
+  });
+
+  it("prunes the per-agent operational history to 10,000 newest rows", () => {
+    const env = createEnv();
+    const database = openOpenClawAgentDatabase({ agentId: "main", env });
+    database.db.exec(`
+      WITH RECURSIVE rows(value) AS (
+        SELECT 1
+        UNION ALL
+        SELECT value + 1 FROM rows WHERE value <= 10000
+      )
+      INSERT INTO message_tool_run_outcomes (
+        run_id, session_key, agent_id, provider, model, outcome, run_status, occurred_at
+      )
+      SELECT
+        'seed-' || value, 'agent:main:main', 'main', 'openai', 'gpt-5.6-luna',
+        'mute', 'completed', value
+      FROM rows;
+    `);
+
+    recordMessageToolRunOutcome({
+      runId: "newest",
+      sessionKey: "agent:main:main",
+      agentId: "main",
+      provider: "openai",
+      model: "gpt-5.6-luna",
+      outcome: "tool_delivered",
+      runStatus: "completed",
+      occurredAt: 20_000,
+      env,
+    });
+
+    expect(
+      database.db
+        .prepare(
+          "SELECT COUNT(*) AS count, MIN(occurred_at) AS oldest, MAX(occurred_at) AS newest FROM message_tool_run_outcomes",
+        )
+        .get(),
+    ).toEqual({ count: 10_000, oldest: 3, newest: 20_000 });
+  });
+});

src/infra/message-tool-run-outcome-store.ts

@@ -0,0 +1,69 @@
+import type { Insertable } from "kysely";
+import {
+  resolveSqliteScope,
+  toDatabaseOptions,
+} from "../config/sessions/session-accessor.sqlite-scope.js";
+import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js";
+import {
+  openOpenClawAgentDatabase,
+  runOpenClawAgentWriteTransaction,
+} from "../state/openclaw-agent-db.js";
+import { ensureMessageToolRunOutcomeSchema } from "../state/openclaw-agent-message-tool-outcome-schema.js";
+import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js";
+
+const MESSAGE_TOOL_RUN_OUTCOME_MAX_ROWS = 10_000;
+
+type MessageToolRunOutcomeTable = OpenClawAgentKyselyDatabase["message_tool_run_outcomes"];
+type MessageToolRunOutcomeDatabase = Pick<OpenClawAgentKyselyDatabase, "message_tool_run_outcomes">;
+type MessageToolRunOutcomeInsert = Insertable<MessageToolRunOutcomeTable>;
+
+/** Records one bounded completion fact for a message-tool-only run. */
+export function recordMessageToolRunOutcome(params: {
+  runId: string;
+  sessionKey: string;
+  agentId: string;
+  provider: string;
+  model: string;
+  outcome: "tool_delivered" | "mute";
+  runStatus: "completed" | "errored" | "aborted";
+  occurredAt: number;
+  storePath?: string;
+  env?: NodeJS.ProcessEnv;
+}): void {
+  const values: MessageToolRunOutcomeInsert = {
+    run_id: params.runId,
+    session_key: params.sessionKey,
+    agent_id: params.agentId,
+    provider: params.provider,
+    model: params.model,
+    outcome: params.outcome,
+    run_status: params.runStatus,
+    occurred_at: params.occurredAt,
+  };
+  const databaseOptions = toDatabaseOptions(resolveSqliteScope(params));
+  ensureMessageToolRunOutcomeSchema(openOpenClawAgentDatabase(databaseOptions).db);
+  runOpenClawAgentWriteTransaction(
+    ({ db }) => {
+      const agentDb = getNodeSqliteKysely<MessageToolRunOutcomeDatabase>(db);
+      executeSqliteQuerySync(db, agentDb.insertInto("message_tool_run_outcomes").values(values));
+      executeSqliteQuerySync(
+        db,
+        agentDb
+          .deleteFrom("message_tool_run_outcomes")
+          .where(
+            "id",
+            "in",
+            agentDb
+              .selectFrom("message_tool_run_outcomes")
+              .select("id")
+              .orderBy("occurred_at", "desc")
+              .orderBy("id", "desc")
+              .limit(2_147_483_647)
+              .offset(MESSAGE_TOOL_RUN_OUTCOME_MAX_ROWS),
+          ),
+      );
+    },
+    databaseOptions,
+    { operationLabel: "message-tool.run-outcome.record" },
+  );
+}

src/infra/state-migrations.media-persistence.historical-schema.test-support.ts

@@ -49,6 +49,11 @@ export function historicalV15AgentSchemaSql(): string {
     "CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_entry_valid_pending",
     "CREATE TABLE IF NOT EXISTS session_windows (",
   );
+  sql = removeSchemaRange(
+    sql,
+    "CREATE TABLE IF NOT EXISTS message_tool_run_outcomes (",
+    "CREATE TABLE IF NOT EXISTS transcript_events (",
+  );
   sql = removeSchemaRange(
     sql,
     "CREATE TABLE IF NOT EXISTS context_engine_turn_outbox (",

src/state/openclaw-agent-db-schema-helpers.ts

@@ -25,6 +25,7 @@ import { CONTEXT_ENGINE_TURN_OUTBOX_TABLE } from "./openclaw-agent-context-engin
 import { OPENCLAW_AGENT_SCHEMA_VERSION } from "./openclaw-agent-db-contract.js";
 import { OpenClawAgentDatabaseMediaMigrationRequiredError } from "./openclaw-agent-db-migration-required.js";
 import { ensureSessionEntryValidityProjection } from "./openclaw-agent-db-session-migrations.js";
+import { MESSAGE_TOOL_RUN_OUTCOMES_TABLE } from "./openclaw-agent-message-tool-outcome-schema.js";
 import { OPENCLAW_AGENT_SCHEMA_SQL } from "./openclaw-agent-schema.js";
 import {
   AGENT_V14_ADDITIVE_SCHEMA_SQL,
@@ -50,6 +51,7 @@ const AGENT_SCHEMA_COMPATIBILITY = {
     MEMORY_INDEX_CHUNK_PROVENANCE_TABLE,
     MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE,
     CONTEXT_ENGINE_TURN_OUTBOX_TABLE,
+    MESSAGE_TOOL_RUN_OUTCOMES_TABLE,
     SESSION_TRANSCRIPT_ARCHIVES_TABLE,
     STANDING_INTENTS_TABLE,
     STANDING_INTENTS_FTS_TABLE,

src/state/openclaw-agent-db.generated.d.ts

@@ -194,6 +194,18 @@ export interface MemoryIndexState {
   revision: number;
 }
 
+export interface MessageToolRunOutcomes {
+  agent_id: string;
+  id: Generated<number>;
+  model: string;
+  occurred_at: number;
+  outcome: string;
+  provider: string;
+  run_id: string;
+  run_status: string;
+  session_key: string;
+}
+
 export interface SchemaMeta {
   agent_id: string | null;
   app_version: string | null;
@@ -459,6 +471,7 @@ export interface DB {
   memory_index_meta: MemoryIndexMeta;
   memory_index_sources: MemoryIndexSources;
   memory_index_state: MemoryIndexState;
+  message_tool_run_outcomes: MessageToolRunOutcomes;
   schema_meta: SchemaMeta;
   session_conversations: SessionConversations;
   session_key_contract: SessionKeyContract;

src/state/openclaw-agent-message-tool-outcome-schema.ts

@@ -0,0 +1,29 @@
+import type { DatabaseSync } from "node:sqlite";
+import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js";
+import { OPENCLAW_AGENT_SCHEMA_SQL } from "./openclaw-agent-schema.js";
+
+export const MESSAGE_TOOL_RUN_OUTCOMES_TABLE = "message_tool_run_outcomes";
+
+const SCHEMA_START = `CREATE TABLE IF NOT EXISTS ${MESSAGE_TOOL_RUN_OUTCOMES_TABLE} (`;
+const SCHEMA_END = "CREATE TABLE IF NOT EXISTS transcript_events (";
+const ENSURED_DATABASES = new WeakSet<DatabaseSync>();
+
+function messageToolRunOutcomeSchemaSql(): string {
+  const start = OPENCLAW_AGENT_SCHEMA_SQL.indexOf(SCHEMA_START);
+  const end = OPENCLAW_AGENT_SCHEMA_SQL.indexOf(SCHEMA_END, start);
+  if (start === -1 || end === -1) {
+    throw new Error("OpenClaw message-tool run outcome schema markers are missing.");
+  }
+  return OPENCLAW_AGENT_SCHEMA_SQL.slice(start, end);
+}
+
+/** Lazily installs the additive outcome table on first use. */
+export function ensureMessageToolRunOutcomeSchema(db: DatabaseSync): void {
+  if (ENSURED_DATABASES.has(db)) {
+    return;
+  }
+  runSqliteImmediateTransactionSync(db, () => {
+    db.exec(messageToolRunOutcomeSchemaSql()); // sqlite-allow-raw -- Canonical additive DDL only.
+  });
+  ENSURED_DATABASES.add(db);
+}

src/state/openclaw-agent-schema.sql

@@ -315,6 +315,21 @@ CREATE TABLE IF NOT EXISTS heartbeat_outcomes (
   FOREIGN KEY (session_key) REFERENCES session_nodes(session_key) ON DELETE CASCADE
 ) STRICT;
 
+CREATE TABLE IF NOT EXISTS message_tool_run_outcomes (
+  id INTEGER PRIMARY KEY,
+  run_id TEXT NOT NULL,
+  session_key TEXT NOT NULL,
+  agent_id TEXT NOT NULL,
+  provider TEXT NOT NULL,
+  model TEXT NOT NULL,
+  outcome TEXT NOT NULL CHECK (outcome IN ('tool_delivered', 'mute')),
+  run_status TEXT NOT NULL CHECK (run_status IN ('completed', 'errored', 'aborted')),
+  occurred_at INTEGER NOT NULL
+) STRICT;
+
+CREATE INDEX IF NOT EXISTS idx_agent_message_tool_run_outcomes_occurred
+  ON message_tool_run_outcomes(occurred_at DESC, id DESC);
+
 CREATE TABLE IF NOT EXISTS transcript_events (
   session_id TEXT NOT NULL,
   seq INTEGER NOT NULL,