fix(code-mode): reject overflow before dispatch (#128030)

Reject over-cap QuickJS bridge frontiers before any admitted host calls start, avoiding partial side effects and ambiguous mutation outcomes. Covers #128012. Reported by Hannes Rudolph.

Co-authored-by: RoboClaw <309084314+roboclaw-bot@users.noreply.github.com>
Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com>

src/agents/code-mode.bridge.test.ts

@@ -155,6 +155,50 @@ describe("Code Mode bridge settlement and cancellation", () => {
     expect(testing.activeRuns.size).toBe(0);
   });
 
+  it("rejects an over-cap bridge frontier before dispatching its admitted prefix", async () => {
+    const catalogRef = createToolSearchCatalogRef();
+    const config = {
+      tools: { codeMode: { enabled: true, maxPendingToolCalls: 2 } },
+    } as never;
+    const ctx = {
+      config,
+      runtimeConfig: config,
+      sessionId: "session-code-mode",
+      sessionKey: "agent:main:main",
+      runId: "run-code-mode",
+      catalogRef,
+    };
+    const codeModeTools = createCodeModeTools(ctx);
+    const mutation = pluginTool("fake_mutation", "Record a side effect");
+    applyCodeModeCatalog({
+      tools: [...codeModeTools, mutation],
+      config,
+      sessionId: "session-code-mode",
+      sessionKey: "agent:main:main",
+      runId: "run-code-mode",
+      catalogRef,
+    });
+
+    const details = resultDetails(
+      await expectDefined(codeModeTools[0], "Code Mode exec test invariant").execute(
+        "code-call-frontier-overflow",
+        {
+          code: `return await Promise.all(
+            Array.from({ length: 3 }, (_, index) => fake_mutation({ index })),
+          );`,
+        },
+      ),
+    );
+
+    expect(mutation.execute).not.toHaveBeenCalled();
+    expect(details).toMatchObject({
+      status: "failed",
+      code: "invalid_input",
+      bridgeDispatchStarted: false,
+    });
+    expect(testing.activeRuns.size).toBe(0);
+  });
+
   it("yields nested exec before the Code Mode deadline when continuation args are omitted", async () => {
     const catalogRef = createToolSearchCatalogRef();
     const config = {

src/agents/code-mode.worker.ts

@@ -34,8 +34,9 @@ type VmRun = {
   didTimeout: () => boolean;
 };
 
-// Each worker handles exactly one exec/resume payload, so cancellations are run-scoped.
+// Each worker handles exactly one exec/resume payload, so bridge state is run-scoped.
 const canceledBridgeRequestIds: string[] = [];
+let bridgeAdmissionFailure: CodeModeWorkerFailure | undefined;
 
 // QuickJS error stacks are backtrace frames only ("    at file:line:col"), with
 // no leading "Name: message" header like V8. Returning .stack alone therefore
@@ -76,7 +77,11 @@ function createHostRequestHandler(params: {
 ) => JSValueHandle {
   return (methodHandle, argsHandle, bridgeIdHandle) => {
     if (params.pendingRequests.length >= params.config.maxPendingToolCalls) {
-      throw new Error("too many pending code mode tool calls");
+      bridgeAdmissionFailure ??= new CodeModeWorkerFailure(
+        "invalid_input",
+        "too many pending code mode tool calls",
+      );
+      throw bridgeAdmissionFailure;
     }
     const method = methodHandle.toString();
     if (
@@ -363,6 +368,9 @@ async function runVmExecution(params: {
   try {
     params.prepare();
     params.vm.executePendingJobs();
+    if (bridgeAdmissionFailure) {
+      throw bridgeAdmissionFailure;
+    }
     output = takeOutput(params.vm);
     const resultHandle = params.vm.global.getProp("__openclawResult");
     try {