fix(gateway): warn on placement session evidence pipeline failure (#124635)

The worker placement session evidence resolver swallowed every pipeline
failure into a bare catch that returned "unknown" for all placements.
Fail-open is correct for retirement safety, but the silent catch hid a
broken evidence pipeline (bad config, store corruption) behind indefinite
placement retention with no operator-visible signal. Record the fact at
the boundary that owns it: warn with the error before returning unknown.

Also collapse the byte-identical publish branches in
acquirePreparedModelRuntimeLeaseFromOwners: staleDynamicOwner and
missing-owner both published the exact same snapshot call; one branch
now owns publication with the invariant comment preserved.
This commit is contained in:
Peter Steinberger
2026-08-16 07:29:13 -07:00
committed by GitHub
parent a11e9d673b
commit 29e8bebcbe
3 changed files with 46 additions and 15 deletions
+4 -14
View File
@@ -189,22 +189,12 @@ export async function acquirePreparedModelRuntimeLeaseFromOwners(
workspacePluginRootPresent,
context,
);
if (staleDynamicOwner) {
// Existing leases retain their immutable snapshot. Publish a distinct owner so their release
// cannot delete the replacement generation admitted for new work at the same dynamic key.
snapshot = await publishModelRuntimeSnapshot(
input,
context.owners,
context.agentBuildCompletions,
context.getBuildTimeoutMs(),
undefined,
provenance,
options.catalogMode,
reusablePluginGeneration,
);
} else if (existing) {
if (existing && !staleDynamicOwner) {
snapshot = await context.prepareSnapshot(input);
} else {
// Fresh keys publish a first generation; stale dynamic owners publish a distinct
// replacement owner because existing leases retain their immutable snapshot, so
// their release cannot delete the generation admitted for new work at this key.
snapshot = await publishModelRuntimeSnapshot(
input,
context.owners,
@@ -1,6 +1,21 @@
import fsSync from "node:fs";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
const evidenceWarnSpy = vi.hoisted(() => vi.fn());
vi.mock("../logging/subsystem.js", async () => {
const actual =
await vi.importActual<typeof import("../logging/subsystem.js")>("../logging/subsystem.js");
return {
...actual,
createSubsystemLogger: (subsystem: string) => {
const logger = actual.createSubsystemLogger(subsystem);
return subsystem === "gateway/placement-session-evidence"
? { ...logger, warn: evidenceWarnSpy }
: logger;
},
};
});
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { resetConfigRuntimeState, setRuntimeConfigSnapshot } from "../config/config.js";
import * as sessionAccessor from "../config/sessions/session-accessor.js";
@@ -27,6 +42,7 @@ afterEach(() => {
resetConfigRuntimeState();
resolveTargetsReadOnlySpy.mockClear();
readIdentityEvidenceBatchSpy.mockClear();
evidenceWarnSpy.mockClear();
});
function localPlacement(
@@ -209,6 +225,23 @@ describe("worker placement session evidence", () => {
});
});
it("warns instead of silently swallowing resolver pipeline failures", async () => {
const stateDir = tempDirs.make("openclaw-placement-session-pipeline-failure-");
await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => {
resolveTargetsReadOnlySpy.mockImplementationOnce(() => {
throw new Error("evidence pipeline exploded");
});
const placement = localPlacement("session-pipeline-failure", "agent:main:pipeline-failure");
await expect(resolvePlacementEvidence(placement)).resolves.toBe("unknown");
expect(evidenceWarnSpy).toHaveBeenCalledOnce();
expect(evidenceWarnSpy).toHaveBeenCalledWith(
expect.stringContaining("session evidence resolution failed"),
{ error: expect.objectContaining({ message: "evidence pipeline exploded" }) },
);
});
});
it("prepares targets once and reads only exact session rows for a placement batch", async () => {
const stateDir = tempDirs.make("openclaw-placement-session-evidence-batch-");
await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => {
@@ -1,6 +1,7 @@
import { getRuntimeConfig } from "../config/config.js";
import type { SessionStoreTargetsReadCache } from "../config/sessions/targets-read-availability.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import {
isIncognitoSessionKey,
normalizeAgentId,
@@ -15,6 +16,8 @@ import type {
PlacementSessionEvidenceResolver,
} from "./worker-environments/placement-session-retirement.js";
const log = createSubsystemLogger("gateway/placement-session-evidence");
const loadPlacementSessionEvidenceRuntime = createLazyRuntimeModule(async () => {
const [sessionTargetsReadAvailability, sessionAccessor] = await Promise.all([
import("../config/sessions/targets-read-availability.js"),
@@ -138,7 +141,12 @@ export async function createWorkerPlacementSessionEvidenceResolver(
}
}
return async (placement) => evidenceByPlacement.get(placement) ?? "unknown";
} catch {
} catch (error) {
// "unknown" keeps retirement fail-open, but a silent catch would hide a broken
// evidence pipeline (bad config, store corruption) behind indefinite retention.
log.warn("worker placement session evidence resolution failed; treating all as unknown", {
error,
});
return async () => "unknown";
}
}