mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-23 10:55:31 -06:00
fix(gateway): warn on placement session evidence pipeline failure (#124635)
The worker placement session evidence resolver swallowed every pipeline failure into a bare catch that returned "unknown" for all placements. Fail-open is correct for retirement safety, but the silent catch hid a broken evidence pipeline (bad config, store corruption) behind indefinite placement retention with no operator-visible signal. Record the fact at the boundary that owns it: warn with the error before returning unknown. Also collapse the byte-identical publish branches in acquirePreparedModelRuntimeLeaseFromOwners: staleDynamicOwner and missing-owner both published the exact same snapshot call; one branch now owns publication with the invariant comment preserved.
This commit is contained in:
committed by
GitHub
parent
a11e9d673b
commit
29e8bebcbe
@@ -189,22 +189,12 @@ export async function acquirePreparedModelRuntimeLeaseFromOwners(
|
||||
workspacePluginRootPresent,
|
||||
context,
|
||||
);
|
||||
if (staleDynamicOwner) {
|
||||
// Existing leases retain their immutable snapshot. Publish a distinct owner so their release
|
||||
// cannot delete the replacement generation admitted for new work at the same dynamic key.
|
||||
snapshot = await publishModelRuntimeSnapshot(
|
||||
input,
|
||||
context.owners,
|
||||
context.agentBuildCompletions,
|
||||
context.getBuildTimeoutMs(),
|
||||
undefined,
|
||||
provenance,
|
||||
options.catalogMode,
|
||||
reusablePluginGeneration,
|
||||
);
|
||||
} else if (existing) {
|
||||
if (existing && !staleDynamicOwner) {
|
||||
snapshot = await context.prepareSnapshot(input);
|
||||
} else {
|
||||
// Fresh keys publish a first generation; stale dynamic owners publish a distinct
|
||||
// replacement owner because existing leases retain their immutable snapshot, so
|
||||
// their release cannot delete the generation admitted for new work at this key.
|
||||
snapshot = await publishModelRuntimeSnapshot(
|
||||
input,
|
||||
context.owners,
|
||||
|
||||
@@ -1,6 +1,21 @@
|
||||
import fsSync from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const evidenceWarnSpy = vi.hoisted(() => vi.fn());
|
||||
vi.mock("../logging/subsystem.js", async () => {
|
||||
const actual =
|
||||
await vi.importActual<typeof import("../logging/subsystem.js")>("../logging/subsystem.js");
|
||||
return {
|
||||
...actual,
|
||||
createSubsystemLogger: (subsystem: string) => {
|
||||
const logger = actual.createSubsystemLogger(subsystem);
|
||||
return subsystem === "gateway/placement-session-evidence"
|
||||
? { ...logger, warn: evidenceWarnSpy }
|
||||
: logger;
|
||||
},
|
||||
};
|
||||
});
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { resetConfigRuntimeState, setRuntimeConfigSnapshot } from "../config/config.js";
|
||||
import * as sessionAccessor from "../config/sessions/session-accessor.js";
|
||||
@@ -27,6 +42,7 @@ afterEach(() => {
|
||||
resetConfigRuntimeState();
|
||||
resolveTargetsReadOnlySpy.mockClear();
|
||||
readIdentityEvidenceBatchSpy.mockClear();
|
||||
evidenceWarnSpy.mockClear();
|
||||
});
|
||||
|
||||
function localPlacement(
|
||||
@@ -209,6 +225,23 @@ describe("worker placement session evidence", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("warns instead of silently swallowing resolver pipeline failures", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-placement-session-pipeline-failure-");
|
||||
await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => {
|
||||
resolveTargetsReadOnlySpy.mockImplementationOnce(() => {
|
||||
throw new Error("evidence pipeline exploded");
|
||||
});
|
||||
const placement = localPlacement("session-pipeline-failure", "agent:main:pipeline-failure");
|
||||
|
||||
await expect(resolvePlacementEvidence(placement)).resolves.toBe("unknown");
|
||||
expect(evidenceWarnSpy).toHaveBeenCalledOnce();
|
||||
expect(evidenceWarnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("session evidence resolution failed"),
|
||||
{ error: expect.objectContaining({ message: "evidence pipeline exploded" }) },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("prepares targets once and reads only exact session rows for a placement batch", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-placement-session-evidence-batch-");
|
||||
await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { getRuntimeConfig } from "../config/config.js";
|
||||
import type { SessionStoreTargetsReadCache } from "../config/sessions/targets-read-availability.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import {
|
||||
isIncognitoSessionKey,
|
||||
normalizeAgentId,
|
||||
@@ -15,6 +16,8 @@ import type {
|
||||
PlacementSessionEvidenceResolver,
|
||||
} from "./worker-environments/placement-session-retirement.js";
|
||||
|
||||
const log = createSubsystemLogger("gateway/placement-session-evidence");
|
||||
|
||||
const loadPlacementSessionEvidenceRuntime = createLazyRuntimeModule(async () => {
|
||||
const [sessionTargetsReadAvailability, sessionAccessor] = await Promise.all([
|
||||
import("../config/sessions/targets-read-availability.js"),
|
||||
@@ -138,7 +141,12 @@ export async function createWorkerPlacementSessionEvidenceResolver(
|
||||
}
|
||||
}
|
||||
return async (placement) => evidenceByPlacement.get(placement) ?? "unknown";
|
||||
} catch {
|
||||
} catch (error) {
|
||||
// "unknown" keeps retirement fail-open, but a silent catch would hide a broken
|
||||
// evidence pipeline (bad config, store corruption) behind indefinite retention.
|
||||
log.warn("worker placement session evidence resolution failed; treating all as unknown", {
|
||||
error,
|
||||
});
|
||||
return async () => "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user