mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-23 02:45:38 -06:00
fix(cloud-workers): preserve reclaim teardown failures (#121520)
* fix(cloud-workers): preserve reclaim teardown failures * fix(test): remove unused session hint import
This commit is contained in:
committed by
GitHub
parent
c034670b72
commit
ca4dc1bc9e
@@ -65,9 +65,16 @@ describe("worker placement dispatch reclaim", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("reconciles the workspace before destroying and reclaiming an active worker", async () => {
|
||||
const harness = createHarness(placementStore);
|
||||
await harness.service.dispatch(REQUEST);
|
||||
it("reclaims an unchanged active placement through the fenced teardown lifecycle", async () => {
|
||||
const harness = createHarness(placementStore, {
|
||||
reconcileChanged: false,
|
||||
reconcileCommitsManifest: false,
|
||||
});
|
||||
await expect(harness.service.dispatch(REQUEST)).resolves.toMatchObject({
|
||||
state: "active",
|
||||
turnClaim: null,
|
||||
workspaceBaseManifestRef: MANIFEST_REF,
|
||||
});
|
||||
|
||||
await expect(
|
||||
harness.service.reclaim({
|
||||
@@ -77,9 +84,11 @@ describe("worker placement dispatch reclaim", () => {
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
state: "reclaimed",
|
||||
workspaceBaseManifestRef: harness.reconciledManifestRef,
|
||||
turnClaim: null,
|
||||
workspaceBaseManifestRef: MANIFEST_REF,
|
||||
});
|
||||
|
||||
expect(placementStore.listPendingWorkspaceResults()).toEqual([]);
|
||||
expect(harness.log.slice(-11)).toEqual([
|
||||
"tunnel:attached",
|
||||
"workspace:quiesce",
|
||||
|
||||
@@ -461,6 +461,30 @@ describe("worker placement dispatch", () => {
|
||||
expect(harness.log).toContain("workspace:resume");
|
||||
});
|
||||
|
||||
it("preserves a provider destroy failure after teardown owns the stopped tunnel", async () => {
|
||||
const harness = createHarness(placementStore, {
|
||||
destroyFails: true,
|
||||
destroyFailureState: "destroying",
|
||||
resumeFails: true,
|
||||
});
|
||||
const active = await harness.service.dispatch(REQUEST);
|
||||
|
||||
await expect(harness.service.reclaim(REQUEST)).rejects.toThrow("destroy pending");
|
||||
|
||||
expect(harness.environments.get(active.environmentId)).toMatchObject({
|
||||
state: "destroying",
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
});
|
||||
expect(harness.placements.current()).toMatchObject({
|
||||
state: "active",
|
||||
turnClaim: { owner: "worker" },
|
||||
});
|
||||
expect(placementStore.listPendingWorkspaceResults()).toMatchObject([
|
||||
{ workspaceAcceptedAtMs: expect.any(Number) },
|
||||
]);
|
||||
expect(harness.log).not.toContain("workspace:resume");
|
||||
});
|
||||
|
||||
it.each<DispatchStage>([
|
||||
"barrier",
|
||||
"workspace",
|
||||
|
||||
@@ -90,6 +90,19 @@ function requireProvisionedEnvironment(
|
||||
};
|
||||
}
|
||||
|
||||
function isExactAttachedEnvironment(
|
||||
environment: ReturnType<WorkerDispatchEnvironmentService["get"]>,
|
||||
placement: WorkerActiveDispatchPlacement,
|
||||
): boolean {
|
||||
return (
|
||||
environment?.environmentId === placement.environmentId &&
|
||||
environment.state === "attached" &&
|
||||
environment.ownerEpoch === placement.activeOwnerEpoch &&
|
||||
environment.attachedSessionIds.length === 1 &&
|
||||
environment.attachedSessionIds[0] === placement.sessionId
|
||||
);
|
||||
}
|
||||
|
||||
export function createWorkerPlacementDispatchService(options: WorkerPlacementDispatchOptions) {
|
||||
const { environments, placements } = options;
|
||||
const failure = createPlacementFailureActions({ environments, placements });
|
||||
@@ -250,13 +263,7 @@ export function createWorkerPlacementDispatchService(options: WorkerPlacementDis
|
||||
);
|
||||
}
|
||||
const environment = environments.get(current.environmentId);
|
||||
if (
|
||||
!environment ||
|
||||
environment.state !== "attached" ||
|
||||
environment.ownerEpoch !== current.activeOwnerEpoch ||
|
||||
environment.attachedSessionIds.length !== 1 ||
|
||||
environment.attachedSessionIds[0] !== current.sessionId
|
||||
) {
|
||||
if (!isExactAttachedEnvironment(environment, current)) {
|
||||
throw new Error("Active cloud worker does not match its session placement");
|
||||
}
|
||||
const journalOwner = {
|
||||
@@ -432,7 +439,10 @@ export function createWorkerPlacementDispatchService(options: WorkerPlacementDis
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
if (!destroyed) {
|
||||
if (
|
||||
!destroyed &&
|
||||
isExactAttachedEnvironment(environments.get(current.environmentId), current)
|
||||
) {
|
||||
await quiescence.resume();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user