mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 05:16:23 -06:00
feat(cloud-workers): session placement, dispatch, and worker turn routing (#106332)
* feat(gateway-protocol): add session placement schema Closed state discriminator for session execution placement (local/requested/provisioning/syncing/starting/active/draining/reconciling/reclaimed/failed), sessions.dispatch params, and worker-admission transcript/live cursor extensions. Swift protocol models mirror the schema. * feat(state): add worker session placement table worker_session_placements rows carry placement state, transition generation, worker ownership metadata, ACK cursors, and the turn claim columns used for atomic admission. * feat(cloud-workers): add durable placement state machine store SQLite-backed placement store split by concern: state table (placement-state), discriminated record types + shape invariants (placement-record), row codec + CAS transition values (placement-row-codec), atomic turn-claim admission/release/waiters (placement-turn-claims), and lifecycle CAS transitions (placement-store). * feat(cloud-workers): sync workspaces and attach sessions to worker environments Environment service session attachment + turn credentials, tunnel workspace commands over a dedicated SSH runner, and git/plain workspace sync into $HOME/.openclaw-worker/workspaces with an immutable manifest. Symlink escapes are rejected locally before transfer (macOS openrsync stat-fails them opaquely) and again by the remote manifest guard. * feat(worker): run one-shot embedded turns from launch descriptors Worker runtime executes a single embedded turn from a stdin launch descriptor and reports completed/failed/fenced on stdout for the gateway launcher. Terminal lifecycle live events are deferred past the final transcript flush; transcript projection helpers are shared via transcript-message instead of duplicated in the runtime. * feat(cloud-workers): dispatch placements and route worker turns Dispatch service drives local->requested->provisioning->syncing->starting->active with failure teardown (placement-dispatch-failure) and restart/runtime recovery incl. lost-worker reclaim (placement-dispatch-recovery). Worker turn launcher claims the placement turn atomically, builds a windowed launch descriptor (worker-turn-payload), runs the remote one-shot worker, and reconciles the committed transcript; agent runners route turns through the session placement admission provider. * feat(gateway): expose session placement RPCs and startup reconciliation sessions.dispatch RPC with lifecycle admission barriers, operator-facing placement projection on session listings, placement-aware session reset guard, and startup/interval reconciliation wiring for worker placements.
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
06b27b9e1d
commit
e98c7dfbcb
@@ -166,6 +166,7 @@ type WorkerLiveRuntime = {
|
||||
handleSessionEvent: (event: AgentSessionEvent) => void;
|
||||
enqueueRunFailure: (failure: { aborted: boolean; error: Error }) => void;
|
||||
flush: () => Promise<void>;
|
||||
emitTerminal: () => Promise<void>;
|
||||
};
|
||||
|
||||
export function createWorkerLiveRuntime(client: WorkerLiveClient): WorkerLiveRuntime {
|
||||
@@ -219,6 +220,9 @@ export function createWorkerLiveRuntime(client: WorkerLiveClient): WorkerLiveRun
|
||||
};
|
||||
const startedAt = Date.now();
|
||||
let lifecycleFinished = false;
|
||||
// Terminal lifecycle events are deferred past the final transcript flush so the
|
||||
// gateway never sees an end/error before the authoritative transcript commit.
|
||||
let terminalLiveEvent: WorkerLiveEvent | undefined;
|
||||
let streamedText = "";
|
||||
let streamedThinking = "";
|
||||
const handleSessionEvent = (event: AgentSessionEvent) => {
|
||||
@@ -310,17 +314,18 @@ export function createWorkerLiveRuntime(client: WorkerLiveClient): WorkerLiveRun
|
||||
.toReversed()
|
||||
.find((message): message is AssistantMessage => message.role === "assistant");
|
||||
if (lastAssistant?.stopReason === "error") {
|
||||
enqueueLive({
|
||||
terminalLiveEvent = {
|
||||
kind: "lifecycle",
|
||||
payload: {
|
||||
phase: "error",
|
||||
startedAt,
|
||||
endedAt: Date.now(),
|
||||
error: lastAssistant.errorMessage ?? "Worker inference failed.",
|
||||
fallbackExhaustedFailure: true,
|
||||
},
|
||||
});
|
||||
};
|
||||
} else if (lastAssistant?.stopReason === "aborted") {
|
||||
enqueueLive({
|
||||
terminalLiveEvent = {
|
||||
kind: "lifecycle",
|
||||
payload: {
|
||||
phase: "end",
|
||||
@@ -329,12 +334,12 @@ export function createWorkerLiveRuntime(client: WorkerLiveClient): WorkerLiveRun
|
||||
stopReason: "aborted",
|
||||
aborted: true,
|
||||
},
|
||||
});
|
||||
};
|
||||
} else {
|
||||
enqueueLive({
|
||||
terminalLiveEvent = {
|
||||
kind: "lifecycle",
|
||||
payload: { phase: "end", startedAt, endedAt: Date.now() },
|
||||
});
|
||||
};
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -343,7 +348,7 @@ export function createWorkerLiveRuntime(client: WorkerLiveClient): WorkerLiveRun
|
||||
return;
|
||||
}
|
||||
if (failure.aborted) {
|
||||
enqueueLive({
|
||||
terminalLiveEvent = {
|
||||
kind: "lifecycle",
|
||||
payload: {
|
||||
phase: "end",
|
||||
@@ -352,18 +357,27 @@ export function createWorkerLiveRuntime(client: WorkerLiveClient): WorkerLiveRun
|
||||
stopReason: "aborted",
|
||||
aborted: true,
|
||||
},
|
||||
});
|
||||
};
|
||||
} else {
|
||||
enqueueLive({
|
||||
terminalLiveEvent = {
|
||||
kind: "lifecycle",
|
||||
payload: {
|
||||
phase: "error",
|
||||
startedAt,
|
||||
endedAt: Date.now(),
|
||||
error: failure.error.message,
|
||||
fallbackExhaustedFailure: true,
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
};
|
||||
return { handleSessionEvent, enqueueRunFailure, flush };
|
||||
// Emits directly (not via the degradable preview queue): the terminal event drives
|
||||
// gateway turn settlement and must survive a degraded live stream.
|
||||
const emitTerminal = async () => {
|
||||
if (!terminalLiveEvent) {
|
||||
return;
|
||||
}
|
||||
await client.emit(boundLiveEvent(terminalLiveEvent));
|
||||
};
|
||||
return { handleSessionEvent, enqueueRunFailure, flush, emitTerminal };
|
||||
}
|
||||
|
||||
@@ -4,126 +4,14 @@ import type { WorkerInferenceContext } from "../../packages/gateway-protocol/src
|
||||
import { WORKER_INFERENCE_MAX_CONTEXT_MESSAGES } from "../../packages/gateway-protocol/src/schema/worker-inference.js";
|
||||
import type { AgentMessage } from "../agents/runtime/index.js";
|
||||
import type { AgentSessionWriteLockRunner } from "../agents/sessions/agent-session.js";
|
||||
import type { AssistantMessage, Context, Message } from "../llm/types.js";
|
||||
import { isWorkerTranscriptMessageFrameSafe } from "./transcript-message.js";
|
||||
|
||||
function cloneTextContent(part: { type: "text"; text: string; textSignature?: string }) {
|
||||
return {
|
||||
type: "text" as const,
|
||||
text: part.text,
|
||||
...(part.textSignature ? { textSignature: part.textSignature } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function cloneImageContent(part: { type: "image"; data: string; mimeType: string }) {
|
||||
return { type: "image" as const, data: part.data, mimeType: part.mimeType };
|
||||
}
|
||||
|
||||
function cloneUsage(message: AssistantMessage): WorkerTranscriptMessage & { role: "assistant" } {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: message.content.map((part) => {
|
||||
if (part.type === "text") {
|
||||
return cloneTextContent(part);
|
||||
}
|
||||
if (part.type === "thinking") {
|
||||
return {
|
||||
type: "thinking" as const,
|
||||
thinking: part.thinking,
|
||||
...(part.thinkingSignature ? { thinkingSignature: part.thinkingSignature } : {}),
|
||||
...(part.redacted === undefined ? {} : { redacted: part.redacted }),
|
||||
};
|
||||
}
|
||||
return {
|
||||
type: "toolCall" as const,
|
||||
id: part.id,
|
||||
name: part.name,
|
||||
arguments: structuredClone(part.arguments),
|
||||
...(part.thoughtSignature ? { thoughtSignature: part.thoughtSignature } : {}),
|
||||
...(part.executionMode ? { executionMode: part.executionMode } : {}),
|
||||
};
|
||||
}),
|
||||
api: message.api,
|
||||
provider: message.provider,
|
||||
model: message.model,
|
||||
...(message.responseModel ? { responseModel: message.responseModel } : {}),
|
||||
...(message.responseId ? { responseId: message.responseId } : {}),
|
||||
...(message.diagnostics
|
||||
? {
|
||||
diagnostics: message.diagnostics.map((diagnostic) => ({
|
||||
type: diagnostic.type,
|
||||
timestamp: diagnostic.timestamp,
|
||||
...(diagnostic.error
|
||||
? {
|
||||
error: {
|
||||
...(diagnostic.error.name ? { name: diagnostic.error.name } : {}),
|
||||
message: diagnostic.error.message,
|
||||
...(diagnostic.error.stack ? { stack: diagnostic.error.stack } : {}),
|
||||
...(diagnostic.error.code === undefined ? {} : { code: diagnostic.error.code }),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(diagnostic.details ? { details: structuredClone(diagnostic.details) } : {}),
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
usage: {
|
||||
input: message.usage.input,
|
||||
output: message.usage.output,
|
||||
cacheRead: message.usage.cacheRead,
|
||||
cacheWrite: message.usage.cacheWrite,
|
||||
...(message.usage.contextUsage
|
||||
? { contextUsage: structuredClone(message.usage.contextUsage) }
|
||||
: {}),
|
||||
totalTokens: message.usage.totalTokens,
|
||||
cost: {
|
||||
input: message.usage.cost.input,
|
||||
output: message.usage.cost.output,
|
||||
cacheRead: message.usage.cost.cacheRead,
|
||||
cacheWrite: message.usage.cost.cacheWrite,
|
||||
total: message.usage.cost.total,
|
||||
...(message.usage.cost.totalOrigin ? { totalOrigin: message.usage.cost.totalOrigin } : {}),
|
||||
},
|
||||
},
|
||||
stopReason: message.stopReason,
|
||||
...(message.errorMessage ? { errorMessage: message.errorMessage } : {}),
|
||||
...(message.errorCode ? { errorCode: message.errorCode } : {}),
|
||||
...(message.errorType ? { errorType: message.errorType } : {}),
|
||||
...(message.errorBody ? { errorBody: message.errorBody } : {}),
|
||||
timestamp: message.timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
export function toWorkerTranscriptMessage(
|
||||
message: AgentMessage,
|
||||
): WorkerTranscriptMessage | undefined {
|
||||
if (message.role === "user") {
|
||||
const content =
|
||||
typeof message.content === "string"
|
||||
? [{ type: "text" as const, text: message.content }]
|
||||
: message.content.map((part) =>
|
||||
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
|
||||
);
|
||||
return { role: "user", content, timestamp: message.timestamp };
|
||||
}
|
||||
if (message.role === "assistant") {
|
||||
return cloneUsage(message);
|
||||
}
|
||||
if (message.role === "toolResult") {
|
||||
return {
|
||||
role: "toolResult",
|
||||
toolCallId: message.toolCallId,
|
||||
toolName: message.toolName,
|
||||
content: message.content.map((part) =>
|
||||
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
|
||||
),
|
||||
...(message.details === undefined ? {} : { details: structuredClone(message.details) }),
|
||||
isError: message.isError,
|
||||
timestamp: message.timestamp,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
import type { Context, Message } from "../llm/types.js";
|
||||
import {
|
||||
cloneImageContent,
|
||||
cloneTextContent,
|
||||
cloneUsage,
|
||||
isWorkerTranscriptMessageFrameSafe,
|
||||
toWorkerTranscriptMessage,
|
||||
} from "./transcript-message.js";
|
||||
|
||||
export function toAgentMessage(message: WorkerTranscriptMessage): Message {
|
||||
if (message.role === "user") {
|
||||
|
||||
@@ -26,8 +26,8 @@ import {
|
||||
createWorkerTranscriptRuntime,
|
||||
toAgentMessage,
|
||||
toWorkerInferenceContext,
|
||||
toWorkerTranscriptMessage,
|
||||
} from "./embedded-agent-transcript.runtime.js";
|
||||
import { toWorkerTranscriptMessage } from "./transcript-message.js";
|
||||
|
||||
const LOCAL_WORKER_TOOL_NAMES = [
|
||||
"read",
|
||||
@@ -75,6 +75,7 @@ type RunWorkerEmbeddedTurnParams = {
|
||||
transcript: WorkerEmbeddedTranscriptClient;
|
||||
live: WorkerEmbeddedLiveClient;
|
||||
initialMessages?: WorkerTranscriptMessage[];
|
||||
suppressPromptTranscript?: boolean;
|
||||
systemPrompt?: string;
|
||||
inferenceOptions?: WorkerInferenceOptions;
|
||||
signal?: AbortSignal;
|
||||
@@ -110,7 +111,7 @@ export async function runWorkerEmbeddedTurn(
|
||||
noPromptTemplates: true,
|
||||
noThemes: true,
|
||||
noContextFiles: true,
|
||||
...(params.systemPrompt === undefined ? {} : { systemPrompt: params.systemPrompt }),
|
||||
...(params.systemPrompt === undefined ? {} : { appendSystemPrompt: [params.systemPrompt] }),
|
||||
agentsFilesOverride: () => ({ agentsFiles: contextFiles }),
|
||||
});
|
||||
await resourceLoader.reload();
|
||||
@@ -122,6 +123,7 @@ export async function runWorkerEmbeddedTurn(
|
||||
|
||||
const transcriptRuntime = createWorkerTranscriptRuntime(params.transcript);
|
||||
const sessionManager = guardSessionManager(baseSessionManager, {
|
||||
suppressNextUserMessagePersistence: params.suppressPromptTranscript,
|
||||
onMessagePersisted: transcriptRuntime.onMessagePersisted,
|
||||
});
|
||||
|
||||
@@ -221,13 +223,14 @@ export async function runWorkerEmbeddedTurn(
|
||||
|
||||
let finalTranscriptFailure: Error | undefined;
|
||||
try {
|
||||
if (!params.signal?.aborted) {
|
||||
try {
|
||||
await transcriptRuntime.withSessionWriteLock(() => undefined);
|
||||
} catch (error) {
|
||||
finalTranscriptFailure = toError(error, "Worker transcript flush failed.");
|
||||
}
|
||||
await liveRuntime.flush();
|
||||
try {
|
||||
await transcriptRuntime.withSessionWriteLock(() => undefined);
|
||||
} catch (error) {
|
||||
finalTranscriptFailure = toError(error, "Worker transcript flush failed.");
|
||||
}
|
||||
await liveRuntime.flush();
|
||||
if (finalTranscriptFailure === undefined) {
|
||||
await liveRuntime.emitTerminal();
|
||||
}
|
||||
} finally {
|
||||
params.signal?.removeEventListener("abort", abortTurn);
|
||||
|
||||
@@ -28,6 +28,7 @@ function launchDescriptor(): WorkerLaunchDescriptor {
|
||||
runId: "run-1",
|
||||
turnId: "turn-1",
|
||||
prompt: "Inspect the workspace.",
|
||||
suppressPromptTranscript: false,
|
||||
workspaceDir: "/tmp/openclaw-worker/workspace",
|
||||
modelRef: { provider: "provider-1", model: "model-1" },
|
||||
inferenceOptions: { reasoning: "medium", maxTokens: 512 },
|
||||
@@ -52,7 +53,7 @@ describe("worker launch descriptor", () => {
|
||||
expect(buildWorkerConnectParams(descriptor)).toMatchObject({
|
||||
role: "worker",
|
||||
client: { id: "openclaw-worker", mode: "worker", version: "2026.7.12" },
|
||||
admission: descriptor.admission,
|
||||
admission: { ...descriptor.admission, runId: descriptor.assignment.runId },
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ type WorkerLaunchAssignment = {
|
||||
runId: string;
|
||||
turnId: string;
|
||||
prompt: string;
|
||||
suppressPromptTranscript: boolean;
|
||||
workspaceDir: string;
|
||||
modelRef: WorkerInferenceModelRef;
|
||||
inferenceOptions: WorkerInferenceOptions;
|
||||
@@ -46,7 +47,7 @@ type WorkerLaunchAssignment = {
|
||||
};
|
||||
};
|
||||
|
||||
type WorkerLaunchAdmission = WorkerConnectParams["admission"] & {
|
||||
type WorkerLaunchAdmission = Omit<WorkerConnectParams["admission"], "runId"> & {
|
||||
sessionId: string;
|
||||
};
|
||||
|
||||
@@ -94,6 +95,7 @@ function parseAssignment(value: unknown): WorkerLaunchAssignment | undefined {
|
||||
"runId",
|
||||
"turnId",
|
||||
"prompt",
|
||||
"suppressPromptTranscript",
|
||||
"workspaceDir",
|
||||
"modelRef",
|
||||
"inferenceOptions",
|
||||
@@ -110,6 +112,7 @@ function parseAssignment(value: unknown): WorkerLaunchAssignment | undefined {
|
||||
!isIdentifier(value.runId) ||
|
||||
!isIdentifier(value.turnId) ||
|
||||
typeof value.prompt !== "string" ||
|
||||
typeof value.suppressPromptTranscript !== "boolean" ||
|
||||
!isIdentifier(value.workspaceDir) ||
|
||||
!path.isAbsolute(value.workspaceDir) ||
|
||||
(value.systemPrompt !== undefined && typeof value.systemPrompt !== "string") ||
|
||||
@@ -146,7 +149,7 @@ function parseAssignment(value: unknown): WorkerLaunchAssignment | undefined {
|
||||
}
|
||||
|
||||
export function buildWorkerConnectParams(
|
||||
descriptor: Pick<WorkerLaunchDescriptor, "admission">,
|
||||
descriptor: Pick<WorkerLaunchDescriptor, "admission" | "assignment">,
|
||||
): WorkerConnectParams {
|
||||
return {
|
||||
minProtocol: PROTOCOL_VERSION,
|
||||
@@ -158,7 +161,10 @@ export function buildWorkerConnectParams(
|
||||
mode: GATEWAY_CLIENT_MODES.WORKER,
|
||||
},
|
||||
role: "worker",
|
||||
admission: descriptor.admission,
|
||||
admission: {
|
||||
...descriptor.admission,
|
||||
runId: descriptor.assignment.runId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -6,9 +6,131 @@ import {
|
||||
WORKER_PROTOCOL_MAX_IDENTIFIER_LENGTH,
|
||||
WORKER_PROTOCOL_MAX_PAYLOAD_BYTES,
|
||||
} from "../../packages/gateway-protocol/src/schema/worker-admission.js";
|
||||
import type { AgentMessage } from "../agents/runtime/index.js";
|
||||
import type { AssistantMessage } from "../llm/types.js";
|
||||
|
||||
const SIZE_FRAME_ID = "00000000-0000-4000-8000-000000000000";
|
||||
|
||||
export function cloneTextContent(part: { type: "text"; text: string; textSignature?: string }) {
|
||||
return {
|
||||
type: "text" as const,
|
||||
text: part.text,
|
||||
...(part.textSignature ? { textSignature: part.textSignature } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function cloneImageContent(part: { type: "image"; data: string; mimeType: string }) {
|
||||
return { type: "image" as const, data: part.data, mimeType: part.mimeType };
|
||||
}
|
||||
|
||||
export function cloneUsage(
|
||||
message: AssistantMessage,
|
||||
): WorkerTranscriptMessage & { role: "assistant" } {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: message.content.map((part) => {
|
||||
if (part.type === "text") {
|
||||
return cloneTextContent(part);
|
||||
}
|
||||
if (part.type === "thinking") {
|
||||
return {
|
||||
type: "thinking" as const,
|
||||
thinking: part.thinking,
|
||||
...(part.thinkingSignature ? { thinkingSignature: part.thinkingSignature } : {}),
|
||||
...(part.redacted === undefined ? {} : { redacted: part.redacted }),
|
||||
};
|
||||
}
|
||||
return {
|
||||
type: "toolCall" as const,
|
||||
id: part.id,
|
||||
name: part.name,
|
||||
arguments: structuredClone(part.arguments),
|
||||
...(part.thoughtSignature ? { thoughtSignature: part.thoughtSignature } : {}),
|
||||
...(part.executionMode ? { executionMode: part.executionMode } : {}),
|
||||
};
|
||||
}),
|
||||
api: message.api,
|
||||
provider: message.provider,
|
||||
model: message.model,
|
||||
...(message.responseModel ? { responseModel: message.responseModel } : {}),
|
||||
...(message.responseId ? { responseId: message.responseId } : {}),
|
||||
...(message.diagnostics
|
||||
? {
|
||||
diagnostics: message.diagnostics.map((diagnostic) => ({
|
||||
type: diagnostic.type,
|
||||
timestamp: diagnostic.timestamp,
|
||||
...(diagnostic.error
|
||||
? {
|
||||
error: {
|
||||
...(diagnostic.error.name ? { name: diagnostic.error.name } : {}),
|
||||
message: diagnostic.error.message,
|
||||
...(diagnostic.error.stack ? { stack: diagnostic.error.stack } : {}),
|
||||
...(diagnostic.error.code === undefined ? {} : { code: diagnostic.error.code }),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(diagnostic.details ? { details: structuredClone(diagnostic.details) } : {}),
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
usage: {
|
||||
input: message.usage.input,
|
||||
output: message.usage.output,
|
||||
cacheRead: message.usage.cacheRead,
|
||||
cacheWrite: message.usage.cacheWrite,
|
||||
...(message.usage.contextUsage
|
||||
? { contextUsage: structuredClone(message.usage.contextUsage) }
|
||||
: {}),
|
||||
totalTokens: message.usage.totalTokens,
|
||||
cost: {
|
||||
input: message.usage.cost.input,
|
||||
output: message.usage.cost.output,
|
||||
cacheRead: message.usage.cost.cacheRead,
|
||||
cacheWrite: message.usage.cost.cacheWrite,
|
||||
total: message.usage.cost.total,
|
||||
...(message.usage.cost.totalOrigin ? { totalOrigin: message.usage.cost.totalOrigin } : {}),
|
||||
},
|
||||
},
|
||||
stopReason: message.stopReason,
|
||||
...(message.errorMessage ? { errorMessage: message.errorMessage } : {}),
|
||||
...(message.errorCode ? { errorCode: message.errorCode } : {}),
|
||||
...(message.errorType ? { errorType: message.errorType } : {}),
|
||||
...(message.errorBody ? { errorBody: message.errorBody } : {}),
|
||||
timestamp: message.timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
export function toWorkerTranscriptMessage(
|
||||
message: AgentMessage,
|
||||
): WorkerTranscriptMessage | undefined {
|
||||
if (message.role === "user") {
|
||||
const content =
|
||||
typeof message.content === "string"
|
||||
? [{ type: "text" as const, text: message.content }]
|
||||
: message.content.map((part) =>
|
||||
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
|
||||
);
|
||||
return { role: "user", content, timestamp: message.timestamp };
|
||||
}
|
||||
if (message.role === "assistant") {
|
||||
return cloneUsage(message);
|
||||
}
|
||||
if (message.role === "toolResult") {
|
||||
return {
|
||||
role: "toolResult",
|
||||
toolCallId: message.toolCallId,
|
||||
toolName: message.toolName,
|
||||
content: message.content.map((part) =>
|
||||
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
|
||||
),
|
||||
...(message.details === undefined ? {} : { details: structuredClone(message.details) }),
|
||||
isError: message.isError,
|
||||
timestamp: message.timestamp,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function isWorkerTranscriptMessageFrameSafe(message: WorkerTranscriptMessage): boolean {
|
||||
const frame: WorkerTranscriptCommitRequestFrame = {
|
||||
type: "req",
|
||||
|
||||
@@ -320,6 +320,7 @@ class ComposedGatewayHarness {
|
||||
baseLeafId?: string | null;
|
||||
initialSeq?: number;
|
||||
initialAckedSeq?: number;
|
||||
runId?: string;
|
||||
} = {},
|
||||
): WorkerClients {
|
||||
const epoch = params.epoch ?? this.epoch;
|
||||
@@ -336,12 +337,13 @@ class ComposedGatewayHarness {
|
||||
handshake: HANDSHAKE,
|
||||
},
|
||||
assignment: {
|
||||
runId: RUN_ID,
|
||||
runId: params.runId ?? RUN_ID,
|
||||
turnId: "fault-turn",
|
||||
prompt: "fault injection",
|
||||
workspaceDir: this.root,
|
||||
modelRef: MODEL_REF,
|
||||
inferenceOptions: {},
|
||||
suppressPromptTranscript: false,
|
||||
initialMessages: [],
|
||||
transcript: { baseLeafId: params.baseLeafId ?? null, nextSeq: params.initialSeq ?? 1 },
|
||||
liveEvents: {
|
||||
@@ -857,10 +859,13 @@ describe("cloud worker milestone 2 fault injection", () => {
|
||||
await oldInferenceRejected;
|
||||
|
||||
harness.providerPlan = { kind: "immediate", text: "new owner reply" };
|
||||
// Milestone-3 admission binds the worker to a single run; the fresh owner
|
||||
// must be admitted for the run it executes.
|
||||
const fresh = harness.createClients({
|
||||
admissionProof: REPLACEMENT_CREDENTIAL,
|
||||
epoch: newEpoch,
|
||||
baseLeafId: oldCommit.newLeafId,
|
||||
runId: "fresh-run",
|
||||
});
|
||||
clients.push(fresh);
|
||||
await fresh.connection.start();
|
||||
|
||||
@@ -128,6 +128,7 @@ class FakeWorkerGateway {
|
||||
readonly acceptedTranscriptRequests: WorkerTranscriptCommitParams[] = [];
|
||||
readonly liveEventRequests: WorkerLiveEventParams[] = [];
|
||||
readonly inferenceRequests: WorkerInferenceStartParams[] = [];
|
||||
readonly applicationOrder: string[] = [];
|
||||
|
||||
constructor(private readonly options: FakeGatewayOptions = {}) {
|
||||
this.httpServer = createServer();
|
||||
@@ -315,6 +316,7 @@ class FakeWorkerGateway {
|
||||
return;
|
||||
}
|
||||
this.acceptedTranscriptRequests.push(structuredClone(frame.params));
|
||||
this.applicationOrder.push(`transcript:${frame.params.seq}`);
|
||||
this.send(socket, {
|
||||
type: "res",
|
||||
id: frame.id,
|
||||
@@ -331,6 +333,11 @@ class FakeWorkerGateway {
|
||||
private handleLiveEvent(socket: WebSocket, frame: WorkerLiveEventRequestFrame): void {
|
||||
this.methods.push(frame.method);
|
||||
this.liveEventRequests.push(structuredClone(frame.params));
|
||||
this.applicationOrder.push(
|
||||
frame.params.event.kind === "lifecycle"
|
||||
? `live:lifecycle:${frame.params.event.payload.phase}`
|
||||
: `live:${frame.params.event.kind}`,
|
||||
);
|
||||
if (this.options.silenceFirstLiveEvent && !this.droppedLiveEvent) {
|
||||
this.droppedLiveEvent = true;
|
||||
return;
|
||||
@@ -706,6 +713,7 @@ function descriptor(socketPath: string, workspaceDir: string): WorkerLaunchDescr
|
||||
runId: RUN_ID,
|
||||
turnId: "worker-turn",
|
||||
prompt: "Complete the worker turn.",
|
||||
suppressPromptTranscript: false,
|
||||
workspaceDir,
|
||||
modelRef: MODEL_REF,
|
||||
inferenceOptions: { reasoning: "off" },
|
||||
@@ -754,6 +762,14 @@ describe("worker runtime", () => {
|
||||
expect(gateway.inferenceRequests[0]?.context.systemPrompt).toContain("worker-bootstrap-marker");
|
||||
const toolNames = gateway.inferenceRequests[0]?.context.tools?.map((tool) => tool.name) ?? [];
|
||||
expect(toolNames).toHaveLength(6);
|
||||
const terminalIndex = gateway.applicationOrder.findIndex(
|
||||
(entry) => entry === "live:lifecycle:end",
|
||||
);
|
||||
const finalTranscriptIndex = gateway.applicationOrder.findLastIndex((entry) =>
|
||||
entry.startsWith("transcript:"),
|
||||
);
|
||||
expect(finalTranscriptIndex).toBeGreaterThanOrEqual(0);
|
||||
expect(terminalIndex).toBeGreaterThan(finalTranscriptIndex);
|
||||
expect(toolNames).toEqual(
|
||||
expect.arrayContaining(["read", "write", "edit", "apply_patch", "exec", "process"]),
|
||||
);
|
||||
@@ -806,7 +822,7 @@ describe("worker runtime", () => {
|
||||
gateway.liveEventRequests.some(
|
||||
(request) => request.event.kind === "lifecycle" && request.event.payload.phase === "error",
|
||||
),
|
||||
).toBe(true);
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("renumbers live events after a gateway cursor reset without aborting the run", async () => {
|
||||
@@ -824,10 +840,10 @@ describe("worker runtime", () => {
|
||||
expect(gateway.liveEventRequests[1]?.event).toEqual(gateway.liveEventRequests[0]?.event);
|
||||
});
|
||||
|
||||
it("degrades hard live-event failures without affecting inference or transcript commits", async () => {
|
||||
it("requires authoritative terminal delivery after degrading preview live events", async () => {
|
||||
const { gateway, launch } = await setup({ liveFailure: "capacity-exceeded" });
|
||||
|
||||
await expect(runWorkerDescriptor(launch)).resolves.toMatchObject({ status: "completed" });
|
||||
await expect(runWorkerDescriptor(launch)).rejects.toThrow("worker live event rejected");
|
||||
|
||||
expect(gateway.inferenceRequests).toHaveLength(1);
|
||||
expect(
|
||||
@@ -835,7 +851,11 @@ describe("worker runtime", () => {
|
||||
.flatMap((request) => request.messages)
|
||||
.map((message) => message.role),
|
||||
).toEqual(["user", "assistant"]);
|
||||
expect(gateway.liveEventRequests).toHaveLength(1);
|
||||
expect(gateway.liveEventRequests.length).toBeGreaterThanOrEqual(2);
|
||||
expect(gateway.liveEventRequests.at(-1)?.event).toMatchObject({
|
||||
kind: "lifecycle",
|
||||
payload: { phase: "end" },
|
||||
});
|
||||
});
|
||||
|
||||
it("degrades a repeated no-progress live resync without hanging the run", async () => {
|
||||
@@ -849,7 +869,11 @@ describe("worker runtime", () => {
|
||||
|
||||
expect(gateway.inferenceRequests).toHaveLength(1);
|
||||
expect(gateway.acceptedTranscriptRequests).toHaveLength(2);
|
||||
expect(gateway.liveEventRequests).toHaveLength(2);
|
||||
expect(gateway.liveEventRequests).toHaveLength(3);
|
||||
expect(gateway.liveEventRequests.at(-1)?.event).toMatchObject({
|
||||
kind: "lifecycle",
|
||||
payload: { phase: "end" },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when worker admission is rejected", async () => {
|
||||
@@ -887,6 +911,10 @@ describe("worker runtime", () => {
|
||||
|
||||
await expect(result).rejects.toThrow("operator stopped worker");
|
||||
expect(gateway.methods).toContain("worker.inference.cancel");
|
||||
expect(gateway.liveEventRequests.at(-1)?.event).toMatchObject({
|
||||
kind: "lifecycle",
|
||||
payload: { phase: "end", aborted: true },
|
||||
});
|
||||
});
|
||||
|
||||
it("bounds shutdown when remote inference cancellation cannot settle", async () => {
|
||||
@@ -905,14 +933,17 @@ describe("worker runtime", () => {
|
||||
});
|
||||
|
||||
it.each([
|
||||
["error", "fixture provider failed", "error", "error"],
|
||||
["cancelled", "fixture inference cancelled", "aborted", "end"],
|
||||
["error", "error", "error"],
|
||||
["cancelled", "aborted", "end"],
|
||||
] as const)(
|
||||
"reports remote inference %s terminals as failed turns",
|
||||
async (plan, message, stopReason, lifecyclePhase) => {
|
||||
async (plan, stopReason, lifecyclePhase) => {
|
||||
const { gateway, launch } = await setup({ inferencePlans: [plan] });
|
||||
|
||||
await expect(runWorkerDescriptor(launch)).rejects.toThrow(message);
|
||||
await expect(runWorkerDescriptor(launch)).resolves.toEqual({
|
||||
status: "failed",
|
||||
reason: "turn-failed",
|
||||
});
|
||||
const assistant = gateway.transcriptRequests
|
||||
.flatMap((request) => request.messages)
|
||||
.toReversed()
|
||||
@@ -926,6 +957,19 @@ describe("worker runtime", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps an unacknowledged failed-turn terminal as an infrastructure failure", async () => {
|
||||
const { gateway, launch } = await setup({
|
||||
inferencePlans: ["error"],
|
||||
liveFailure: "capacity-exceeded",
|
||||
});
|
||||
|
||||
await expect(runWorkerDescriptor(launch)).rejects.toThrow("worker live event rejected");
|
||||
expect(gateway.liveEventRequests.at(-1)?.event).toMatchObject({
|
||||
kind: "lifecycle",
|
||||
payload: { phase: "error" },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when a heartbeat is rejected without fencing", async () => {
|
||||
const { launch } = await setup({
|
||||
inferencePlans: ["hold"],
|
||||
@@ -958,9 +1002,10 @@ describe("worker runtime", () => {
|
||||
async (plan) => {
|
||||
const { gateway, launch } = await setup({ inferencePlans: [plan] });
|
||||
|
||||
await expect(runWorkerDescriptor(launch)).rejects.toThrow(
|
||||
"Worker inference result exceeds the transcript message limit.",
|
||||
);
|
||||
await expect(runWorkerDescriptor(launch)).resolves.toEqual({
|
||||
status: "failed",
|
||||
reason: "turn-failed",
|
||||
});
|
||||
const assistant = gateway.transcriptRequests
|
||||
.flatMap((request) => request.messages)
|
||||
.toReversed()
|
||||
|
||||
@@ -9,8 +9,11 @@ import {
|
||||
WorkerTranscriptCommitClient,
|
||||
} from "./worker-rpc-clients.js";
|
||||
|
||||
type WorkerRuntimeResult =
|
||||
// Cross-process contract: serialized to stdout by runWorkerCommand and parsed by the
|
||||
// gateway worker turn launcher.
|
||||
export type WorkerRuntimeResult =
|
||||
| { status: "completed"; transcriptLeafId: string | null; transcriptNextSeq: number }
|
||||
| { status: "failed"; reason: "turn-failed" }
|
||||
| { status: "fenced"; reason: "credential-replaced" | "owner-epoch-mismatch" };
|
||||
|
||||
const WORKER_REMOTE_CANCEL_GRACE_MS = 1_000;
|
||||
@@ -52,6 +55,7 @@ export async function runWorkerDescriptor(
|
||||
|
||||
const abortController = new AbortController();
|
||||
let turnStarted = false;
|
||||
let terminalLiveAcked = false;
|
||||
let forcedStopTimer: NodeJS.Timeout | undefined;
|
||||
const connection = createWorkerConnection({
|
||||
socketPath: descriptor.socketPath,
|
||||
@@ -121,6 +125,7 @@ export async function runWorkerDescriptor(
|
||||
sessionKey: `worker:${descriptor.admission.sessionId}`,
|
||||
runId: descriptor.assignment.runId,
|
||||
prompt: descriptor.assignment.prompt,
|
||||
suppressPromptTranscript: descriptor.assignment.suppressPromptTranscript,
|
||||
modelRef: descriptor.assignment.modelRef,
|
||||
initialMessages: descriptor.assignment.initialMessages,
|
||||
...(descriptor.assignment.systemPrompt === undefined
|
||||
@@ -136,6 +141,12 @@ export async function runWorkerDescriptor(
|
||||
live: {
|
||||
emit: async (event) => {
|
||||
await live.emit(descriptor.assignment.runId, event);
|
||||
if (
|
||||
event.kind === "lifecycle" &&
|
||||
(event.payload.phase === "end" || event.payload.phase === "error")
|
||||
) {
|
||||
terminalLiveAcked = true;
|
||||
}
|
||||
},
|
||||
},
|
||||
signal: abortController.signal,
|
||||
@@ -148,6 +159,12 @@ export async function runWorkerDescriptor(
|
||||
if (fenced) {
|
||||
return fenced;
|
||||
}
|
||||
if (options.signal?.aborted) {
|
||||
throw toError(options.signal.reason, "worker interrupted");
|
||||
}
|
||||
if (terminalLiveAcked && connection.state.kind === "ready") {
|
||||
return { status: "failed", reason: "turn-failed" };
|
||||
}
|
||||
throw toError(error, "worker session failed");
|
||||
}
|
||||
const fenced = fencedResult(connection.state);
|
||||
|
||||
Reference in New Issue
Block a user