fix(worker): preserve long Responses sessions across cloud handoff (#120803)

* fix(worker): preserve Responses replay across handoff

Long OpenAI Responses cloud-worker sessions retain authoritative compaction replay through windowing and fail visibly when it cannot be preserved.

* test(worker): align replay fixture with frame limit
This commit is contained in:
Peter Steinberger
2026-08-08 19:11:29 -07:00
committed by GitHub
parent 7417801580
commit 121ce7faed
17 changed files with 922 additions and 262 deletions
@@ -49,9 +49,9 @@ export const WORKER_PROTOCOL_MAX_FEATURE_LENGTH = 128;
export const WORKER_TRANSCRIPT_MAX_BATCH_MESSAGES = 64;
export const WORKER_TRANSCRIPT_MAX_CONTENT_PARTS = 128;
export const WORKER_TRANSCRIPT_MAX_JSON_DEPTH = 32;
// Reserve at least 12 KiB for the complete transcript commit envelope and
// assistant content. A 60 KiB replay leaves effectively no safe frame headroom.
export const WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES = 48 * 1024;
// Replay is opaque and cannot be truncated. Transcript projection separately
// verifies that the complete commit frame fits the protocol payload ceiling.
export const WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES = WORKER_PROTOCOL_MAX_PAYLOAD_BYTES;
const WorkerCredentialSchema = Type.String({ minLength: 16, maxLength: 256 });
const WorkerProtocolFeatureSchema = Type.String({
@@ -20,7 +20,10 @@ import { onTrustedInternalDiagnosticEvent } from "../../infra/diagnostic-events.
import { bindModelLlmRuntime } from "../../llm/model-runtime-binding.js";
import type { AssistantMessage, Model, StreamFn, Usage } from "../../llm/types.js";
import { createAssistantMessageEventStream } from "../../llm/utils/event-stream.js";
import { isWorkerTranscriptMessageFrameSafe } from "../../worker/transcript-message.js";
import {
isWorkerTranscriptMessageFrameSafe,
WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE,
} from "../../worker/transcript-message.js";
import type { WorkerConnectionIdentity } from "./connection-identity.js";
import {
createWorkerInferenceExecutor,
@@ -499,7 +502,7 @@ describe("worker inference provider runtime", () => {
});
});
it("keeps inference successful while omitting over-budget replay with a redacted diagnostic", async () => {
it("returns a typed error when authoritative replay cannot be persisted", async () => {
const runtime = setup();
const message = finalMessage();
message.providerReplay = {
@@ -520,8 +523,12 @@ describe("worker inference provider runtime", () => {
const outcome = await runtime.executor(params(request(), vi.fn())).finally(unsubscribe);
expect(outcome).toMatchObject({ type: "done", message: { stopReason: "stop" } });
expect(outcome.type === "done" ? outcome.message.providerReplay : undefined).toBeUndefined();
expect(outcome).toMatchObject({
type: "error",
reason: "provider-error",
message: WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE,
usage: message.usage,
});
expect(payloadEvents).toEqual([
expect.objectContaining({
type: "payload.large",
@@ -535,13 +542,14 @@ describe("worker inference provider runtime", () => {
expect(JSON.stringify(payloadEvents)).not.toContain(message.providerReplay.data);
});
it("keeps a maximum-budget replay frame-safe through the terminal projection", async () => {
it("keeps a maximum fitting replay exact through the terminal projection", async () => {
const runtime = setup();
const message = finalMessage();
const ciphertext = `cipher-${"x".repeat(60 * 1024)}-€`;
message.providerReplay = {
v: 1,
type: "openai-responses-compaction",
data: "x".repeat(WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES),
data: ciphertext,
provider: "openai",
api: "openai-responses",
model: MODEL,
@@ -554,9 +562,7 @@ describe("worker inference provider runtime", () => {
if (outcome.type !== "done") {
throw new Error("expected successful worker inference");
}
expect(outcome.message.providerReplay?.data).toHaveLength(
WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES,
);
expect(outcome.message.providerReplay?.data).toBe(ciphertext);
expect(isWorkerTranscriptMessageFrameSafe(outcome.message)).toBe(true);
});
@@ -69,6 +69,7 @@ import type {
} from "../../llm/types.js";
import { resolveProviderModelRoutes } from "../../plugins/provider-model-routes.js";
import { estimateUsageCost, resolveModelCostConfig } from "../../utils/usage-format.js";
import { WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE } from "../../worker/transcript-message.js";
import {
projectWorkerInferenceTerminalMessage,
type WorkerInferenceModelIdentity,
@@ -146,11 +147,12 @@ const ERROR_MESSAGES = {
function inferenceError(
reason: Extract<WorkerInferenceTerminalOutcome, { type: "error" }>["reason"],
usage?: Usage,
message: string = ERROR_MESSAGES[reason],
): WorkerInferenceTerminalOutcome {
return {
type: "error",
reason,
message: ERROR_MESSAGES[reason],
message,
...(usage ? { usage: structuredClone(usage) } : {}),
};
}
@@ -766,28 +768,31 @@ export function createWorkerInferenceExecutor(
if (!toolCalls.matchesTerminal(event.message)) {
return inferenceError("provider-error");
}
return {
type: "done",
message: projectWorkerInferenceTerminalMessage({
message: event.message,
modelIdentity,
stopReason: event.reason,
onProviderReplayOmitted: ({ bytes, limitBytes, reason }) => {
if (!isDiagnosticsEnabled(approved.config)) {
return;
}
emitTrustedDiagnosticEvent({
type: "payload.large",
surface: "worker.provider-replay",
action: "rejected",
bytes,
limitBytes,
reason,
trace: freezeDiagnosticTraceContext(trace),
});
},
}),
};
const terminal = projectWorkerInferenceTerminalMessage({
message: event.message,
modelIdentity,
stopReason: event.reason,
});
if (terminal.kind === "provider-replay-unavailable") {
if (isDiagnosticsEnabled(approved.config)) {
const { bytes, limitBytes, reason } = terminal.details;
emitTrustedDiagnosticEvent({
type: "payload.large",
surface: "worker.provider-replay",
action: "rejected",
bytes,
limitBytes,
reason,
trace: freezeDiagnosticTraceContext(trace),
});
}
return inferenceError(
"provider-error",
event.message.usage,
WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE,
);
}
return { type: "done", message: terminal.message };
}
if (event.type === "error") {
recordUsage(event.error.usage);
@@ -2,7 +2,7 @@ import type { WorkerInferenceTerminalOutcome } from "../../../packages/gateway-p
import type { AssistantMessage } from "../../llm/types.js";
import {
projectWorkerProviderReplay,
type WorkerProviderReplayOmission,
type WorkerMessageProjection,
} from "../../worker/transcript-message.js";
export type WorkerInferenceModelIdentity = {
@@ -15,8 +15,7 @@ export function projectWorkerInferenceTerminalMessage(params: {
message: AssistantMessage;
modelIdentity: WorkerInferenceModelIdentity;
stopReason: Extract<AssistantMessage["stopReason"], "stop" | "length" | "toolUse">;
onProviderReplayOmitted?: (omission: WorkerProviderReplayOmission) => void;
}): Extract<WorkerInferenceTerminalOutcome, { type: "done" }>["message"] {
}): WorkerMessageProjection<Extract<WorkerInferenceTerminalOutcome, { type: "done" }>["message"]> {
const content = params.message.content.map((part) => {
switch (part.type) {
case "text":
@@ -88,6 +87,6 @@ export function projectWorkerInferenceTerminalMessage(params: {
return projectWorkerProviderReplay({
message: projected,
providerReplay: params.message.providerReplay,
onOmitted: params.onProviderReplayOmitted,
purpose: "transcript",
});
}
@@ -3,6 +3,7 @@ import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES } from "../../../packages/gateway-protocol/src/schema/worker-inference.js";
import { createEmbeddedRunLaneController } from "../../agents/embedded-agent-runner/run/lane-controller.js";
import { installSessionPlacementAdmissionProvider } from "../../agents/session-placement-admission.js";
import { SessionManager } from "../../agents/sessions/session-manager.js";
@@ -37,6 +38,7 @@ import {
parseWorkerLaunchDescriptor,
type WorkerLaunchDescriptor,
} from "../../worker/launch-descriptor.js";
import { WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE } from "../../worker/transcript-message.js";
import type { MintedWorkerCredential } from "./credential.js";
import {
createWorkerSessionPlacementStore,
@@ -1116,6 +1118,86 @@ describe("worker turn launcher", () => {
expect(placements.get(SESSION_ID)).toMatchObject({ state: "active", turnClaim: null });
});
it("fails impossible replay before handoff and keeps the active placement reusable", async () => {
seedActivePlacement();
const manager = openSessionManager();
manager.appendMessage(
makeAgentAssistantMessage({
content: [{ type: "toolCall", id: "call-replay", name: "read", arguments: {} }],
model: "gpt-test",
providerReplay: {
v: 1,
type: "openai-responses-compaction",
data: "gAAAAlauncherReplayCiphertext",
provider: "openai",
api: "openai-responses",
model: "gpt-test",
baseUrlHash: "ozhevd1smnk8s",
},
stopReason: "toolUse",
timestamp: 1,
}),
);
manager.appendMessage({
role: "toolResult",
toolCallId: "call-replay",
toolName: "read",
content: [{ type: "text", text: "result" }],
details: { payload: "x".repeat(WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES) },
isError: false,
timestamp: 2,
});
const runWorkspaceCommand = vi.fn(async (): Promise<SpawnResult> => {
throw new Error("unexpected worker handoff");
});
const acknowledgeCredentialDelivery = vi.fn(() => true);
const startTunnel = vi.fn(
async (): Promise<WorkerTunnelHandle> => ({
environmentId: ENVIRONMENT_ID,
ownerEpoch: OWNER_EPOCH,
remoteSocketPath: "/worker/gateway.sock",
quiesceWorkspace: vi.fn(),
runWorkspaceCommand,
syncWorkspace: vi.fn(),
reconcileWorkspace: vi.fn(),
stop: vi.fn(async () => {}),
}),
);
const stopTunnel = vi.fn(async () => {});
const destroy = vi.fn(async () => attachedEnvironment());
const environments: WorkerTurnEnvironmentService = {
get: vi.fn(() => attachedEnvironment()),
acquireTurnCredential: vi.fn(async () => credential()),
acknowledgeCredentialDelivery,
startTunnel,
stopTunnel,
destroy,
};
const provider = createWorkerSessionTurnPlacementProvider({ environments, placements });
const runLocal = vi.fn(async () => ({ meta: { durationMs: 1 } }));
await expect(
provider.executeTurn(
{
sessionId: SESSION_ID,
sessionKey: SESSION_KEY,
agentId: "main",
runId: "run-replay-local-fallback",
},
turn("run-replay-local-fallback"),
runLocal,
),
).rejects.toThrow(WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE);
expect(startTunnel).toHaveBeenCalledOnce();
expect(runWorkspaceCommand).not.toHaveBeenCalled();
expect(runLocal).not.toHaveBeenCalled();
expect(acknowledgeCredentialDelivery).not.toHaveBeenCalled();
expect(stopTunnel).not.toHaveBeenCalled();
expect(destroy).not.toHaveBeenCalled();
expect(placements.get(SESSION_ID)).toMatchObject({ state: "active", turnClaim: null });
});
it("preserves a terminal workspace result when the worker child later exits nonzero", async () => {
seedActivePlacement();
const destroy = vi.fn(async () => attachedEnvironment());
@@ -13,6 +13,7 @@ import { emitTrustedDiagnosticEvent, isDiagnosticsEnabled } from "../../infra/di
import { formatErrorMessage } from "../../infra/errors.js";
import { redactSensitiveText } from "../../logging/redact.js";
import { parseWorkerLaunchDescriptor } from "../../worker/launch-descriptor.js";
import { WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE } from "../../worker/transcript-message.js";
import type {
WorkerSessionPlacementRecord,
WorkerSessionPlacementStore,
@@ -81,6 +82,20 @@ type WorkerTurnLauncherOptions = {
class WorkerTurnExecutionError extends Error {}
class WorkerWorkspaceReconciliationError extends Error {}
function emitProviderReplayRejected(
config: SessionPlacementTurnParams["config"],
details: { reason: string; bytes?: number; limitBytes?: number; count?: number },
): void {
if (isDiagnosticsEnabled(config)) {
emitTrustedDiagnosticEvent({
type: "payload.large",
surface: "worker.provider-replay",
action: "rejected",
...details,
});
}
}
async function executeLocalTurn<T>(params: {
claim: LocalTurnPlacementClaim;
placements: WorkerSessionPlacementStore;
@@ -245,24 +260,20 @@ async function executeWorkerTurn(params: {
turn.userTurnTranscriptRecorder?.hasPersisted() === true;
const contextMessages = convertToLlm(manager.buildSessionContext().messages);
const leaf = manager.getLeafEntry();
const initialMessages = windowInitialMessages(
const initialMessagePlan = windowInitialMessages(
userMessageAlreadyPersisted && leaf?.type === "message" && leaf.message.role === "user"
? contextMessages.slice(0, -1)
: contextMessages,
({ bytes, limitBytes, reason }) => {
if (!isDiagnosticsEnabled(turn.config)) {
return;
}
emitTrustedDiagnosticEvent({
type: "payload.large",
surface: "worker.provider-replay",
action: "rejected",
bytes,
limitBytes,
reason,
});
},
);
if (initialMessagePlan.kind === "provider-replay-unavailable") {
const details = initialMessagePlan.details;
emitProviderReplayRejected(
turn.config,
"bytes" in details ? details : { count: details.messageCount, reason: details.reason },
);
throw new WorkerTurnExecutionError(WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE);
}
const initialMessages = initialMessagePlan.messages;
let baseLeafId = manager.getLeafId();
if (!userMessageAlreadyPersisted) {
const persisted = turn.userTurnTranscriptRecorder
@@ -308,7 +319,7 @@ async function executeWorkerTurn(params: {
});
const reasoning = mapThinkingLevelForProvider(turn.thinkLevel);
const toolAuthority = resolveWorkerToolAuthority({ modelRef, turn });
const descriptor = fitLaunchDescriptor(
const launchPlan = fitLaunchDescriptor(
(windowedMessages) =>
parseWorkerLaunchDescriptor({
version: 2,
@@ -344,6 +355,15 @@ async function executeWorkerTurn(params: {
}),
initialMessages,
);
if (launchPlan.kind === "local-fallback") {
emitProviderReplayRejected(turn.config, {
bytes: launchPlan.bytes,
limitBytes: launchPlan.limitBytes,
reason: launchPlan.reason,
});
throw new WorkerTurnExecutionError(WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE);
}
const descriptor = launchPlan.descriptor;
turn.userTurnTranscriptRecorder?.markSentToProvider?.();
turn.onExecutionPhase?.({ phase: "attempt_dispatch", backend: "cloud-worker" });
const handoffAbort = new AbortController();
@@ -1,8 +1,97 @@
import { describe, expect, it, vi } from "vitest";
import { WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES } from "../../../packages/gateway-protocol/src/schema/worker-admission.js";
import { describe, expect, it } from "vitest";
import {
WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES,
WORKER_INFERENCE_MAX_CONTEXT_MESSAGES,
} from "../../../packages/gateway-protocol/src/schema/worker-inference.js";
import type { AgentMessage } from "../../agents/runtime/index.js";
import type { SessionPlacementTurnParams } from "../../agents/session-placement-admission.js";
import { assertSupportedTurn, windowInitialMessages } from "./worker-turn-payload.js";
import type { WorkerLaunchDescriptor } from "../../worker/launch-descriptor.js";
import {
assertSupportedTurn,
fitLaunchDescriptor,
windowInitialMessages,
} from "./worker-turn-payload.js";
const PROVIDER_REPLAY = {
v: 1 as const,
type: "openai-responses-compaction",
data: "opaque-worker-replay",
provider: "openai",
api: "openai-responses",
model: "gpt-5.6-luna",
baseUrlHash: "ozhevd1smnk8s",
};
function userMessage(text: string, timestamp: number): AgentMessage {
return { role: "user", content: [{ type: "text", text }], timestamp };
}
function assistantMessage(timestamp: number, replay = false): AgentMessage {
return {
role: "assistant",
content: [{ type: "text", text: "visible" }],
api: "openai-responses",
provider: "openai",
model: "gpt-5.6-luna",
...(replay ? { providerReplay: structuredClone(PROVIDER_REPLAY) } : {}),
usage: {
input: 1,
output: 1,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 2,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp,
};
}
function toolResultMessage(details: unknown, timestamp: number): AgentMessage {
return {
role: "toolResult",
toolCallId: "call-replay",
toolName: "read",
content: [{ type: "text", text: "result" }],
details,
isError: false,
timestamp,
};
}
function buildDescriptor(
initialMessages: WorkerLaunchDescriptor["assignment"]["initialMessages"],
): WorkerLaunchDescriptor {
return {
version: 2,
socketPath: "/tmp/worker.sock",
admission: {
environmentId: "environment",
credential: "worker-fixture-credential",
sessionId: "session",
ownerEpoch: 1,
rpcSetVersion: 1,
handshake: {
bundleHash: "a".repeat(64),
openclawVersion: "test",
protocolFeatures: [],
},
},
assignment: {
runId: "run",
turnId: "turn",
prompt: "prompt",
suppressPromptTranscript: true,
workspaceDir: "/tmp/workspace",
modelRef: { provider: "openai", model: "gpt-5.6-luna" },
inferenceOptions: {},
initialMessages,
transcript: { baseLeafId: null, nextSeq: 1 },
liveEvents: { ackedSeq: 0, nextSeq: 1 },
toolAuthority: { allowedToolNames: [] },
},
};
}
describe("assertSupportedTurn", () => {
it("accepts scheduled authority for the worker launch envelope", () => {
@@ -36,46 +125,146 @@ describe("assertSupportedTurn", () => {
});
describe("windowInitialMessages", () => {
it("reports replay omitted from the worker launch envelope", () => {
const onOmitted = vi.fn();
const messages = windowInitialMessages(
[
{
role: "assistant",
content: [{ type: "text", text: "visible" }],
api: "openai-responses",
provider: "openai",
model: "gpt-5.5",
providerReplay: {
v: 1,
type: "openai-responses-compaction",
data: "x".repeat(WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES + 1),
provider: "openai",
api: "openai-responses",
model: "gpt-5.5",
baseUrlHash: "ozhevd1smnk8s",
},
usage: {
input: 1,
output: 1,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 2,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: 1,
} as AgentMessage,
],
onOmitted,
it("pins the newest replay carrier when the normal cutoff would pass it", () => {
const history = [userMessage("old", 1), assistantMessage(2, true)];
history.push(
...Array.from({ length: WORKER_INFERENCE_MAX_CONTEXT_MESSAGES - 2 }, (_value, index) =>
userMessage(`suffix-${index}`, index + 3),
),
);
expect(messages).toHaveLength(1);
expect(messages[0]).not.toHaveProperty("providerReplay");
expect(onOmitted).toHaveBeenCalledWith({
bytes: WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES + 1,
limitBytes: WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES,
reason: "provider-replay-data-budget",
const result = windowInitialMessages(history);
expect(result.kind).toBe("complete");
if (result.kind !== "complete") {
throw new Error("expected complete window");
}
expect(result.messages).toHaveLength(WORKER_INFERENCE_MAX_CONTEXT_MESSAGES - 1);
expect(result.messages[0]).toMatchObject({
role: "assistant",
providerReplay: PROVIDER_REPLAY,
});
});
it("reserves one context slot for the current prompt", () => {
const history = Array.from({ length: WORKER_INFERENCE_MAX_CONTEXT_MESSAGES }, (_value, index) =>
userMessage(`history-${index}`, index + 1),
);
const result = windowInitialMessages(history);
expect(result.kind).toBe("complete");
if (result.kind !== "complete") {
throw new Error("expected complete window");
}
expect(result.messages).toHaveLength(WORKER_INFERENCE_MAX_CONTEXT_MESSAGES - 1);
expect(result.messages[0]).toMatchObject({
role: "user",
content: [{ type: "text", text: "history-1" }],
});
});
it("keeps historical replay that fits launch inference but not a transcript commit frame", () => {
const message = assistantMessage(1, true);
if (message.role !== "assistant" || !message.providerReplay) {
throw new Error("expected replay carrier");
}
const ciphertext = "\0".repeat(12_000);
message.providerReplay = {
...message.providerReplay,
id: "i".repeat(65_536),
data: ciphertext,
};
const result = windowInitialMessages([message]);
expect(result.kind).toBe("complete");
if (result.kind !== "complete") {
throw new Error("expected complete window");
}
expect(result.messages[0]).toMatchObject({
role: "assistant",
providerReplay: { id: "i".repeat(65_536), data: ciphertext },
});
});
it("returns a typed degraded result instead of slicing past replay", () => {
const history = [assistantMessage(1, true)];
history.push(
...Array.from({ length: WORKER_INFERENCE_MAX_CONTEXT_MESSAGES - 1 }, (_value, index) =>
userMessage(`suffix-${index}`, index + 2),
),
);
expect(windowInitialMessages(history)).toEqual({
kind: "provider-replay-unavailable",
details: {
reason: "provider-replay-message-limit",
messageCount: WORKER_INFERENCE_MAX_CONTEXT_MESSAGES,
limitMessages: WORKER_INFERENCE_MAX_CONTEXT_MESSAGES - 1,
},
});
});
});
describe("fitLaunchDescriptor", () => {
it("drops complete old turns while retaining the replay anchor", () => {
const large = "x".repeat(13 * 1024 * 1024);
const projected = windowInitialMessages([
userMessage(large, 1),
userMessage(large, 2),
assistantMessage(3, true),
]);
if (projected.kind !== "complete") {
throw new Error("expected complete projection");
}
const plan = fitLaunchDescriptor(buildDescriptor, projected.messages);
expect(plan.kind).toBe("launch");
if (plan.kind !== "launch") {
throw new Error("expected launch plan");
}
expect(plan.descriptor.assignment.initialMessages).toHaveLength(2);
expect(plan.descriptor.assignment.initialMessages[1]).toMatchObject({
role: "assistant",
providerReplay: PROVIDER_REPLAY,
});
});
it("drops a non-user prefix directly to the replay owner", () => {
const projected = windowInitialMessages([
toolResultMessage({ payload: "x".repeat(WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES) }, 1),
assistantMessage(2, true),
]);
if (projected.kind !== "complete") {
throw new Error("expected complete projection");
}
const plan = fitLaunchDescriptor(buildDescriptor, projected.messages);
expect(plan.kind).toBe("launch");
if (plan.kind !== "launch") {
throw new Error("expected launch plan");
}
expect(plan.descriptor.assignment.initialMessages).toEqual([
expect.objectContaining({ role: "assistant", providerReplay: PROVIDER_REPLAY }),
]);
});
it("requires local fallback when the replay unit cannot fit the descriptor", () => {
const projected = windowInitialMessages([
assistantMessage(1, true),
toolResultMessage({ payload: "x".repeat(WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES) }, 2),
]);
if (projected.kind !== "complete") {
throw new Error("expected complete projection");
}
expect(fitLaunchDescriptor(buildDescriptor, projected.messages)).toMatchObject({
kind: "local-fallback",
reason: "provider-replay-launch-payload-limit",
limitBytes: WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES,
});
});
});
@@ -22,53 +22,84 @@ import type { SessionPlacementTurnParams } from "../../agents/session-placement-
import { resolveEffectiveAgentRuntime } from "../../agents/thinking-runtime.js";
import { hasNonzeroUsage, normalizeUsage } from "../../agents/usage.js";
import type { WorkerLaunchDescriptor } from "../../worker/launch-descriptor.js";
import {
windowWorkerReplayMessages,
type WorkerReplayMessageWindowUnavailable,
} from "../../worker/replay-message-window.js";
import {
toWorkerTranscriptMessage,
type WorkerProviderReplayOmission,
type WorkerProviderReplayUnavailable,
} from "../../worker/transcript-message.js";
import type { WorkerRuntimeResult } from "../../worker/worker.runtime.js";
export function windowInitialMessages(
messages: AgentMessage[],
onProviderReplayOmitted?: (omission: WorkerProviderReplayOmission) => void,
): WorkerTranscriptMessage[] {
const projected = messages.flatMap((message) => {
const value = toWorkerTranscriptMessage(message, { onProviderReplayOmitted });
return value ? [value] : [];
});
if (projected.length <= WORKER_INFERENCE_MAX_CONTEXT_MESSAGES) {
return projected;
type WorkerInitialMessagePlan =
| { kind: "complete"; messages: WorkerTranscriptMessage[] }
| {
kind: "provider-replay-unavailable";
details: WorkerProviderReplayUnavailable | WorkerReplayMessageWindowUnavailable;
};
export function windowInitialMessages(messages: AgentMessage[]): WorkerInitialMessagePlan {
const windowed = windowWorkerReplayMessages(messages, WORKER_INFERENCE_MAX_CONTEXT_MESSAGES - 1);
if (windowed.kind === "provider-replay-unavailable") {
return windowed;
}
const minimumStart = projected.length - WORKER_INFERENCE_MAX_CONTEXT_MESSAGES;
const completeTurnStart = projected.findIndex(
(message, index) => index >= minimumStart && message.role === "user",
);
if (completeTurnStart < 0) {
throw new Error("Worker turn transcript has no complete context window");
const projected: WorkerTranscriptMessage[] = [];
for (const message of windowed.messages) {
const result = toWorkerTranscriptMessage(message, "inference");
if (!result) {
continue;
}
if (result.kind === "provider-replay-unavailable") {
return result;
}
projected.push(result.message);
}
return projected.slice(completeTurnStart);
return { kind: "complete", messages: projected };
}
type WorkerLaunchPlan =
| { kind: "launch"; descriptor: WorkerLaunchDescriptor }
| {
kind: "local-fallback";
reason: "provider-replay-launch-payload-limit";
bytes: number;
limitBytes: number;
};
export function fitLaunchDescriptor(
build: (initialMessages: WorkerTranscriptMessage[]) => WorkerLaunchDescriptor,
messages: WorkerTranscriptMessage[],
): WorkerLaunchDescriptor {
): WorkerLaunchPlan {
let initialMessages = messages;
while (true) {
const descriptor = build(initialMessages);
if (
Buffer.byteLength(JSON.stringify(descriptor), "utf8") <=
WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES
) {
return descriptor;
const bytes = Buffer.byteLength(JSON.stringify(descriptor), "utf8");
if (bytes <= WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES) {
return { kind: "launch", descriptor };
}
const replayIndex = initialMessages.findLastIndex(
(message) => message.role === "assistant" && message.providerReplay !== undefined,
);
if (replayIndex === 0) {
return {
kind: "local-fallback",
reason: "provider-replay-launch-payload-limit",
bytes,
limitBytes: WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES,
};
}
const nextTurn = initialMessages.findIndex(
(message, index) => index > 0 && message.role === "user",
);
if (nextTurn < 0) {
// A replay owner is a valid context start because its checkpoint replaces
// the discarded prefix; never advance past it to reach a later user turn.
const nextStart =
replayIndex > 0 && (nextTurn < 0 || nextTurn > replayIndex) ? replayIndex : nextTurn;
if (nextStart < 0) {
throw new Error("Worker turn context exceeds the launch descriptor payload limit");
}
initialMessages = initialMessages.slice(nextTurn);
initialMessages = initialMessages.slice(nextStart);
}
}
+71 -48
View File
@@ -5,12 +5,17 @@ import { WORKER_INFERENCE_MAX_CONTEXT_MESSAGES } from "../../packages/gateway-pr
import type { AgentMessage } from "../agents/runtime/index.js";
import type { AgentSessionWriteLockRunner } from "../agents/sessions/agent-session.js";
import type { Context, Message } from "../llm/types.js";
import {
windowWorkerReplayMessages,
type WorkerReplayMessageWindowUnavailable,
} from "./replay-message-window.js";
import {
cloneImageContent,
cloneTextContent,
cloneUsage,
isWorkerTranscriptMessageFrameSafe,
toWorkerTranscriptMessage,
type WorkerMessageProjection,
type WorkerProviderReplayUnavailable,
} from "./transcript-message.js";
export function toAgentMessage(message: WorkerTranscriptMessage): Message {
@@ -36,61 +41,73 @@ export function toAgentMessage(message: WorkerTranscriptMessage): Message {
timestamp: message.timestamp,
};
}
return {
...cloneUsage(message),
diagnostics: message.diagnostics?.map((diagnostic) => structuredClone(diagnostic)),
};
return structuredClone(message);
}
function toWorkerInferenceMessage(message: Message): WorkerInferenceContext["messages"][number] {
function toWorkerInferenceMessage(
message: Message,
): WorkerMessageProjection<WorkerInferenceContext["messages"][number]> {
if (message.role === "user") {
return {
role: "user",
content:
typeof message.content === "string"
? message.content
: message.content.map((part) =>
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
),
timestamp: message.timestamp,
...(message.runtimeContextCarrier ? { runtimeContextCarrier: true } : {}),
kind: "complete",
message: {
role: "user",
content:
typeof message.content === "string"
? message.content
: message.content.map((part) =>
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
),
timestamp: message.timestamp,
...(message.runtimeContextCarrier ? { runtimeContextCarrier: true } : {}),
},
};
}
const projected = toWorkerTranscriptMessage(message);
const projected = toWorkerTranscriptMessage(message, "inference");
if (!projected) {
throw new Error(`Unsupported inference message role: ${message.role}`);
}
return projected;
}
function windowWorkerInferenceMessages(messages: Context["messages"]): Context["messages"] {
if (messages.length <= WORKER_INFERENCE_MAX_CONTEXT_MESSAGES) {
return messages;
}
const minimumStart = messages.length - WORKER_INFERENCE_MAX_CONTEXT_MESSAGES;
// Start at a user turn when possible so truncation cannot orphan a tool result
// from the assistant tool call that owns it.
for (let index = minimumStart; index < messages.length; index += 1) {
if (messages[index]?.role === "user") {
return messages.slice(index);
}
}
throw new Error("Worker inference context has no complete user turn within the message limit.");
}
type WorkerInferenceContextProjection =
| { kind: "complete"; context: WorkerInferenceContext }
| {
kind: "provider-replay-unavailable";
details: WorkerProviderReplayUnavailable | WorkerReplayMessageWindowUnavailable;
};
export function toWorkerInferenceContext(context: Context): WorkerInferenceContext {
export function toWorkerInferenceContext(context: Context): WorkerInferenceContextProjection {
const windowed = windowWorkerReplayMessages(
context.messages,
WORKER_INFERENCE_MAX_CONTEXT_MESSAGES,
);
if (windowed.kind === "provider-replay-unavailable") {
return windowed;
}
const messages: WorkerInferenceContext["messages"] = [];
for (const message of windowed.messages) {
const projected = toWorkerInferenceMessage(message);
if (projected.kind === "provider-replay-unavailable") {
return projected;
}
messages.push(projected.message);
}
return {
...(context.systemPrompt === undefined ? {} : { systemPrompt: context.systemPrompt }),
messages: windowWorkerInferenceMessages(context.messages).map(toWorkerInferenceMessage),
...(context.tools
? {
tools: context.tools.map((tool) => ({
name: tool.name,
description: tool.description,
parameters: structuredClone(tool.parameters),
})),
}
: {}),
kind: "complete",
context: {
...(context.systemPrompt === undefined ? {} : { systemPrompt: context.systemPrompt }),
messages,
...(context.tools
? {
tools: context.tools.map((tool) => ({
name: tool.name,
description: tool.description,
parameters: structuredClone(tool.parameters),
})),
}
: {}),
},
};
}
@@ -108,13 +125,19 @@ export function createWorkerTranscriptRuntime(
): WorkerTranscriptRuntime {
const pendingTranscriptMessages: WorkerTranscriptMessage[] = [];
const onMessagePersisted = (message: AgentMessage) => {
const projected = toWorkerTranscriptMessage(message);
if (projected) {
if (!isWorkerTranscriptMessageFrameSafe(projected)) {
throw new Error("Worker transcript message exceeds the protocol payload limit.");
}
pendingTranscriptMessages.push(projected);
const projected = toWorkerTranscriptMessage(message, "transcript");
if (!projected) {
return;
}
if (projected.kind === "provider-replay-unavailable") {
throw new Error(
`Worker transcript cannot persist authoritative provider replay: ${projected.details.reason}.`,
);
}
if (!isWorkerTranscriptMessageFrameSafe(projected.message)) {
throw new Error("Worker transcript message exceeds the protocol payload limit.");
}
pendingTranscriptMessages.push(projected.message);
};
const flushTranscript = async () => {
while (pendingTranscriptMessages.length > 0) {
+12 -18
View File
@@ -33,7 +33,7 @@ import {
toWorkerInferenceContext,
} from "./embedded-agent-transcript.runtime.js";
import { WORKER_LOCAL_TOOL_NAMES, type WorkerLocalToolName } from "./tool-authority.js";
import { toWorkerTranscriptMessage } from "./transcript-message.js";
import { WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE } from "./transcript-message.js";
function toError(value: unknown, fallback: string): Error {
return value instanceof Error ? value : new Error(fallback, { cause: value });
@@ -79,15 +79,9 @@ type RunWorkerEmbeddedTurnParams = {
signal?: AbortSignal;
};
type RunWorkerEmbeddedTurnResult = {
messages: WorkerTranscriptMessage[];
};
const WORKER_TOOL_CONFIG = { plugins: { enabled: false } } satisfies OpenClawConfig;
export async function runWorkerEmbeddedTurn(
params: RunWorkerEmbeddedTurnParams,
): Promise<RunWorkerEmbeddedTurnResult> {
export async function runWorkerEmbeddedTurn(params: RunWorkerEmbeddedTurnParams): Promise<void> {
const model = createNativeModelOwnedRuntimeModel({
provider: params.modelRef.provider,
modelId: params.modelRef.model,
@@ -203,13 +197,20 @@ export async function runWorkerEmbeddedTurn(
});
session.agent.sessionId = params.sessionId;
session.setActiveToolsByName([...activeToolNames]);
session.agent.streamFn = (_model, context, options) =>
params.inference.stream({
session.agent.streamFn = (_model, context, options) => {
const projected = toWorkerInferenceContext(context);
if (projected.kind === "provider-replay-unavailable") {
throw new Error(
`${WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE} (${projected.details.reason})`,
);
}
return params.inference.stream({
modelRef: params.modelRef,
context: toWorkerInferenceContext(context),
context: projected.context,
options: structuredClone(params.inferenceOptions ?? {}),
...(options?.signal ? { signal: options.signal } : {}),
});
};
const liveRuntime = createWorkerLiveRuntime(params.live);
const unsubscribe = session.subscribe(liveRuntime.handleSessionEvent);
@@ -270,11 +271,4 @@ export async function runWorkerEmbeddedTurn(
if (finalTranscriptFailure !== undefined) {
throw finalTranscriptFailure;
}
return {
messages: session.agent.state.messages.flatMap((message) => {
const projected = toWorkerTranscriptMessage(message);
return projected ? [projected] : [];
}),
};
}
+45
View File
@@ -0,0 +1,45 @@
export type WorkerReplayMessageWindowUnavailable = {
reason: "provider-replay-message-limit";
messageCount: number;
limitMessages: number;
};
type WorkerReplayMessageWindow<T> =
| { kind: "complete"; messages: T[] }
| { kind: "provider-replay-unavailable"; details: WorkerReplayMessageWindowUnavailable };
type ReplayWindowMessage = { role: string; providerReplay?: unknown };
export function windowWorkerReplayMessages<T extends ReplayWindowMessage>(
messages: T[],
limitMessages: number,
): WorkerReplayMessageWindow<T> {
if (messages.length <= limitMessages) {
return { kind: "complete", messages };
}
const minimumStart = messages.length - limitMessages;
// Replay owner plus suffix is one authoritative unit. Starting after the
// owner leaves a context-blind suffix, so fail instead of trimming through it.
const replayIndex = messages.findLastIndex((message) => message.providerReplay !== undefined);
if (replayIndex >= 0 && messages.length - replayIndex > limitMessages) {
return {
kind: "provider-replay-unavailable",
details: {
reason: "provider-replay-message-limit",
messageCount: messages.length - replayIndex,
limitMessages,
},
};
}
const completeTurnStart = messages.findIndex(
(message, index) => index >= minimumStart && message.role === "user",
);
const start =
replayIndex >= 0 && (completeTurnStart < 0 || completeTurnStart > replayIndex)
? replayIndex
: completeTurnStart;
if (start < 0) {
throw new Error("Worker context has no complete user turn within the message limit.");
}
return { kind: "complete", messages: messages.slice(start) };
}
+28 -19
View File
@@ -52,11 +52,12 @@ describe("worker transcript provider replay", () => {
const message = assistantWithReplay();
Object.assign(message.providerReplay!, { providerScratch: "private" });
const projected = toWorkerTranscriptMessage(message);
expect(projected?.role).toBe("assistant");
if (!projected || projected.role !== "assistant") {
const result = toWorkerTranscriptMessage(message, "transcript");
expect(result?.kind).toBe("complete");
if (!result || result.kind !== "complete" || result.message.role !== "assistant") {
throw new Error("expected projected assistant message");
}
const projected = result.message;
expect(projected.providerReplay).toEqual(providerReplay);
expect(JSON.stringify(projected)).not.toContain("providerScratch");
expect(isWorkerTranscriptMessageFrameSafe(projected)).toBe(true);
@@ -71,43 +72,51 @@ describe("worker transcript provider replay", () => {
expect(toAgentMessage(projected)).toMatchObject({ providerReplay });
});
it("keeps the maximum replay data budget inside a complete commit frame", () => {
it("keeps replay above 48 KiB whole when the complete commit frame fits", () => {
const ciphertext = `cipher-${"x".repeat(60 * 1024)}-€`;
const message = assistantWithReplay({
...providerReplay,
data: "x".repeat(WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES),
data: ciphertext,
});
const projected = toWorkerTranscriptMessage(message);
const result = toWorkerTranscriptMessage(message, "transcript");
expect(projected?.role).toBe("assistant");
expect(projected?.role === "assistant" ? projected.providerReplay?.data.length : 0).toBe(
WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES,
);
expect(projected && isWorkerTranscriptMessageFrameSafe(projected)).toBe(true);
expect(result?.kind).toBe("complete");
if (!result || result.kind !== "complete" || result.message.role !== "assistant") {
throw new Error("expected projected assistant message");
}
expect(result.message.providerReplay?.data).toBe(ciphertext);
expect(isWorkerTranscriptMessageFrameSafe(result.message)).toBe(true);
});
it.each([
{
name: "raw UTF-8 data over budget",
name: "raw UTF-8 data over the replay field budget",
replay: { ...providerReplay, data: "x".repeat(WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES + 1) },
reason: "provider-replay-data-budget" as const,
},
{
name: "multibyte data over its byte budget",
replay: { ...providerReplay, data: "€".repeat(20_000) },
name: "multibyte data whose complete frame is over budget",
replay: { ...providerReplay, data: "€".repeat(21_845) },
reason: "transcript-commit-frame-budget" as const,
},
{
name: "JSON-escaped data over the complete frame budget",
replay: { ...providerReplay, data: "\0".repeat(12_000) },
reason: "transcript-commit-frame-budget" as const,
},
{
name: "a schema-valid id over the complete frame budget",
replay: { ...providerReplay, id: "i".repeat(65_536), data: "opaque" },
reason: "transcript-commit-frame-budget" as const,
},
])("omits the entire replay for $name", ({ replay }) => {
const projected = toWorkerTranscriptMessage(assistantWithReplay(replay));
])("degrades without ciphertext for $name", ({ replay, reason }) => {
const result = toWorkerTranscriptMessage(assistantWithReplay(replay), "transcript");
expect(projected?.role).toBe("assistant");
expect(projected?.role === "assistant" ? projected.providerReplay : undefined).toBeUndefined();
expect(projected && isWorkerTranscriptMessageFrameSafe(projected)).toBe(true);
if (!result || result.kind !== "provider-replay-unavailable") {
throw new Error("expected degraded replay projection");
}
expect(result.details).toMatchObject({ reason });
expect(JSON.stringify(result.details)).not.toContain(replay.data);
});
});
+49 -34
View File
@@ -12,11 +12,22 @@ import type { AssistantMessage, ProviderReplayState } from "../llm/types.js";
const SIZE_FRAME_ID = "00000000-0000-4000-8000-000000000000";
type WorkerTranscriptAssistantMessage = Extract<WorkerTranscriptMessage, { role: "assistant" }>;
export type WorkerProviderReplayOmission = {
export type WorkerProviderReplayUnavailable = {
bytes: number;
limitBytes: number;
reason: "provider-replay-data-budget" | "transcript-commit-frame-budget";
};
type WorkerProviderReplayUnavailableProjection = {
kind: "provider-replay-unavailable";
details: WorkerProviderReplayUnavailable;
};
export type WorkerMessageProjection<T> =
| { kind: "complete"; message: T }
| WorkerProviderReplayUnavailableProjection;
type WorkerMessageProjectionPurpose = "inference" | "transcript";
export const WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE =
"Cloud worker could not preserve authoritative provider replay. " +
"Stop or reclaim the cloud worker, then retry locally.";
export function cloneTextContent(part: { type: "text"; text: string; textSignature?: string }) {
return {
@@ -30,6 +41,12 @@ export function cloneImageContent(part: { type: "image"; data: string; mimeType:
return { type: "image" as const, data: part.data, mimeType: part.mimeType };
}
function providerReplayUnavailable(
details: WorkerProviderReplayUnavailable,
): WorkerProviderReplayUnavailableProjection {
return { kind: "provider-replay-unavailable", details };
}
function cloneProviderReplay(state: ProviderReplayState): ProviderReplayState {
return {
v: state.v,
@@ -70,41 +87,39 @@ export function projectWorkerProviderReplay<
>(params: {
message: TMessage;
providerReplay: ProviderReplayState | undefined;
onOmitted?: (omission: WorkerProviderReplayOmission) => void;
}): TMessage {
purpose: WorkerMessageProjectionPurpose;
}): WorkerMessageProjection<TMessage> {
if (!params.providerReplay) {
return params.message;
return { kind: "complete", message: params.message };
}
const dataBytes = Buffer.byteLength(params.providerReplay.data, "utf8");
if (dataBytes > WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES) {
params.onOmitted?.({
return providerReplayUnavailable({
bytes: dataBytes,
limitBytes: WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES,
reason: "provider-replay-data-budget",
});
return params.message;
}
const candidate = {
...params.message,
providerReplay: cloneProviderReplay(params.providerReplay),
};
if (params.purpose === "inference") {
return { kind: "complete", message: candidate };
}
const frameBytes = workerTranscriptMessageFrameBytes(candidate);
if (frameBytes === undefined || frameBytes > WORKER_PROTOCOL_MAX_PAYLOAD_BYTES) {
params.onOmitted?.({
return providerReplayUnavailable({
bytes: frameBytes ?? WORKER_PROTOCOL_MAX_PAYLOAD_BYTES + 1,
limitBytes: WORKER_PROTOCOL_MAX_PAYLOAD_BYTES,
reason: "transcript-commit-frame-budget",
});
return params.message;
}
return candidate;
return { kind: "complete", message: candidate };
}
export function cloneUsage(
message: AssistantMessage,
onProviderReplayOmitted?: (omission: WorkerProviderReplayOmission) => void,
): WorkerTranscriptMessage & { role: "assistant" } {
const projected: WorkerTranscriptAssistantMessage = {
function toWorkerAssistantMessage(message: AssistantMessage): WorkerTranscriptAssistantMessage {
return {
role: "assistant",
content: message.content.map((part) => {
if (part.type === "text") {
@@ -176,19 +191,12 @@ export function cloneUsage(
...(message.errorBody ? { errorBody: message.errorBody } : {}),
timestamp: message.timestamp,
};
return projectWorkerProviderReplay({
message: projected,
providerReplay: message.providerReplay,
onOmitted: onProviderReplayOmitted,
});
}
export function toWorkerTranscriptMessage(
message: AgentMessage,
options?: {
onProviderReplayOmitted?: (omission: WorkerProviderReplayOmission) => void;
},
): WorkerTranscriptMessage | undefined {
purpose: WorkerMessageProjectionPurpose,
): WorkerMessageProjection<WorkerTranscriptMessage> | undefined {
if (message.role === "user") {
const content =
typeof message.content === "string"
@@ -196,22 +204,29 @@ export function toWorkerTranscriptMessage(
: message.content.map((part) =>
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
);
return { role: "user", content, timestamp: message.timestamp };
return { kind: "complete", message: { role: "user", content, timestamp: message.timestamp } };
}
if (message.role === "assistant") {
return cloneUsage(message, options?.onProviderReplayOmitted);
return projectWorkerProviderReplay({
message: toWorkerAssistantMessage(message),
providerReplay: message.providerReplay,
purpose,
});
}
if (message.role === "toolResult") {
return {
role: "toolResult",
toolCallId: message.toolCallId,
toolName: message.toolName,
content: message.content.map((part) =>
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
),
...(message.details === undefined ? {} : { details: structuredClone(message.details) }),
isError: message.isError,
timestamp: message.timestamp,
kind: "complete",
message: {
role: "toolResult",
toolCallId: message.toolCallId,
toolName: message.toolName,
content: message.content.map((part) =>
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
),
...(message.details === undefined ? {} : { details: structuredClone(message.details) }),
isError: message.isError,
timestamp: message.timestamp,
},
};
}
return undefined;
+6 -7
View File
@@ -1,9 +1,8 @@
import { describe, expect, it, vi } from "vitest";
import {
WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES,
type WorkerHelloOk,
type WorkerLiveEvent,
type WorkerTranscriptMessage,
import type {
WorkerHelloOk,
WorkerLiveEvent,
WorkerTranscriptMessage,
} from "../../packages/gateway-protocol/src/schema/worker-admission.js";
import type {
WorkerInferenceEventFrame,
@@ -288,7 +287,7 @@ describe("worker transcript commit client", () => {
});
});
it("commits a terminal assistant message at the provider replay budget", async () => {
it("commits a terminal assistant message with replay near the frame ceiling", async () => {
const harness = connectionHarness();
harness.requestTranscriptCommit.mockResolvedValueOnce({
type: "res",
@@ -309,7 +308,7 @@ describe("worker transcript commit client", () => {
providerReplay: {
v: 1,
type: "openai-responses-compaction",
data: "x".repeat(WORKER_PROVIDER_REPLAY_MAX_DATA_BYTES),
data: "x".repeat(60 * 1024),
provider: "openai",
api: "openai-responses",
model: "gpt-5.6-sol",
+30 -13
View File
@@ -13,6 +13,7 @@ import type {
WorkerInferenceStartParams,
WorkerInferenceTerminalOutcome,
} from "../../packages/gateway-protocol/src/schema/worker-inference.js";
import { runWorkerProviderReplayRoundTrip } from "../../test/helpers/worker-provider-replay-roundtrip.js";
import { SessionManager } from "../agents/sessions/session-manager.js";
import {
resolveSessionTranscriptRuntimeTarget,
@@ -180,7 +181,7 @@ type TranscriptGate = {
};
type ProviderPlan =
| { kind: "immediate"; text: string }
| { kind: "immediate"; text: string; outcome?: WorkerInferenceTerminalOutcome }
| {
kind: "partitioned";
firstRelease: Deferred<void>;
@@ -197,6 +198,15 @@ type WorkerClients = {
inference: WorkerInferenceProxyClient;
};
type WorkerClientOptions = {
admissionProof?: string;
epoch?: number;
baseLeafId?: string | null;
initialSeq?: number;
initialAckedSeq?: number;
runId?: string;
};
class ComposedGatewayHarness {
readonly root: string;
readonly stateDir: string;
@@ -316,19 +326,10 @@ class ComposedGatewayHarness {
this.faults.push(rule);
}
createClients(
params: {
admissionProof?: string;
epoch?: number;
baseLeafId?: string | null;
initialSeq?: number;
initialAckedSeq?: number;
runId?: string;
} = {},
): WorkerClients {
createDescriptor(params: WorkerClientOptions = {}): WorkerLaunchDescriptor {
const epoch = params.epoch ?? this.epoch;
const credential = params.admissionProof ?? CREDENTIAL;
const descriptor: WorkerLaunchDescriptor = {
return {
version: 2,
socketPath: this.socketPath,
admission: {
@@ -358,6 +359,11 @@ class ComposedGatewayHarness {
},
},
};
}
createClients(params: WorkerClientOptions = {}): WorkerClients {
const descriptor = this.createDescriptor(params);
const epoch = descriptor.admission.ownerEpoch;
const connection = createWorkerConnection({
socketPath: this.socketPath,
connectParams: buildWorkerConnectParams(descriptor),
@@ -559,7 +565,7 @@ class ComposedGatewayHarness {
}
const plan = this.providerPlan;
if (plan.kind === "immediate") {
return doneOutcome(plan.text);
return structuredClone(plan.outcome ?? doneOutcome(plan.text));
}
if (plan.kind === "pending") {
plan.started.resolve();
@@ -715,6 +721,17 @@ describe("cloud worker milestone 2 fault injection", () => {
await harness.close();
});
it("replays captured compaction exactly after worker commit and canonical reopen", async () => {
await runWorkerProviderReplayRoundTrip({
createDescriptor: (options) => harness.createDescriptor(options),
requestParams: (method) => harness.requestParams(method),
sessionTarget: harness.sessionTarget,
setOutcome: (outcome) => {
harness.providerPlan = { kind: "immediate", text: "roundtrip", outcome };
},
});
});
it("survives repeated tunnel partitions without transcript duplication, live replay, or rebilling", async () => {
const current = harness.createClients();
clients.push(current);
+58 -2
View File
@@ -34,6 +34,7 @@ import {
import { listRunningSessions } from "../agents/bash-process-registry.js";
import { rawDataToString } from "../infra/ws.js";
import { buildWorkerConnectParams, type WorkerLaunchDescriptor } from "./launch-descriptor.js";
import { WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE } from "./transcript-message.js";
import { WorkerAdmissionDeadlineExceededError } from "./worker-connection-contract.js";
import { createWorkerConnection, WorkerConnectionStoppedError } from "./worker-connection.js";
import {
@@ -54,6 +55,15 @@ const SESSION_ID = "worker-session";
const RUN_ID = "worker-run";
const OWNER_EPOCH = 4;
const MODEL_REF = { provider: "openai", model: "gpt-5.6-luna" } as const;
const WORKER_LOOP_REPLAY = {
v: 1 as const,
type: "openai-responses-compaction",
data: "opaque-worker-loop-replay",
provider: "openai",
api: "openai-responses",
model: MODEL_REF.model,
baseUrlHash: "ozhevd1smnk8s",
};
const BUNDLE_HASH = Array.from({ length: 64 }, () => "a").join("");
const CREDENTIAL = ["worker", "fixture", "admission"].join("-");
@@ -1148,16 +1158,20 @@ describe("worker runtime", () => {
).toBe(true);
});
it("windows near-limit history for every local tool-loop inference", async () => {
it("keeps a pinned replay anchor through repeated local tool-loop inference", async () => {
const { gateway, launch } = await setup({ inferencePlans: ["tool", "text"] });
launch.assignment.initialMessages = Array.from(
{ length: WORKER_INFERENCE_MAX_CONTEXT_MESSAGES },
{ length: WORKER_INFERENCE_MAX_CONTEXT_MESSAGES - 2 },
(_value, index): WorkerTranscriptMessage => ({
role: "user",
content: [{ type: "text", text: `history-${index}` }],
timestamp: index + 1,
}),
);
launch.assignment.initialMessages[2] = {
...assistantMessage([{ type: "text", text: "checkpoint suffix" }], "stop"),
providerReplay: structuredClone(WORKER_LOOP_REPLAY),
};
await expect(runWorkerDescriptor(launch)).resolves.toMatchObject({ status: "completed" });
@@ -1167,6 +1181,11 @@ describe("worker runtime", () => {
WORKER_INFERENCE_MAX_CONTEXT_MESSAGES,
);
expect(request.context.messages[0]?.role).toBe("user");
expect(
request.context.messages.find(
(message) => message.role === "assistant" && message.providerReplay,
),
).toMatchObject({ providerReplay: WORKER_LOOP_REPLAY });
}
expect(
gateway.inferenceRequests[1]?.context.messages.some(
@@ -1182,6 +1201,43 @@ describe("worker runtime", () => {
.map((message) => message.role),
).toEqual(["user", "assistant", "toolResult", "assistant"]);
});
it("fails before a second inference when the replay unit outgrows the window", async () => {
const { gateway, launch } = await setup({ inferencePlans: ["tool", "text"] });
launch.assignment.initialMessages = Array.from(
{ length: WORKER_INFERENCE_MAX_CONTEXT_MESSAGES - 1 },
(_value, index): WorkerTranscriptMessage => ({
role: "user",
content: [{ type: "text", text: `history-${index}` }],
timestamp: index + 1,
}),
);
launch.assignment.initialMessages[0] = {
...assistantMessage([{ type: "text", text: "checkpoint suffix" }], "stop"),
providerReplay: structuredClone(WORKER_LOOP_REPLAY),
};
await expect(runWorkerDescriptor(launch)).resolves.toEqual({
status: "failed",
reason: "turn-failed",
});
expect(gateway.inferenceRequests).toHaveLength(1);
expect(gateway.inferenceRequests[0]?.context.messages).toHaveLength(
WORKER_INFERENCE_MAX_CONTEXT_MESSAGES,
);
expect(gateway.inferenceRequests[0]?.context.messages[0]).toMatchObject({
providerReplay: WORKER_LOOP_REPLAY,
});
const terminal = gateway.transcriptRequests
.flatMap((request) => request.messages)
.toReversed()
.find((message) => message.role === "assistant");
expect(terminal).toMatchObject({
stopReason: "error",
errorMessage: `${WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE} (provider-replay-message-limit)`,
});
});
});
describe("worker reconnect clients", () => {
@@ -0,0 +1,170 @@
import { expect } from "vitest";
import { convertResponsesMessages } from "../../packages/ai/src/providers/openai-responses-shared.js";
import { captureOpenAIResponsesCompaction } from "../../packages/ai/src/transports/openai-responses-compaction-replay.js";
import type { WorkerTranscriptMessage } from "../../packages/gateway-protocol/src/schema/worker-admission.js";
import type {
WorkerInferenceStartParams,
WorkerInferenceTerminalOutcome,
} from "../../packages/gateway-protocol/src/schema/worker-inference.js";
import { validateWorkerInferenceTerminalOutcome } from "../../packages/gateway-protocol/src/schema/worker-inference.js";
import type { Context, Model } from "../../packages/llm-core/src/types.js";
import { SessionManager } from "../../src/agents/sessions/session-manager.js";
import { windowInitialMessages } from "../../src/gateway/worker-environments/worker-turn-payload.js";
import type { WorkerLaunchDescriptor } from "../../src/worker/launch-descriptor.js";
import { runWorkerDescriptor } from "../../src/worker/worker.runtime.js";
type DescriptorOptions = {
baseLeafId?: string | null;
initialSeq?: number;
initialAckedSeq?: number;
runId?: string;
};
type RoundTripHarness = {
createDescriptor(options?: DescriptorOptions): WorkerLaunchDescriptor;
requestParams(method: string): unknown[];
sessionTarget: Parameters<typeof SessionManager.open>[0];
setOutcome(outcome: WorkerInferenceTerminalOutcome): void;
};
type WorkerDoneMessage = Extract<WorkerInferenceTerminalOutcome, { type: "done" }>["message"];
function doneMessage(
model: Model<"openai-responses">,
content: WorkerDoneMessage["content"],
): WorkerDoneMessage {
return {
role: "assistant",
content,
api: model.api,
provider: model.provider,
model: model.id,
usage: {
input: 1,
output: 1,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 2,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: 1,
};
}
export async function runWorkerProviderReplayRoundTrip(harness: RoundTripHarness): Promise<void> {
const baseDescriptor = harness.createDescriptor({ runId: "replay-run-1" });
const model = {
id: baseDescriptor.assignment.modelRef.model,
name: "Fault replay model",
api: "openai-responses",
provider: baseDescriptor.assignment.modelRef.provider,
baseUrl: "https://api.openai.com/v1",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 1_000_000,
maxTokens: 8_192,
} satisfies Model<"openai-responses">;
const ciphertext = "gAAAAworkerReplayCiphertextExact_123";
const captured = doneMessage(model, [
{ type: "text", text: "before checkpoint" },
{ type: "text", text: "after checkpoint" },
]);
captureOpenAIResponsesCompaction(
captured,
{ type: "compaction", id: "cmp_worker_roundtrip", encrypted_content: ciphertext },
1,
model,
);
expect(validateWorkerInferenceTerminalOutcome({ type: "done", message: captured })).toBe(true);
harness.setOutcome({ type: "done", message: captured });
baseDescriptor.assignment.turnId = "replay-turn-1";
baseDescriptor.assignment.prompt = "capture replay";
const first = await runWorkerDescriptor(baseDescriptor);
expect(first.status).toBe("completed");
if (first.status !== "completed") {
throw new Error("expected completed first worker turn");
}
const committed = (
harness.requestParams("worker.transcript.commit") as Array<{
messages: WorkerTranscriptMessage[];
}>
).flatMap((request) => request.messages);
const committedReplay = committed.find(
(message) => message.role === "assistant" && message.providerReplay,
);
expect(
committedReplay?.role === "assistant" ? committedReplay.providerReplay?.data : undefined,
).toBe(ciphertext);
const canonicalMessages = SessionManager.open(harness.sessionTarget)
.getBranch()
.flatMap((entry) => (entry.type === "message" ? [entry.message] : []));
const replayOwner = canonicalMessages.find(
(message) => message.role === "assistant" && message.providerReplay,
);
expect(replayOwner?.role === "assistant" ? replayOwner.providerReplay?.data : undefined).toBe(
ciphertext,
);
const initial = windowInitialMessages(canonicalMessages);
if (initial.kind !== "complete") {
throw new Error("expected replayable canonical history");
}
harness.setOutcome({
type: "done",
message: doneMessage(model, [{ type: "text", text: "next reply" }]),
});
const liveAckedSeq = Math.max(
...(harness.requestParams("worker.live-event") as Array<{ seq: number }>).map(
(request) => request.seq,
),
);
const secondDescriptor = harness.createDescriptor({
runId: "replay-run-2",
baseLeafId: first.transcriptLeafId,
initialSeq: first.transcriptNextSeq,
initialAckedSeq: liveAckedSeq,
});
secondDescriptor.assignment.turnId = "replay-turn-2";
secondDescriptor.assignment.prompt = "next worker turn";
secondDescriptor.assignment.initialMessages = initial.messages;
await expect(runWorkerDescriptor(secondDescriptor)).resolves.toMatchObject({
status: "completed",
});
const requests = harness.requestParams("worker.inference.start") as WorkerInferenceStartParams[];
const nextContext = requests[1]?.context;
if (!nextContext) {
throw new Error("missing next worker inference context");
}
const converted = convertResponsesMessages(
model,
nextContext as Context,
new Set([model.provider]),
);
const compactionIndex = converted.findIndex((item) => item.type === "compaction");
expect(converted[compactionIndex]).toEqual({
type: "compaction",
id: "cmp_worker_roundtrip",
encrypted_content: ciphertext,
});
const suffixIndex = converted.findIndex(
(item) =>
item.type === "message" &&
item.role === "assistant" &&
Array.isArray(item.content) &&
item.content.some(
(content) => content.type === "output_text" && content.text === "after checkpoint",
),
);
expect(suffixIndex).toBeGreaterThan(compactionIndex);
expect(converted).toContainEqual(
expect.objectContaining({
type: "message",
role: "user",
content: [expect.objectContaining({ type: "input_text", text: "next worker turn" })],
}),
);
}