diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5cd2415bbb8..4fff59268608 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3511,15 +3511,16 @@ jobs: run: | set -euo pipefail swift_test_args=(--package-path apps/macos --enable-code-coverage) - # Hosted release and retry runs have shown test-process contention; - # keep the faster first-attempt Blacksmith path parallel. - if [[ "$SWIFT_TEST_EXECUTION" == "parallel" ]]; then - swift_test_args+=(--parallel) - else - swift_test_args+=(--no-parallel) - fi for attempt in 1 2 3; do - if swift test "${swift_test_args[@]}"; then + attempt_args=("${swift_test_args[@]}") + # Keep the fast first-attempt Blacksmith path parallel, but make + # in-job retries serial after any contention-driven failure. + if [[ "$SWIFT_TEST_EXECUTION" == "parallel" && "$attempt" -eq 1 ]]; then + attempt_args+=(--parallel) + else + attempt_args+=(--no-parallel) + fi + if swift test "${attempt_args[@]}"; then exit 0 fi echo "swift test failed (attempt $attempt/3). Retrying…" diff --git a/docs/ci.md b/docs/ci.md index 0b77f373d1ff..3b4bbbec299d 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -208,6 +208,8 @@ for commands and recovery. ### Runner backend modes +The `macos-swift` lane runs its first Blacksmith test attempt in parallel. If that attempt fails, its two in-job retries run serially to escape process and timer contention; manual dispatches, hosted fallbacks, and workflow-level reruns remain serial from their first attempt. + The repository variable `OPENCLAW_CI_RUNNER_BACKEND` controls the runner backend for `ci.yml`: | Value | Light lanes | Heavy lanes | Rerun behavior | diff --git a/docs/nodes/index.md b/docs/nodes/index.md index 9ae4f51c0470..904bec3e5f82 100644 --- a/docs/nodes/index.md +++ b/docs/nodes/index.md @@ -439,11 +439,12 @@ only while those declarations match, and provisioning requires the node and Gateway versions to match exactly. If they differ, update the node before retrying. -This setting completes device-environment provisioning and session-host status; -it does **not** yet make device turn dispatch succeed. The Gateway still returns -`device-runner-transport-unimplemented` until the local-install chain adds -supervised launch and workspace transport. Do not treat the status as proof that -a complete turn can run on the device yet. +This setting enables supervised session turns on the paired device, including +Gateway-owned workspace transfer and result reconciliation. If the device is +offline before a turn is dispatched, the Gateway waits up to 10 seconds and +then returns a visible retry/reconnect error while keeping the session placement +available for a later attempt. Gateway restart likewise preserves an idle device +placement and reconnects its tunnel lazily on the next turn. See [Anthropic: Claude sessions across computers](/providers/anthropic#claude-sessions-across-computers) for the Control UI behavior and storage sources. diff --git a/docs/plan/runners.md b/docs/plan/runners.md index f6245db67442..0829a3243734 100644 --- a/docs/plan/runners.md +++ b/docs/plan/runners.md @@ -188,10 +188,12 @@ stated honestly (revision 1 undersold this): credentials, and staged refs. Device-side GC of per-session workspace dirs and superseded bundles is a milestone exit gate, not an open question: persistent machines otherwise leak the user's own disk. -- **Placement `runner-offline`.** Heartbeat/presence loss marks the placement - with a recorded, operator-visible reason; staged results are preserved by - the existing fence machinery; the session offers "continue on gateway" - (reclaim) or "wait for device". Never a silent non-outcome. +- **Placement `runner-offline`.** Pre-dispatch device loss waits up to 10 + seconds, then returns an operator-visible coordination error without failing + the active placement or consuming model fallbacks. Idle active device + placements survive Gateway restart and validate their reconnect-scoped + tunnel lazily on the next turn. Durable status projection and the explicit + "continue on gateway" / "wait for device" actions remain milestone work. - **Dispatch target union.** `sessions.dispatch` accepts `{ profileId } | { deviceId }`; the device → environment mapping resolves server-side. Devices are not smuggled through synthesized @@ -316,7 +318,7 @@ speak. Additions: local gateway, execution-capable nodes, worker environments, and the separate cloud profiles list. Device-runner inventory adds `sessionHost` without creating another place ontology. -- **Where picker regrouped** (`ui/src/pages/new-session/place-picker.ts`): +- **Where picker regrouped** (`ui/src/pages/new-session/place-picker-sections.ts`): sections "This gateway" / "Devices" / "Cloud". Device rows intersect the environment catalog with execution-capable paired nodes; connected rows are selectable, while remembered offline rows stay visible but disabled. Cloud @@ -332,9 +334,10 @@ speak. Additions: - **Placement chip** on the session header: shows quiet current placement; active cloud placements reclaim through `sessions.reclaim` with "Bring home". Stop-and-continue moves arrive with milestone 8. -- **Remaining milestone work**: the admin-gated "Connect a machine…" foot and - busy/slot state. `runner-offline` then shows a banner with the recorded - reason and its recovery verbs. +- **Remaining milestone work**: the admin-gated "Connect a machine…" foot, + busy/slot state, and durable `runner-offline` recovery actions. Pre-dispatch + offline attempts already fail visibly after a 10-second grace without + terminalizing the placement. ### Cloud convergence (milestone 10) diff --git a/src/agents/failover-error.test.ts b/src/agents/failover-error.test.ts index ba264bd5980d..b3caa89152a6 100644 --- a/src/agents/failover-error.test.ts +++ b/src/agents/failover-error.test.ts @@ -1285,6 +1285,15 @@ describe("failover-error", () => { expect(isNonProviderRuntimeCoordinationError(wrappedRebound)).toBe(true); }); + it("returns true for direct and nested runner availability failures", () => { + const unavailable = new Error("The device runner is offline"); + unavailable.name = "WorkerRunnerUnavailableError"; + for (const error of [unavailable, new Error("worker turn failed", { cause: unavailable })]) { + expect(isNonProviderRuntimeCoordinationError(error)).toBe(true); + expect(resolveModelFallbackError(error)).toEqual({ kind: "coordination", error }); + } + }); + it("returns true for Codex missing tool-result local execution failures", () => { const missingToolResultMessage = "OpenClaw recorded a native Codex tool.call without a matching tool.result before the turn completed."; diff --git a/src/agents/failover-error.ts b/src/agents/failover-error.ts index 42f86e46a838..51bc6f822767 100644 --- a/src/agents/failover-error.ts +++ b/src/agents/failover-error.ts @@ -481,11 +481,19 @@ function hasStaleAgentRunLifecycleFailure(err: unknown): boolean { ); } -function hasGatewayDrainingFailure(err: unknown): boolean { +function errorGraphHasName(err: unknown, name: string): boolean { return collectErrorGraphCandidates(err, (candidate) => { const errors = candidate.errors; return [candidate.error, candidate.cause, ...(Array.isArray(errors) ? errors : [])]; - }).some((candidate) => readErrorName(candidate) === "GatewayDrainingError"); + }).some((candidate) => readErrorName(candidate) === name); +} + +function hasGatewayDrainingFailure(err: unknown): boolean { + return errorGraphHasName(err, "GatewayDrainingError"); +} + +function hasWorkerRunnerUnavailableFailure(err: unknown): boolean { + return errorGraphHasName(err, "WorkerRunnerUnavailableError"); } function hasDirectProviderFailureIdentity(err: unknown): boolean { @@ -883,7 +891,7 @@ export function resolveModelFallbackError( } // Gateway admission can fail before any provider turn starts. Preserve that // identity through wrappers and aggregates so fallback cannot blame a model. - if (hasGatewayDrainingFailure(err)) { + if (hasGatewayDrainingFailure(err) || hasWorkerRunnerUnavailableFailure(err)) { return { kind: "coordination", error: err }; } const staleLifecycleFailure = hasStaleAgentRunLifecycleFailure(err); diff --git a/src/auto-reply/reply/agent-runner-direct-runtime-config.test.ts b/src/auto-reply/reply/agent-runner-direct-runtime-config.test.ts index 3fcf913bd4ba..9b6d172ef88a 100644 --- a/src/auto-reply/reply/agent-runner-direct-runtime-config.test.ts +++ b/src/auto-reply/reply/agent-runner-direct-runtime-config.test.ts @@ -149,6 +149,8 @@ function createReplyOperation(): TestReplyOperation { updateSessionKey: vi.fn(), hasOwnedSessionId: vi.fn(() => false), bindToolAuthorityFingerprint: vi.fn(), + bindToolAuthorityProjector: vi.fn(), + projectToolAuthorityFingerprint: vi.fn(), bindToolAuthorityRoute: vi.fn(), attachBackend: vi.fn(), detachBackend: vi.fn(), diff --git a/src/auto-reply/reply/agent-runner-memory.test.ts b/src/auto-reply/reply/agent-runner-memory.test.ts index 4f7c8b9d0dd7..a68a5f55f5da 100644 --- a/src/auto-reply/reply/agent-runner-memory.test.ts +++ b/src/auto-reply/reply/agent-runner-memory.test.ts @@ -96,6 +96,8 @@ function createReplyOperation(): TestReplyOperation { updateSessionId: vi.fn(), updateSessionKey: vi.fn(), bindToolAuthorityFingerprint: vi.fn(), + bindToolAuthorityProjector: vi.fn(), + projectToolAuthorityFingerprint: vi.fn(), bindToolAuthorityRoute: vi.fn(), attachBackend: vi.fn(), detachBackend: vi.fn(), diff --git a/src/auto-reply/reply/agent-runner-run.ts b/src/auto-reply/reply/agent-runner-run.ts index ad28b2a41256..a1ba3491ca8f 100644 --- a/src/auto-reply/reply/agent-runner-run.ts +++ b/src/auto-reply/reply/agent-runner-run.ts @@ -48,7 +48,10 @@ import * as replyRunState from "./reply-operation-run-state.js"; import { type ReplyOperation, replyRunRegistry } from "./reply-run-registry.js"; import { bindReplyOperationTyping } from "./reply-run-typing.js"; import { createReplyToModeFilterForChannel, resolveReplyToMode } from "./reply-threading.js"; -import { resolveFollowupRunToolAuthorityFingerprint } from "./reply-tool-authority.js"; +import { + createFollowupRunToolAuthorityProjector, + resolveFollowupRunToolAuthorityFingerprint, +} from "./reply-tool-authority.js"; import { admitReplyTurn, resolveReplyTurnKind } from "./reply-turn-admission.js"; import { isDuplicateRestartRecoverySource, @@ -477,7 +480,10 @@ export async function runReplyAgent( } } } - replyOperation.bindToolAuthorityFingerprint(incomingToolAuthorityFingerprint); + replyOperation.bindToolAuthorityProjector(createFollowupRunToolAuthorityProjector(followupRun)); + replyOperation.bindToolAuthorityFingerprint( + resolveFollowupRunToolAuthorityFingerprint(followupRun), + ); bindReplyOperationTyping(replyOperation, typing); let runFollowupTurn = queuedRunFollowupTurn; let shouldDrainQueuedFollowupsAfterClear = false; diff --git a/src/auto-reply/reply/reply-run-registry.contracts.ts b/src/auto-reply/reply/reply-run-registry.contracts.ts index f75448077daf..fd820625cf4c 100644 --- a/src/auto-reply/reply/reply-run-registry.contracts.ts +++ b/src/auto-reply/reply/reply-run-registry.contracts.ts @@ -1,11 +1,18 @@ +import type { ScheduledToolPolicyContext } from "../../agents/scheduled-tool-policy.js"; +import type { TrustedSubagentCompletionHandoff } from "../../agents/subagents/announce/subagent-announce-handoff.js"; +import type { ChatType } from "../../channels/chat-type.js"; +import type { GroupToolPolicyConfig } from "../../config/types.tools.js"; import type { ImageContent } from "../../llm/types.js"; import type { MediaFact } from "../../media/media-facts.js"; import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js"; +import type { RuntimePluginToolGrant } from "../../plugins/runtime/tool-grant.js"; +import type { InputProvenance } from "../../sessions/input-provenance.js"; import type { UserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.types.js"; import type { SourceReplyDeliveryMode, TaskSuggestionDeliveryMode, } from "../get-reply-options.types.js"; +import type { OriginatingChannelType } from "../templating.js"; import type { ReplyFollowupAdmissionBarrierTimeoutPolicy } from "./reply-dispatcher.types.js"; import * as replyRunSettle from "./reply-run-finalization-lease.js"; @@ -42,11 +49,50 @@ export type ReplyBackendQueueMessageOptions = { userTurnTranscriptRecorder?: UserTurnTranscriptRecorder; }; -type ReplyToolAuthorityRoute = Readonly<{ +export type ReplyMessageInjectionOptions = ReplyBackendQueueMessageOptions & { + /** Consumed by reply ownership and never forwarded to the active backend. */ + toolAuthorityOverlay?: ReplyToolAuthorityOverlay; +}; + +export type ReplyToolAuthorityRoute = Readonly<{ provider: string; model: string; }>; +/** Per-message authority facts projected against an active run's frozen owner state. */ +export type ReplyToolAuthorityOverlay = Readonly<{ + originatingChannel?: OriginatingChannelType; + messageProvider?: string; + chatType?: ChatType; + agentAccountId?: string; + conversationToolPolicy?: GroupToolPolicyConfig; + groupId?: string; + groupChannel?: string; + groupSpace?: string; + memberRoleIds?: string[]; + spawnedBy?: string; + senderId?: string; + senderName?: string; + senderUsername?: string; + senderE164?: string; + senderIsOwner: boolean; + inputProvenance?: InputProvenance; + trustedInternalHandoff?: TrustedSubagentCompletionHandoff; + scheduledToolPolicy?: ScheduledToolPolicyContext; + runtimePluginToolGrant?: RuntimePluginToolGrant; + toolsAllow?: string[]; + disableTools: boolean; + traceAuthorized: boolean; + approvalReviewerDeviceId?: string; + clientCaps?: string[]; + toolBindings?: Readonly>; +}>; + +export type ReplyToolAuthorityProjector = ( + overlay: ReplyToolAuthorityOverlay, + route: ReplyToolAuthorityRoute, +) => string; + export type ReplyBackendQueueMessageResult = { /** Acceptance was irreversible, but the harness could not prove transcript commitment. */ transcriptCommit: "unconfirmed"; @@ -216,6 +262,10 @@ export type ReplyOperation = { markAcceptedSteeredInboundAudio(): void; /** Bind provisional request authority before a concrete backend attempt attaches. */ bindToolAuthorityFingerprint(fingerprint: string): void; + /** Bind the active run's immutable authority projector for direct inbound steering. */ + bindToolAuthorityProjector(projector: ReplyToolAuthorityProjector): void; + /** Project an inbound turn through the current concrete route; settled owners fail closed. */ + projectToolAuthorityFingerprint(overlay: ReplyToolAuthorityOverlay): string | undefined; /** Record the concrete candidate route; fallback attempts may replace it. */ bindToolAuthorityRoute(route: ReplyToolAuthorityRoute): void; updateSessionId(nextSessionId: string): void; diff --git a/src/auto-reply/reply/reply-run-registry.message-injection.ts b/src/auto-reply/reply/reply-run-registry.message-injection.ts index 0ff9ac0f9dc4..a2c6a70ace86 100644 --- a/src/auto-reply/reply/reply-run-registry.message-injection.ts +++ b/src/auto-reply/reply/reply-run-registry.message-injection.ts @@ -7,6 +7,7 @@ import { type ReplyBackendQueueMessageOptions, type ReplyBackendQueueMessageResult, type ReplyMessageInjectionAttempt, + type ReplyMessageInjectionOptions, type ReplyMessageInjectionOutcome, type ReplyMessageInjectionTarget, type ReplyOperation, @@ -141,22 +142,46 @@ export function resolveReplyMessageInjectionRejection(params: { return mismatch ? { reason: mismatch } : { backend, injection }; } +function isLeafOwnershipRejection(reason: ReplyMessageInjectionRejectionReason): boolean { + return ( + reason === "no_active_run" || + reason === "not_running" || + reason === "stale_run" || + reason === "leaf_mismatch" + ); +} + export function beginReplyMessageInjectionTarget( target: ReplyMessageInjectionTarget, text: string, - options?: ReplyBackendQueueMessageOptions, + options?: ReplyMessageInjectionOptions, ): ReplyMessageInjectionAttempt { + const operation = target[replyMessageInjectionTargetOperation]; + const { toolAuthorityOverlay, ...backendOptions } = options ?? {}; + const projectedToolAuthorityFingerprint = toolAuthorityOverlay + ? operation.projectToolAuthorityFingerprint(toolAuthorityOverlay) + : backendOptions.toolAuthorityFingerprint; + const queueOptions: ReplyBackendQueueMessageOptions | undefined = options + ? { + ...backendOptions, + ...(toolAuthorityOverlay + ? { toolAuthorityFingerprint: projectedToolAuthorityFingerprint } + : {}), + } + : undefined; const resolved = resolveReplyMessageInjectionRejection({ - operation: target[replyMessageInjectionTargetOperation], + operation, originatingLeafEntryId: target.originatingLeafEntryId, expectedRunId: target.identity === "run" ? target.runId : undefined, - options, + options: queueOptions, }); if (!("injection" in resolved)) { const immediateRejection = { status: "rejected" as const, ...resolved }; return { targetRunId: target.runId, - ...(target.identity === "leaf" ? { rejectBeforeAck: true as const } : {}), + ...(target.identity === "leaf" && isLeafOwnershipRejection(resolved.reason) + ? { rejectBeforeAck: true as const } + : {}), acceptance: Promise.resolve(false), outcome: Promise.resolve(immediateRejection), }; @@ -174,9 +199,9 @@ export function beginReplyMessageInjectionTarget( acceptanceSettled = true; acceptance.resolve(accepted); }; - const callerOnQueueAccepted = options?.onQueueAccepted; - const queueOptions: ReplyBackendQueueMessageOptions = { - ...options, + const callerOnQueueAccepted = queueOptions?.onQueueAccepted; + const runtimeQueueOptions: ReplyBackendQueueMessageOptions = { + ...queueOptions, onQueueAccepted: (accepted) => { settleAcceptance(accepted); callerOnQueueAccepted?.(accepted); @@ -184,7 +209,7 @@ export function beginReplyMessageInjectionTarget( }; let queued: Promise; try { - queued = resolved.injection.queueMessage(text, queueOptions); + queued = resolved.injection.queueMessage(text, runtimeQueueOptions); } catch (error) { settleAcceptance(false); const immediateRejection = { diff --git a/src/auto-reply/reply/reply-run-registry.operation.ts b/src/auto-reply/reply/reply-run-registry.operation.ts index 0c85bcc8f6e1..02e168ccb940 100644 --- a/src/auto-reply/reply/reply-run-registry.operation.ts +++ b/src/auto-reply/reply/reply-run-registry.operation.ts @@ -16,6 +16,7 @@ import { ReplyRunSuccessorAdmissionBlockedError, type ReplyOperation, type ReplyOperationPhase, + type ReplyToolAuthorityProjector, } from "./reply-run-registry.contracts.js"; import { abortFrozenOperations, @@ -95,6 +96,7 @@ export function createReplyOperation(params: { let terminalRecovery = false; let acceptedSteeredInboundAudio = false; let toolAuthorityFingerprint: string | undefined; + let toolAuthorityProjector: ReplyToolAuthorityProjector | undefined; let toolAuthorityRoute: { provider: string; model: string } | undefined; const ownerSettlement = createDeferredCore(); let ownerSettled = false; @@ -338,6 +340,22 @@ export function createReplyOperation(params: { } toolAuthorityFingerprint = normalized; }, + bindToolAuthorityProjector(projector) { + if (toolAuthorityProjector && toolAuthorityProjector !== projector) { + throw new Error("Reply operation cannot change tool authority projector after admission"); + } + toolAuthorityProjector = projector; + }, + projectToolAuthorityFingerprint(overlay) { + if (result || !toolAuthorityProjector || !toolAuthorityRoute) { + return undefined; + } + try { + return normalizeOptionalString(toolAuthorityProjector(overlay, toolAuthorityRoute)); + } catch { + return undefined; + } + }, bindToolAuthorityRoute(route) { const provider = normalizeOptionalString(route.provider); const model = normalizeOptionalString(route.model); diff --git a/src/auto-reply/reply/reply-run-registry.test.ts b/src/auto-reply/reply/reply-run-registry.test.ts index 6e20a6a2a5c6..eaf69bf60569 100644 --- a/src/auto-reply/reply/reply-run-registry.test.ts +++ b/src/auto-reply/reply/reply-run-registry.test.ts @@ -33,6 +33,7 @@ import { REPLY_RUN_TERMINAL_SETTLE_TIMEOUT_MS, registerReplyOperationSuccessorBarrier, type ReplyBackendQueueMessageOptions, + type ReplyToolAuthorityOverlay, ReplyRunAlreadyActiveError, ReplyRunSuccessorAdmissionBlockedError, replyRunRegistry, @@ -46,7 +47,10 @@ import { waitForReplyRunSuccessorAdmission, } from "./reply-run-registry.js"; import { testing } from "./reply-run-registry.test-support.js"; -import { resolveFollowupRunToolAuthorityFingerprint } from "./reply-tool-authority.js"; +import { + createFollowupRunToolAuthorityProjector, + resolveFollowupRunToolAuthorityFingerprint, +} from "./reply-tool-authority.js"; import { admitReplyTurn } from "./reply-turn-admission.js"; const REPLY_RUN_FINALIZATION_SETTLE_TIMEOUT_MS = 60_000; @@ -62,6 +66,38 @@ function createTestReplyOperation( }); } +function toolAuthorityOverlay( + run: ReturnType, +): ReplyToolAuthorityOverlay { + return { + originatingChannel: run.originatingChannel, + messageProvider: run.run.messageProvider, + chatType: run.run.chatType, + agentAccountId: run.run.agentAccountId, + conversationToolPolicy: run.run.conversationToolPolicy, + groupId: run.run.groupId, + groupChannel: run.run.groupChannel, + groupSpace: run.run.groupSpace, + memberRoleIds: run.run.memberRoleIds, + spawnedBy: run.run.spawnedBy, + senderId: run.run.senderId, + senderName: run.run.senderName, + senderUsername: run.run.senderUsername, + senderE164: run.run.senderE164, + senderIsOwner: run.run.senderIsOwner === true, + inputProvenance: run.run.inputProvenance, + trustedInternalHandoff: run.run.trustedInternalHandoff, + scheduledToolPolicy: run.run.scheduledToolPolicy, + runtimePluginToolGrant: run.run.runtimePluginToolGrant, + toolsAllow: run.toolsAllow, + disableTools: run.disableTools === true, + traceAuthorized: run.run.traceAuthorized === true, + approvalReviewerDeviceId: run.run.approvalReviewerDeviceId, + clientCaps: run.run.clientCaps, + toolBindings: run.run.toolBindings, + }; +} + async function queueCurrentReplyRunMessage( sessionId: string, text: string, @@ -126,6 +162,40 @@ describe("reply run registry", () => { ); }); + it("projects inbound authority through the canonical full fingerprint", () => { + const run = createQueueTestRun({ prompt: "projected authority" }); + run.toolsAllow = attachToolAllowlistIntersection(["exec"], [["exec"], ["message"]]); + const route = { provider: "openai", model: "gpt-fallback" }; + const projector = createFollowupRunToolAuthorityProjector(run); + const overlay = toolAuthorityOverlay(run); + + expect(projector(overlay, route)).toBe(resolveFollowupRunToolAuthorityFingerprint(run, route)); + expect(projector({ ...overlay, clientCaps: ["different-capability"] }, route)).not.toBe( + resolveFollowupRunToolAuthorityFingerprint(run, route), + ); + }); + + it("projects only while the active operation owns a concrete route", () => { + const run = createQueueTestRun({ prompt: "operation projection" }); + const operation = createTestReplyOperation({ sessionId: "session-projector" }); + const projector = createFollowupRunToolAuthorityProjector(run); + const overlay = toolAuthorityOverlay(run); + + operation.bindToolAuthorityProjector(projector); + expect(operation.projectToolAuthorityFingerprint(overlay)).toBeUndefined(); + + operation.bindToolAuthorityRoute({ provider: "openai", model: "gpt-primary" }); + expect(operation.projectToolAuthorityFingerprint(overlay)).toBe( + resolveFollowupRunToolAuthorityFingerprint(run, { + provider: "openai", + model: "gpt-primary", + }), + ); + + operation.complete(); + expect(operation.projectToolAuthorityFingerprint(overlay)).toBeUndefined(); + }); + it("tracks the concrete authority route across fallback candidates", () => { const operation = createTestReplyOperation({ sessionId: "session-route" }); @@ -1855,6 +1925,48 @@ describe("reply run registry", () => { ).resolves.toEqual({ status: "accepted" }); }); + it("projects inbound authority before backend admission without forwarding the overlay", async () => { + const run = createQueueTestRun({ prompt: "projected inbound" }); + const route = { provider: "openai", model: "gpt-primary" }; + const overlay = toolAuthorityOverlay(run); + const queueMessage = vi.fn( + async (_text: string, _options?: ReplyBackendQueueMessageOptions) => {}, + ); + const operation = createTestReplyOperation({ sessionId: "session-projected-authority" }); + operation.bindToolAuthorityProjector(createFollowupRunToolAuthorityProjector(run)); + operation.bindToolAuthorityRoute(route); + operation.bindToolAuthorityFingerprint(resolveFollowupRunToolAuthorityFingerprint(run, route)); + operation.attachBackend({ + kind: "embedded", + cancel: vi.fn(), + isStreaming: () => true, + queueMessage, + }); + operation.setPhase("running"); + + await expect( + queueCurrentReplyRunMessage("session-projected-authority", "same authority", { + isInboundUserMessage: true, + toolAuthorityFingerprint: "caller-cannot-override-projection", + toolAuthorityOverlay: overlay, + }), + ).resolves.toEqual({ status: "accepted" }); + const forwardedOptions = queueMessage.mock.calls[0]?.[1]; + expect(forwardedOptions).toMatchObject({ + isInboundUserMessage: true, + toolAuthorityFingerprint: resolveFollowupRunToolAuthorityFingerprint(run, route), + }); + expect(forwardedOptions).not.toHaveProperty("toolAuthorityOverlay"); + + await expect( + queueCurrentReplyRunMessage("session-projected-authority", "changed authority", { + isInboundUserMessage: true, + toolAuthorityOverlay: { ...overlay, clientCaps: ["changed-capability"] }, + }), + ).resolves.toMatchObject({ status: "rejected", reason: "tool_authority_mismatch" }); + expect(queueMessage).toHaveBeenCalledOnce(); + }); + it("refuses stale injectable owners for admission and delivery until activity resumes", async () => { vi.useFakeTimers(); try { diff --git a/src/auto-reply/reply/reply-run-registry.ts b/src/auto-reply/reply/reply-run-registry.ts index 88502ce621e3..f0f21cc548f6 100644 --- a/src/auto-reply/reply/reply-run-registry.ts +++ b/src/auto-reply/reply/reply-run-registry.ts @@ -15,6 +15,7 @@ export type { ReplyMessageInjectionTarget, ReplyOperation, ReplyOperationPhase, + ReplyToolAuthorityOverlay, } from "./reply-run-registry.contracts.js"; export { abortReplyMessageInjectionTarget, diff --git a/src/auto-reply/reply/reply-tool-authority.ts b/src/auto-reply/reply/reply-tool-authority.ts index 1cfd049d8c60..c3e47c849fd1 100644 --- a/src/auto-reply/reply/reply-tool-authority.ts +++ b/src/auto-reply/reply/reply-tool-authority.ts @@ -4,13 +4,81 @@ import { resolveConversationCapabilityProfile } from "../../agents/conversation- import { resolveSandboxRuntimeStatus } from "../../agents/sandbox/runtime-status.js"; import { readToolAllowlistIntersection } from "../../agents/tool-policy.js"; import type { FollowupRun } from "./queue.js"; +import type { + ReplyToolAuthorityOverlay, + ReplyToolAuthorityProjector, + ReplyToolAuthorityRoute, +} from "./reply-run-registry.contracts.js"; -/** Fingerprints the complete model-facing tool authority owned by one queued turn. */ -export function resolveFollowupRunToolAuthorityFingerprint( - run: FollowupRun, - route?: { provider: string; model: string }, +type ReplyToolAuthoritySnapshot = { + originatingChannel: FollowupRun["originatingChannel"]; + toolsAllow: FollowupRun["toolsAllow"]; + toolsAllowIntersection: readonly string[][] | undefined; + disableTools: boolean; + run: FollowupRun["run"]; +}; + +function snapshotFollowupRunToolAuthority(run: FollowupRun): ReplyToolAuthoritySnapshot { + return { + originatingChannel: run.originatingChannel, + toolsAllow: run.toolsAllow, + toolsAllowIntersection: run.toolsAllow + ? readToolAllowlistIntersection(run.toolsAllow) + : undefined, + disableTools: run.disableTools === true, + run: { + ...run.run, + clientCaps: run.run.clientCaps ? [...run.run.clientCaps] : undefined, + memberRoleIds: run.run.memberRoleIds ? [...run.run.memberRoleIds] : undefined, + }, + }; +} + +function applyReplyToolAuthorityOverlay( + snapshot: ReplyToolAuthoritySnapshot, + overlay: ReplyToolAuthorityOverlay, +): ReplyToolAuthoritySnapshot { + return { + ...snapshot, + originatingChannel: overlay.originatingChannel, + toolsAllow: overlay.toolsAllow, + toolsAllowIntersection: overlay.toolsAllow + ? readToolAllowlistIntersection(overlay.toolsAllow) + : undefined, + disableTools: overlay.disableTools, + run: { + ...snapshot.run, + messageProvider: overlay.messageProvider, + chatType: overlay.chatType, + agentAccountId: overlay.agentAccountId, + conversationToolPolicy: overlay.conversationToolPolicy, + groupId: overlay.groupId, + groupChannel: overlay.groupChannel, + groupSpace: overlay.groupSpace, + memberRoleIds: overlay.memberRoleIds, + spawnedBy: overlay.spawnedBy, + senderId: overlay.senderId, + senderName: overlay.senderName, + senderUsername: overlay.senderUsername, + senderE164: overlay.senderE164, + senderIsOwner: overlay.senderIsOwner, + inputProvenance: overlay.inputProvenance, + trustedInternalHandoff: overlay.trustedInternalHandoff, + scheduledToolPolicy: overlay.scheduledToolPolicy, + runtimePluginToolGrant: overlay.runtimePluginToolGrant, + traceAuthorized: overlay.traceAuthorized, + approvalReviewerDeviceId: overlay.approvalReviewerDeviceId, + clientCaps: overlay.clientCaps, + toolBindings: overlay.toolBindings, + }, + }; +} + +function resolveReplyToolAuthoritySnapshotFingerprint( + snapshot: ReplyToolAuthoritySnapshot, + route?: ReplyToolAuthorityRoute, ): string { - const execution = run.run; + const execution = snapshot.run; const provider = route?.provider ?? execution.provider; const model = route?.model ?? execution.model; const policySessionKey = execution.runtimePolicySessionKey ?? execution.sessionKey; @@ -30,7 +98,7 @@ export function resolveFollowupRunToolAuthorityFingerprint( modelProvider: provider, modelId: model, messageProvider: execution.messageProvider, - messageChannel: run.originatingChannel, + messageChannel: snapshot.originatingChannel, chatType: execution.chatType, conversationToolPolicy: execution.conversationToolPolicy, groupId: execution.groupId, @@ -57,11 +125,9 @@ export function resolveFollowupRunToolAuthorityFingerprint( provider, model, policy: capabilityProfile.policy, - toolsAllow: run.toolsAllow, - toolsAllowIntersection: run.toolsAllow - ? readToolAllowlistIntersection(run.toolsAllow) - : undefined, - disableTools: run.disableTools === true, + toolsAllow: snapshot.toolsAllow, + toolsAllowIntersection: snapshot.toolsAllowIntersection, + disableTools: snapshot.disableTools, sessionFile: execution.sessionFile, agentDir: execution.agentDir, workspaceDir: execution.workspaceDir, @@ -79,3 +145,23 @@ export function resolveFollowupRunToolAuthorityFingerprint( ) .digest("hex"); } + +/** Fingerprints the complete model-facing tool authority owned by one queued turn. */ +export function resolveFollowupRunToolAuthorityFingerprint( + run: FollowupRun, + route?: ReplyToolAuthorityRoute, +): string { + return resolveReplyToolAuthoritySnapshotFingerprint(snapshotFollowupRunToolAuthority(run), route); +} + +/** Projects a new inbound turn against one active run's frozen owner authority. */ +export function createFollowupRunToolAuthorityProjector( + run: FollowupRun, +): ReplyToolAuthorityProjector { + const snapshot = snapshotFollowupRunToolAuthority(run); + return (overlay, route) => + resolveReplyToolAuthoritySnapshotFingerprint( + applyReplyToolAuthorityOverlay(snapshot, overlay), + route, + ); +} diff --git a/src/auto-reply/reply/test-helpers.ts b/src/auto-reply/reply/test-helpers.ts index af9f17a07a52..1081c53a25ec 100644 --- a/src/auto-reply/reply/test-helpers.ts +++ b/src/auto-reply/reply/test-helpers.ts @@ -19,6 +19,9 @@ export function createMockReplyOperation( const updateSessionIdMock = vi.fn(); const sessionId = overrides.sessionId ?? "session"; let toolAuthorityFingerprint = overrides.toolAuthorityFingerprint; + let toolAuthorityProjector: + | Parameters[0] + | undefined; let toolAuthorityRoute: ReplyOperation["toolAuthorityRoute"]; const replyOperation: ReplyOperation = { key: overrides.key ?? "main", @@ -50,6 +53,14 @@ export function createMockReplyOperation( bindToolAuthorityFingerprint: vi.fn((fingerprint) => { toolAuthorityFingerprint = fingerprint; }), + bindToolAuthorityProjector: vi.fn((projector) => { + toolAuthorityProjector = projector; + }), + projectToolAuthorityFingerprint: vi.fn((overlay) => + toolAuthorityProjector && toolAuthorityRoute + ? toolAuthorityProjector(overlay, toolAuthorityRoute) + : undefined, + ), bindToolAuthorityRoute: vi.fn((route) => { toolAuthorityRoute = route; }), diff --git a/src/gateway/server-methods/chat-send-message-injection.ts b/src/gateway/server-methods/chat-send-message-injection.ts index 764e43b65bf3..46e2f6fc6388 100644 --- a/src/gateway/server-methods/chat-send-message-injection.ts +++ b/src/gateway/server-methods/chat-send-message-injection.ts @@ -1,8 +1,13 @@ -import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; +import { + normalizeLowercaseStringOrEmpty, + normalizeOptionalString, +} from "@openclaw/normalization-core/string-coerce"; +import { resolveCommandAuthorization } from "../../auto-reply/command-auth.js"; import { emitInboundMessageAuditTerminal } from "../../auto-reply/reply/dispatch-from-config.audit.js"; import { finalizeInboundContext } from "../../auto-reply/reply/inbound-context.js"; import { hasInboundAudio } from "../../auto-reply/reply/inbound-media.js"; import { emitMessageReceivedHooks } from "../../auto-reply/reply/message-received-hooks.js"; +import { resolveOriginMessageProvider } from "../../auto-reply/reply/origin-routing.js"; import { resolveQueueSettings } from "../../auto-reply/reply/queue/settings-runtime.js"; import { abortReplyMessageInjectionTarget, @@ -12,8 +17,11 @@ import { type ReplyMessageInjectionAttempt, type ReplyMessageInjectionOutcome, type ReplyMessageInjectionTarget, + type ReplyToolAuthorityOverlay, } from "../../auto-reply/reply/reply-run-registry.js"; import type { RuntimeMsgContext } from "../../auto-reply/templating.js"; +import { normalizeChatType } from "../../channels/chat-type.js"; +import { resolveGroupSessionKey } from "../../config/sessions/group.js"; import { updateSessionEntry } from "../../config/sessions/session-accessor.js"; import { isDiagnosticsEnabled } from "../../infra/diagnostic-events.js"; import { logMessageProcessed, logMessageReceived } from "../../logging/diagnostic.js"; @@ -26,23 +34,68 @@ import type { PreparedChatSendSession } from "./chat-send-session.js"; import type { prepareChatSendUserTurn } from "./chat-send-user-turn.js"; import type { GatewayRequestContext } from "./types.js"; +function resolveChatSendToolAuthorityOverlay(params: { + ctx: RuntimeMsgContext; + session: Pick; +}): ReplyToolAuthorityOverlay { + const { ctx, session } = params; + const authorization = resolveCommandAuthorization({ + ctx, + cfg: session.cfg, + commandAuthorized: ctx.CommandAuthorized === true, + }); + const senderIsOwner = authorization.senderIsOwner; + return { + originatingChannel: ctx.OriginatingChannel, + messageProvider: resolveOriginMessageProvider({ + originatingChannel: ctx.OriginatingChannel, + provider: ctx.Provider, + }), + chatType: normalizeChatType(ctx.ChatType), + agentAccountId: ctx.AccountId, + conversationToolPolicy: ctx.ConversationToolPolicy, + groupId: resolveGroupSessionKey(ctx)?.id, + groupChannel: + normalizeOptionalString(ctx.GroupChannel) ?? normalizeOptionalString(ctx.GroupSubject), + groupSpace: normalizeOptionalString(ctx.GroupSpace), + memberRoleIds: Array.isArray(ctx.MemberRoleIds) + ? ctx.MemberRoleIds.map((roleId) => normalizeOptionalString(roleId)).filter( + (roleId): roleId is string => Boolean(roleId), + ) + : undefined, + spawnedBy: session.entry?.spawnedBy, + senderId: normalizeOptionalString(ctx.SenderId), + senderName: normalizeOptionalString(ctx.SenderName), + senderUsername: normalizeOptionalString(ctx.SenderUsername), + senderE164: normalizeOptionalString(ctx.SenderE164), + senderIsOwner, + inputProvenance: ctx.InputProvenance, + trustedInternalHandoff: undefined, + scheduledToolPolicy: undefined, + runtimePluginToolGrant: undefined, + toolsAllow: undefined, + disableTools: false, + traceAuthorized: senderIsOwner || (ctx.GatewayClientScopes ?? []).includes("operator.admin"), + approvalReviewerDeviceId: normalizeOptionalString(ctx.ApprovalReviewerDeviceId), + clientCaps: ctx.GatewayClientCaps, + toolBindings: ctx.GatewayRunToolBindings, + }; +} + /** Captures the prepared request data used by both pre-ACK and detached injection attempts. */ export function createChatSendMessageInjectionStarter(params: { target: ReplyMessageInjectionTarget | undefined; - request: Pick< - NormalizedChatSendRequest, - "p" | "rawMessage" | "supportsTaskSuggestions" | "toolBindings" - >; + request: Pick; session: Pick; turn: ReturnType; imageOrder: ReplyBackendQueueMessageOptions["imageOrder"]; userTurnTranscriptRecorder: ReplyBackendQueueMessageOptions["userTurnTranscriptRecorder"]; }) { - const { p, rawMessage, supportsTaskSuggestions, toolBindings } = params.request; + const { p, rawMessage, supportsTaskSuggestions } = params.request; const { cfg, entry } = params.session; const { ctx, isInternalTextSlashCommandTurn, replyOptionImages, replyOptionMedia } = params.turn; return (): ReplyMessageInjectionAttempt | undefined => { - if (!params.target || isInternalTextSlashCommandTurn || toolBindings !== undefined) { + if (!params.target || isInternalTextSlashCommandTurn) { return undefined; } const { debounceMs } = resolveQueueSettings({ @@ -60,11 +113,7 @@ export function createChatSendMessageInjectionStarter(params: { { steeringMode: "all", isInboundUserMessage: true, - // chat.send cannot alter per-turn tool policy. Preserve the captured - // active authority only when no request-scoped tool bindings are present. - ...(toolBindings === undefined && params.target.toolAuthorityFingerprint - ? { toolAuthorityFingerprint: params.target.toolAuthorityFingerprint } - : {}), + toolAuthorityOverlay: resolveChatSendToolAuthorityOverlay({ ctx, session: params.session }), ...(replyOptionImages?.length ? { images: replyOptionImages } : {}), ...(params.imageOrder?.length ? { imageOrder: params.imageOrder } : {}), ...(replyOptionMedia?.length ? { media: replyOptionMedia } : {}), diff --git a/src/gateway/server-methods/chat.directive-tags.test.ts b/src/gateway/server-methods/chat.directive-tags.test.ts index 1d810a736589..2cab1dc6d23f 100644 --- a/src/gateway/server-methods/chat.directive-tags.test.ts +++ b/src/gateway/server-methods/chat.directive-tags.test.ts @@ -28,6 +28,7 @@ import { markInboundContextLabel } from "../../auto-reply/reply/inbound-context- import { replyRunRegistry as baseReplyRunRegistry, type ReplyBackendQueueMessageOptions, + type ReplyOperation, } from "../../auto-reply/reply/reply-run-registry.js"; import { testing as replyRunRegistryTesting } from "../../auto-reply/reply/reply-run-registry.test-support.js"; import type { MsgContext } from "../../auto-reply/templating.js"; @@ -1136,6 +1137,11 @@ function managedAudioBlocks(content: Array>) { return content.filter((block) => block.type === "audio"); } +function bindTestToolAuthority(operation: ReplyOperation) { + operation.bindToolAuthorityProjector(() => TEST_TOOL_AUTHORITY_FINGERPRINT); + operation.bindToolAuthorityRoute({ provider: "anthropic", model: "test-model" }); +} + function expectManagedAudioBlock( block: Record | undefined, fileName: string, @@ -1481,6 +1487,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => sessionId: mockState.sessionId, resetTriggered: false, }); + bindTestToolAuthority(operation); operation.setPhase("running"); operation.attachBackend({ kind: "embedded", @@ -1524,6 +1531,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: "current-leaf", }); + bindTestToolAuthority(operation); operation.setPhase("running"); operation.attachBackend({ kind: "embedded", @@ -1634,6 +1642,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: "different-owner-leaf", }); + bindTestToolAuthority(operation); operation.setPhase("running"); operation.attachBackend({ kind: "embedded", @@ -1678,6 +1687,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: "current-leaf", }); + bindTestToolAuthority(original); original.setPhase("running"); original.attachBackend({ kind: "embedded", @@ -1720,6 +1730,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: "current-leaf", }); + bindTestToolAuthority(successor); successor.setPhase("running"); successor.attachBackend({ kind: "embedded", @@ -1760,6 +1771,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: "leaf-before-active-run-output", }); + bindTestToolAuthority(operation); operation.setPhase("running"); const queueMessage = vi.fn(async () => {}); operation.attachBackend({ @@ -1811,6 +1823,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: null, }); + bindTestToolAuthority(operation); operation.setPhase("running"); let reportAcceptance: ((accepted: boolean) => void) | undefined; const queueMessage = vi.fn((_text: string, options?: ReplyBackendQueueMessageOptions) => { @@ -1867,6 +1880,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: null, }); + bindTestToolAuthority(operation); operation.setPhase("running"); const queueMessage = vi.fn(async (_text: string, options?: ReplyBackendQueueMessageOptions) => { expect(dispatchInboundMessageMock).toHaveBeenCalledTimes(dispatchCallsBefore); @@ -1954,6 +1968,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: "current-leaf", }); + bindTestToolAuthority(operation); operation.setPhase("running"); operation.attachBackend({ kind: "embedded", @@ -2041,6 +2056,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: "current-leaf", }); + bindTestToolAuthority(original); original.setPhase("running"); original.attachBackend({ kind: "embedded", @@ -2070,6 +2086,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: "current-leaf", }); + bindTestToolAuthority(successor); successor.setPhase("running"); successor.attachBackend({ kind: "embedded", @@ -2166,6 +2183,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: null, }); + bindTestToolAuthority(operation); operation.setPhase("running"); const queueMessage = vi.fn((_text: string, options?: ReplyBackendQueueMessageOptions) => { options?.onQueueAccepted?.(false); @@ -2216,6 +2234,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: null, }); + bindTestToolAuthority(first); first.setPhase("running"); const queueMessage = vi.fn((_text: string, options?: ReplyBackendQueueMessageOptions) => { options?.onQueueAccepted?.(true); @@ -2241,6 +2260,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: null, }); + bindTestToolAuthority(successor); successor.setPhase("running"); successor.attachBackend({ kind: "embedded", @@ -2275,6 +2295,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: null, }); + bindTestToolAuthority(operation); operation.setPhase("running"); const queueMessage = vi.fn((): Promise => { expect(respond).not.toHaveBeenCalled(); @@ -2316,6 +2337,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: "leaf-before-active-run-output", }); + bindTestToolAuthority(operation); operation.setPhase("running"); const successorQueue = vi.fn(async () => {}); const successorCancel = vi.fn(); @@ -2380,6 +2402,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () => resetTriggered: false, originatingLeafEntryId: "leaf-before-stale-run-output", }); + bindTestToolAuthority(operation); operation.setPhase("running"); const staleQueue = vi.fn(async () => {}); const staleCancel = vi.fn(); diff --git a/src/gateway/worker-environments/node-launch-adapter.test.ts b/src/gateway/worker-environments/node-launch-adapter.test.ts index 3e53dee841b0..9b559ffb0496 100644 --- a/src/gateway/worker-environments/node-launch-adapter.test.ts +++ b/src/gateway/worker-environments/node-launch-adapter.test.ts @@ -132,6 +132,30 @@ function launchRequest(input = launchInput()) { } describe("node worker launch adapter", () => { + it("fails with a typed availability result when no node dispatches within the grace", async () => { + vi.useFakeTimers(); + const onDispatchReady = vi.fn(); + const adapter = createNodeWorkerLaunchAdapter({ + getTransport: () => transportWith(vi.fn(), async () => []), + availabilityTimeoutMs: 100, + pollIntervalMs: 10, + }); + try { + const launch = adapter + .launch({ ...launchRequest(), timeoutMs: 1_000, onDispatchReady }) + .catch((error: unknown) => error); + await vi.advanceTimersByTimeAsync(100); + + expect(await launch).toMatchObject({ + name: "WorkerRunnerUnavailableError", + code: "runner-offline", + }); + expect(onDispatchReady).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + it("launches once, polls status, and returns the exact completed receipt", async () => { const input = launchInput(); const invoke = vi.fn(async (request) => diff --git a/src/gateway/worker-environments/node-launch-adapter.ts b/src/gateway/worker-environments/node-launch-adapter.ts index cb4f1008201a..98b267f3cd3d 100644 --- a/src/gateway/worker-environments/node-launch-adapter.ts +++ b/src/gateway/worker-environments/node-launch-adapter.ts @@ -18,11 +18,13 @@ import type { NodeWorkerSupervisorNodeProof, NodeWorkerSupervisorTransport, } from "../node-registry-private.js"; +import { WorkerRunnerUnavailableError } from "./tunnel-contract.js"; const DEFAULT_RPC_TIMEOUT_MS = 30_000; const DEFAULT_POLL_INTERVAL_MS = 250; const MAX_RETRY_DELAY_MS = 2_000; const DEFAULT_CANCELLATION_TIMEOUT_MS = 30_000; +const DEFAULT_AVAILABILITY_TIMEOUT_MS = 10_000; const RETRYABLE_TRANSPORT_CODES = new Set([ "DISCONNECTED", @@ -46,6 +48,7 @@ type DeviceWorkerLaunchRequest = { isCancellationAuthorized: () => boolean; timeoutMs: number; signal?: AbortSignal; + onDispatchReady?: () => void; }; type NodeWorkerLaunchAdapterOptions = { @@ -55,6 +58,7 @@ type NodeWorkerLaunchAdapterOptions = { rpcTimeoutMs?: number; pollIntervalMs?: number; cancellationTimeoutMs?: number; + availabilityTimeoutMs?: number; }; type OperationDeadline = { @@ -203,6 +207,7 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp const rpcTimeoutMs = options.rpcTimeoutMs ?? DEFAULT_RPC_TIMEOUT_MS; const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; const cancellationTimeoutMs = options.cancellationTimeoutMs ?? DEFAULT_CANCELLATION_TIMEOUT_MS; + const availabilityTimeoutMs = options.availabilityTimeoutMs ?? DEFAULT_AVAILABILITY_TIMEOUT_MS; const findNode = async (params: { transport: NodeWorkerSupervisorTransport; @@ -409,18 +414,36 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp ...(request.signal ? { signal: request.signal } : {}), label: "node worker launch", }); + const availabilityDeadline = createDeadline({ + now, + timeoutMs: availabilityTimeoutMs, + signal: deadline.signal, + label: "node worker availability", + }); let mayHaveLaunched = false; + let dispatchReady = false; let pollStatus = false; let delayMs = pollIntervalMs; + const markDispatchReady = () => { + mayHaveLaunched = true; + if (!dispatchReady) { + dispatchReady = true; + stableRequest.onDispatchReady?.(); + } + }; try { while (true) { if (deadline.signal.aborted) { throw signalError(deadline.signal, "node worker launch aborted"); } + if (!dispatchReady && availabilityDeadline.signal.aborted) { + throw new WorkerRunnerUnavailableError(); + } if (!stableRequest.isDispatchAuthorized()) { throw new Error("node worker launch authority closed"); } try { + const attemptDeadline = dispatchReady ? deadline : availabilityDeadline; const receipt = await invoke({ deviceId: stableRequest.deviceId, command: pollStatus @@ -429,18 +452,19 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp payload: pollStatus ? { launchId: input.launchId } : input, ...(!pollStatus ? { expectedWorkerRuns: input.descriptor.admission.handshake } : {}), isAuthorized: stableRequest.isDispatchAuthorized, - deadline, + deadline: attemptDeadline, ...(!pollStatus ? { - onDispatchReady: () => { - mayHaveLaunched = true; - }, + onDispatchReady: markDispatchReady, } : {}), }); if (!receipt) { pollStatus = false; } else { + if (!pollStatus) { + markDispatchReady(); + } const validated = validateReceipt(receipt, expected); mayHaveLaunched = true; if (isTerminalReceipt(validated)) { @@ -453,6 +477,9 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp if (deadline.signal.aborted || !stableRequest.isDispatchAuthorized()) { throw error; } + if (!dispatchReady && availabilityDeadline.signal.aborted) { + throw new WorkerRunnerUnavailableError(); + } if ( !(error instanceof NodeWorkerLaunchTransportError) || !RETRYABLE_TRANSPORT_CODES.has(error.code) @@ -461,9 +488,15 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp } pollStatus = false; } - delayMs = await waitBeforeRetry({ delayMs, deadline }); + delayMs = await waitBeforeRetry({ + delayMs, + deadline: dispatchReady ? deadline : availabilityDeadline, + }); } } catch (error) { + if (!dispatchReady && availabilityDeadline.signal.aborted && !deadline.signal.aborted) { + throw new WorkerRunnerUnavailableError(); + } if (!mayHaveLaunched) { throw error; } @@ -483,6 +516,7 @@ export function createNodeWorkerLaunchAdapter(options: NodeWorkerLaunchAdapterOp } throw error; } finally { + availabilityDeadline.dispose(); deadline.dispose(); } }; diff --git a/src/gateway/worker-environments/node-worker-tunnel.test.ts b/src/gateway/worker-environments/node-worker-tunnel.test.ts index 64c6c9c264ed..44c26b2b8985 100644 --- a/src/gateway/worker-environments/node-worker-tunnel.test.ts +++ b/src/gateway/worker-environments/node-worker-tunnel.test.ts @@ -346,8 +346,10 @@ describe("node worker tunnel manager", () => { it("keeps cancellation authorized until an active launch settles", async () => { const record = environment(); let cancellationWasAuthorized = false; - const launch: NodeWorkerLaunch = async (request): Promise => - await new Promise((resolve) => { + const onDispatchReady = vi.fn(); + const launch: NodeWorkerLaunch = async (request): Promise => { + request.onDispatchReady?.(); + return await new Promise((resolve) => { request.signal?.addEventListener( "abort", () => { @@ -367,6 +369,7 @@ describe("node worker tunnel manager", () => { { once: true }, ); }); + }; const launchNodeWorker = vi.fn(launch); const manager = createNodeWorkerTunnelManager({ gatewayDeviceId: "gateway-device-1", @@ -377,8 +380,14 @@ describe("node worker tunnel manager", () => { workspaceTransfer: workspaceTransfer(), }); const handle = await manager.start(startRequest()); - const launched = handle.launchTurn({ plan: plan(), placementGeneration: 4, timeoutMs: 5_000 }); + const launched = handle.launchTurn({ + plan: plan(), + placementGeneration: 4, + timeoutMs: 5_000, + onDispatchReady, + }); await vi.waitFor(() => expect(launchNodeWorker).toHaveBeenCalledOnce()); + expect(onDispatchReady).toHaveBeenCalledOnce(); await handle.stop(); diff --git a/src/gateway/worker-environments/node-worker-tunnel.ts b/src/gateway/worker-environments/node-worker-tunnel.ts index da340441cc89..9bb9d7f77ada 100644 --- a/src/gateway/worker-environments/node-worker-tunnel.ts +++ b/src/gateway/worker-environments/node-worker-tunnel.ts @@ -71,6 +71,7 @@ type NodeWorkerLaunch = (request: { isCancellationAuthorized: () => boolean; timeoutMs: number; signal?: AbortSignal; + onDispatchReady?: () => void; }) => Promise; type NodeWorkerWorkspaceBinding = { @@ -545,6 +546,7 @@ export function createNodeWorkerTunnelManager(options: NodeWorkerTunnelManagerOp isDispatchAuthorized, isCancellationAuthorized: () => hasDurableBinding(entry as NodeTunnelEntry), timeoutMs: request.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS, + onDispatchReady: request.onDispatchReady, signal: request.signal ? AbortSignal.any([entry.abortController.signal, request.signal]) : entry.abortController.signal, diff --git a/src/gateway/worker-environments/placement-dispatch-device.test.ts b/src/gateway/worker-environments/placement-dispatch-device.test.ts index c978c2144e35..262d7943ff6e 100644 --- a/src/gateway/worker-environments/placement-dispatch-device.test.ts +++ b/src/gateway/worker-environments/placement-dispatch-device.test.ts @@ -111,4 +111,23 @@ describe("device worker placement dispatch", () => { terminalAtMs: 1_000, }); }); + + it("adopts an offline paired-device placement without eagerly starting its tunnel", async () => { + const harness = createHarness(placementStore); + await harness.environments.attachSession({ + environmentId: harness.ready.environmentId, + ownerEpoch: harness.ready.ownerEpoch, + sessionId: REQUEST.sessionId, + }); + harness.placements.seedActive(harness.attached.ownerEpoch); + harness.markEnvironmentProviderId("device"); + harness.log.length = 0; + + await harness.service.reconcile(); + + expect(harness.log).toEqual(["environment:reconcile", "workspace", "placement:adopted"]); + expect(harness.placements.current()).toMatchObject({ state: "active" }); + expect(harness.environments.startTunnel).not.toHaveBeenCalled(); + expect(harness.environments.destroy).not.toHaveBeenCalled(); + }); }); diff --git a/src/gateway/worker-environments/placement-dispatch-recovery.ts b/src/gateway/worker-environments/placement-dispatch-recovery.ts index 1d5e3573c9cf..3e5e65176674 100644 --- a/src/gateway/worker-environments/placement-dispatch-recovery.ts +++ b/src/gateway/worker-environments/placement-dispatch-recovery.ts @@ -1,4 +1,5 @@ import { supportsWorkerExecutionContextLaunch } from "./admission.js"; +import { DEVICE_WORKER_PROVIDER_ID } from "./device-provider.js"; import { isUnavailableEnvironment, type WorkerActiveDispatchPlacement, @@ -122,10 +123,15 @@ export function createPlacementRecoveryActions(deps: PlacementRecoveryDeps) { return; } try { - await environments.startTunnel({ - environmentId: environment.environmentId, - ownerEpoch: environment.ownerEpoch, - }); + // Paired nodes are persistent runners, not one-shot SSH children. Their + // dormant lease remains authoritative while offline; validate and create + // the reconnect-scoped tunnel lazily when the next turn actually launches. + if (environment.providerId !== DEVICE_WORKER_PROVIDER_ID) { + await environments.startTunnel({ + environmentId: environment.environmentId, + ownerEpoch: environment.ownerEpoch, + }); + } placements.adoptActive({ sessionId: placement.sessionId, expectedGeneration: placement.generation, diff --git a/src/gateway/worker-environments/placement-dispatch-test-harness.ts b/src/gateway/worker-environments/placement-dispatch-test-harness.ts index 4c9890a38daa..2a6c4741d0b7 100644 --- a/src/gateway/worker-environments/placement-dispatch-test-harness.ts +++ b/src/gateway/worker-environments/placement-dispatch-test-harness.ts @@ -405,6 +405,9 @@ export function createHarness( markEnvironmentOwnerEpoch: (ownerEpoch: number) => { currentEnvironment = { ...attached, ownerEpoch }; }, + markEnvironmentProviderId: (providerId: string) => { + currentEnvironment = { ...attached, providerId }; + }, markEnvironmentAttachments: (attachedSessionIds: string[]) => { currentEnvironment = { ...attached, attachedSessionIds }; }, diff --git a/src/gateway/worker-environments/tunnel-contract.ts b/src/gateway/worker-environments/tunnel-contract.ts index c13146e352bc..617a3dccbda1 100644 --- a/src/gateway/worker-environments/tunnel-contract.ts +++ b/src/gateway/worker-environments/tunnel-contract.ts @@ -15,6 +15,17 @@ export class WorkerTunnelOwnerDisconnectedError extends Error { } } +export class WorkerRunnerUnavailableError extends Error { + readonly code = "runner-offline"; + + constructor() { + super( + "The device runner is offline. Reconnect it, retry later, or bring the session back to this gateway.", + ); + this.name = "WorkerRunnerUnavailableError"; + } +} + export type WorkerTunnelRequest = { environmentId: string; ownerEpoch: number; @@ -23,6 +34,7 @@ export type WorkerTunnelRequest = { export type WorkerWorkspaceCommand = { argv: readonly string[]; transportRetry: "idempotent" | "never"; + onDispatchReady?: () => void; input?: string; timeoutMs?: number; signal?: AbortSignal; @@ -80,6 +92,7 @@ type WorkerTurnLaunchRequest = { placementGeneration: number; timeoutMs?: number; signal?: AbortSignal; + onDispatchReady?: () => void; }; export type WorkerTunnelHandle = { diff --git a/src/gateway/worker-environments/tunnel.test.ts b/src/gateway/worker-environments/tunnel.test.ts index 4776ea7edbae..4cecc427d6a9 100644 --- a/src/gateway/worker-environments/tunnel.test.ts +++ b/src/gateway/worker-environments/tunnel.test.ts @@ -104,9 +104,11 @@ describe("worker tunnel manager", () => { toolAuthority: { allowedToolNames: [] }, }, }); + const onDispatchReady = vi.fn(); await expect( - handle.launchTurn({ plan, placementGeneration: 1, timeoutMs: 123 }), + handle.launchTurn({ plan, placementGeneration: 1, timeoutMs: 123, onDispatchReady }), ).resolves.toEqual(success()); + expect(onDispatchReady).toHaveBeenCalledOnce(); const launch = fake.runs.at(-1); const remoteLaunchCommand = launch?.argv.at(-1) ?? ""; expect(remoteLaunchCommand).toContain("'sh' '-c'"); diff --git a/src/gateway/worker-environments/tunnel.ts b/src/gateway/worker-environments/tunnel.ts index 60f5346da74f..eac4cec22a91 100644 --- a/src/gateway/worker-environments/tunnel.ts +++ b/src/gateway/worker-environments/tunnel.ts @@ -258,6 +258,7 @@ export function createWorkerTunnelManager(options: WorkerTunnelManagerOptions = ), timeoutMs: request.timeoutMs, signal: request.signal, + onDispatchReady: request.onDispatchReady, }), ...workspace, stop: () => stop(entry.environmentId, entry.ownerEpoch), diff --git a/src/gateway/worker-environments/worker-turn-launcher-claim-admission.test.ts b/src/gateway/worker-environments/worker-turn-launcher-claim-admission.test.ts index 35e3ebd188bf..544c3fc1be6f 100644 --- a/src/gateway/worker-environments/worker-turn-launcher-claim-admission.test.ts +++ b/src/gateway/worker-environments/worker-turn-launcher-claim-admission.test.ts @@ -4,6 +4,7 @@ import { makeAgentAssistantMessage } from "../../agents/test-helpers/agent-messa import type { SpawnResult } from "../../process/exec.js"; import { completeWorkerLaunchDescriptor } from "../../worker/launch-descriptor.js"; import { createWorkerSessionPlacementGate } from "./placement-worker-gate.js"; +import type { WorkerTunnelHandle } from "./tunnel-contract.js"; import { ENVIRONMENT_ID, MANIFEST_REF, @@ -183,7 +184,8 @@ describe("worker turn launcher claim admission", () => { killed: false; termination: "exit"; }>(); - const launchTurn = vi.fn(() => { + const launchTurn = vi.fn((request: Parameters[0]) => { + request.onDispatchReady?.(); commandStarted.resolve(); return commandFinished.promise; }); @@ -308,6 +310,7 @@ describe("worker turn launcher claim admission", () => { })), runWorkspaceCommand: vi.fn(), launchTurn: vi.fn(async (request): Promise => { + request.onDispatchReady?.(); launchCount += 1; const descriptor = completeWorkerLaunchDescriptor(structuredClone(request.plan), { kind: "unix", diff --git a/src/gateway/worker-environments/worker-turn-launcher-failure-recovery.test.ts b/src/gateway/worker-environments/worker-turn-launcher-failure-recovery.test.ts index ce054fa2add9..abc14ed0eba5 100644 --- a/src/gateway/worker-environments/worker-turn-launcher-failure-recovery.test.ts +++ b/src/gateway/worker-environments/worker-turn-launcher-failure-recovery.test.ts @@ -9,7 +9,7 @@ import type { SpawnResult } from "../../process/exec.js"; import { WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE } from "../../worker/transcript-message.js"; import type { WorkerSessionPlacementStore } from "./placement-store.js"; import { createWorkerSessionPlacementGate } from "./placement-worker-gate.js"; -import type { WorkerTunnelHandle } from "./tunnel-contract.js"; +import { WorkerRunnerUnavailableError, type WorkerTunnelHandle } from "./tunnel-contract.js"; import { ENVIRONMENT_ID, MANIFEST_REF, @@ -159,23 +159,26 @@ describe("worker turn launcher failure recovery", () => { it("preserves a terminal workspace result when the worker child later exits nonzero", async () => { seedActivePlacement(); const destroy = vi.fn(async () => attachedEnvironment()); - const launchTurn = vi.fn(async (): Promise => { - createWorkerSessionPlacementGate(placements).updateAckCursors({ - sessionId: SESSION_ID, - environmentId: ENVIRONMENT_ID, - ownerEpoch: OWNER_EPOCH, - runId: "run-terminal-child-failure", - liveSeq: 1, - }); - return { - stdout: "", - stderr: "child cleanup failed", - code: 1, - signal: null, - killed: false, - termination: "exit", - }; - }); + const launchTurn = vi.fn( + async (request: Parameters[0]): Promise => { + request.onDispatchReady?.(); + createWorkerSessionPlacementGate(placements).updateAckCursors({ + sessionId: SESSION_ID, + environmentId: ENVIRONMENT_ID, + ownerEpoch: OWNER_EPOCH, + runId: "run-terminal-child-failure", + liveSeq: 1, + }); + return { + stdout: "", + stderr: "child cleanup failed", + code: 1, + signal: null, + killed: false, + termination: "exit", + }; + }, + ); const environments: WorkerTurnEnvironmentService = { get: vi.fn(() => attachedEnvironment()), acquireTurnCredential: vi.fn(async () => credential()), @@ -306,7 +309,7 @@ describe("worker turn launcher failure recovery", () => { expect(environments.destroy).not.toHaveBeenCalled(); }); - it("fails placement and tears down after an ambiguous remote launch failure", async () => { + it("keeps the placement active when launch fails before transport dispatch", async () => { seedActivePlacement(); const teardownStates: string[] = []; const observedPlacements: WorkerSessionPlacementStore = { @@ -328,10 +331,11 @@ describe("worker turn launcher failure recovery", () => { teardownStates.push(`destroy:${placements.get(SESSION_ID)?.state ?? "missing"}`); return attachedEnvironment(); }); + const acknowledgeCredentialDelivery = vi.fn(() => true); const environments: WorkerTurnEnvironmentService = { get: vi.fn(() => attachedEnvironment()), acquireTurnCredential: vi.fn(async () => credential()), - acknowledgeCredentialDelivery: vi.fn(() => true), + acknowledgeCredentialDelivery, startTunnel: vi.fn(async () => ({ environmentId: ENVIRONMENT_ID, ownerEpoch: OWNER_EPOCH, @@ -341,7 +345,7 @@ describe("worker turn launcher failure recovery", () => { })), runWorkspaceCommand: vi.fn(), launchTurn: vi.fn(async () => { - throw new Error("remote launch failed"); + throw new WorkerRunnerUnavailableError(); }), syncWorkspace: vi.fn(async () => { throw new Error("unexpected workspace sync"); @@ -377,21 +381,13 @@ describe("worker turn launcher failure recovery", () => { turn("run-failed"), runLocal, ), - ).rejects.toThrow("remote launch failed"); + ).rejects.toThrow("The device runner is offline"); expect(runLocal).not.toHaveBeenCalled(); - expect(placements.get(SESSION_ID)).toMatchObject({ - state: "failed", - turnClaim: null, - recoveryError: "remote launch failed", - }); - expect(stopTunnel).toHaveBeenCalledWith(ENVIRONMENT_ID, OWNER_EPOCH); - expect(destroy).toHaveBeenCalledWith(ENVIRONMENT_ID); - expect(teardownStates).toEqual([ - "stop:draining", - "destroy:draining", - "reconcile-before:draining", - "reconcile-after:reconciling", - ]); + expect(placements.get(SESSION_ID)).toMatchObject({ state: "active", turnClaim: null }); + expect(acknowledgeCredentialDelivery).not.toHaveBeenCalled(); + expect(stopTunnel).not.toHaveBeenCalled(); + expect(destroy).not.toHaveBeenCalled(); + expect(teardownStates).toEqual([]); }); it("keeps redacted process failure details on a valid UTF-16 boundary", async () => { @@ -412,16 +408,17 @@ describe("worker turn launcher failure recovery", () => { environmentId: ENVIRONMENT_ID, ownerEpoch: OWNER_EPOCH, runWorkspaceCommand: vi.fn(), - launchTurn: vi.fn( - async (): Promise => ({ + launchTurn: vi.fn(async (request): Promise => { + request.onDispatchReady?.(); + return { stdout: "", stderr, code: 1, signal: null, killed: false, termination: "exit", - }), - ), + }; + }), syncWorkspace: vi.fn(async () => { throw new Error("unexpected workspace sync"); }), diff --git a/src/gateway/worker-environments/worker-turn-launcher-reclaimed-placement.test.ts b/src/gateway/worker-environments/worker-turn-launcher-reclaimed-placement.test.ts index 38cb3681e36c..83dd688f595c 100644 --- a/src/gateway/worker-environments/worker-turn-launcher-reclaimed-placement.test.ts +++ b/src/gateway/worker-environments/worker-turn-launcher-reclaimed-placement.test.ts @@ -21,6 +21,7 @@ import { import { getCommandLaneSnapshot, setCommandLaneConcurrency } from "../../process/command-queue.js"; import type { SpawnResult } from "../../process/exec.js"; import { createWorkerSessionPlacementGate } from "./placement-worker-gate.js"; +import type { WorkerTunnelHandle } from "./tunnel-contract.js"; import { ENVIRONMENT_ID, MANIFEST_REF, @@ -81,41 +82,44 @@ describe("worker turn launcher reclaimed placement", () => { } return active; }; - const launchTurn = vi.fn(async (): Promise => { - workerStarted.resolve(); - await resumeWorker.promise; - expect(placements.get(SESSION_ID)).toMatchObject({ - state: "active", - turnClaim: { owner: "worker", runId }, - }); - const completed = openSessionManager(); - const leafId = completed.appendMessage( - makeAgentAssistantMessage({ - content: [{ type: "text", text: "Redispatched worker reply" }], - timestamp: 51, - }), - ); - createWorkerSessionPlacementGate(placements).updateAckCursors({ - sessionId: SESSION_ID, - environmentId: ENVIRONMENT_ID, - ownerEpoch: OWNER_EPOCH, - runId, - transcriptSeq: 2, - liveSeq: 1, - }); - return { - stdout: JSON.stringify({ - status: "completed", - transcriptLeafId: leafId, - transcriptNextSeq: (placements.get(SESSION_ID)?.lastTranscriptAckCursor ?? 0) + 1, - }), - stderr: "", - code: 0, - signal: null, - killed: false, - termination: "exit", - }; - }); + const launchTurn = vi.fn( + async (request: Parameters[0]): Promise => { + request.onDispatchReady?.(); + workerStarted.resolve(); + await resumeWorker.promise; + expect(placements.get(SESSION_ID)).toMatchObject({ + state: "active", + turnClaim: { owner: "worker", runId }, + }); + const completed = openSessionManager(); + const leafId = completed.appendMessage( + makeAgentAssistantMessage({ + content: [{ type: "text", text: "Redispatched worker reply" }], + timestamp: 51, + }), + ); + createWorkerSessionPlacementGate(placements).updateAckCursors({ + sessionId: SESSION_ID, + environmentId: ENVIRONMENT_ID, + ownerEpoch: OWNER_EPOCH, + runId, + transcriptSeq: 2, + liveSeq: 1, + }); + return { + stdout: JSON.stringify({ + status: "completed", + transcriptLeafId: leafId, + transcriptNextSeq: (placements.get(SESSION_ID)?.lastTranscriptAckCursor ?? 0) + 1, + }), + stderr: "", + code: 0, + signal: null, + killed: false, + termination: "exit", + }; + }, + ); const environments: WorkerTurnEnvironmentService = { get: vi.fn(() => attachedEnvironment()), acquireTurnCredential: vi.fn(async () => credential()), diff --git a/src/gateway/worker-environments/worker-turn-launcher-remote-handoff.test.ts b/src/gateway/worker-environments/worker-turn-launcher-remote-handoff.test.ts index 907c923089ac..7fb42f99dbf8 100644 --- a/src/gateway/worker-environments/worker-turn-launcher-remote-handoff.test.ts +++ b/src/gateway/worker-environments/worker-turn-launcher-remote-handoff.test.ts @@ -171,7 +171,8 @@ describe("worker turn launcher remote handoff", () => { kind: "unix", socketPath: "/worker/gateway.sock", }); - await Promise.resolve(); + expect(acknowledgeCredentialDelivery).not.toHaveBeenCalled(); + request.onDispatchReady?.(); expect(acknowledgeCredentialDelivery).toHaveBeenCalledOnce(); const completed = openSessionManager(); const leafId = completed.appendMessage( @@ -406,6 +407,7 @@ describe("worker turn launcher remote handoff", () => { })), runWorkspaceCommand: vi.fn(), launchTurn: vi.fn(async (request): Promise => { + request.onDispatchReady?.(); descriptor = completeWorkerLaunchDescriptor(structuredClone(request.plan), { kind: "unix", socketPath: "/worker/gateway.sock", diff --git a/src/gateway/worker-environments/worker-turn-launcher-terminal-results.test.ts b/src/gateway/worker-environments/worker-turn-launcher-terminal-results.test.ts index a26efc46d4f7..66f8c0fb2c0a 100644 --- a/src/gateway/worker-environments/worker-turn-launcher-terminal-results.test.ts +++ b/src/gateway/worker-environments/worker-turn-launcher-terminal-results.test.ts @@ -42,7 +42,8 @@ describe("worker turn launcher terminal results", () => { resume: vi.fn(async () => {}), })), runWorkspaceCommand: vi.fn(), - launchTurn: vi.fn(async (): Promise => { + launchTurn: vi.fn(async (request): Promise => { + request.onDispatchReady?.(); const completed = openSessionManager(); const leafId = completed.appendMessage( makeAgentAssistantMessage({ @@ -127,7 +128,8 @@ describe("worker turn launcher terminal results", () => { resume: vi.fn(async () => {}), })), runWorkspaceCommand: vi.fn(), - launchTurn: vi.fn(async (): Promise => { + launchTurn: vi.fn(async (request): Promise => { + request.onDispatchReady?.(); const completed = openSessionManager(); completed.appendMessage( makeAgentAssistantMessage({ diff --git a/src/gateway/worker-environments/worker-turn-launcher.ts b/src/gateway/worker-environments/worker-turn-launcher.ts index 9de595afd8fb..f248c08cf9c8 100644 --- a/src/gateway/worker-environments/worker-turn-launcher.ts +++ b/src/gateway/worker-environments/worker-turn-launcher.ts @@ -18,6 +18,7 @@ import type { WorkerSessionPlacementStore, WorkerSessionTurnClaim, } from "./placement-store.js"; +import { WorkerRunnerUnavailableError } from "./tunnel-contract.js"; import { resolveWorkerBrowserLaunchPlan } from "./worker-browser-launch-plan.js"; import { claimWorkerTurn, @@ -302,7 +303,26 @@ async function executeWorkerTurn(params: { turn.userTurnTranscriptRecorder?.markSentToProvider?.(); turn.onExecutionPhase?.({ phase: "attempt_dispatch", backend: "cloud-worker" }); const handoffAbort = new AbortController(); - params.onHandoff(); + let handoffError: Error | undefined; + let dispatchReady = false; + const onDispatchReady = () => { + if (dispatchReady) { + return; + } + dispatchReady = true; + params.onHandoff(); + turn.onExecutionPhase?.({ phase: "process_spawned", backend: "cloud-worker" }); + try { + if (!params.environments.acknowledgeCredentialDelivery(credential)) { + handoffError = new Error("Cloud worker credential owner changed during process handoff"); + } + } catch (error) { + handoffError = new Error("Cloud worker credential handoff failed", { cause: error }); + } + if (handoffError) { + handoffAbort.abort(handoffError); + } + }; const processPromise = tunnel.launchTurn({ plan, placementGeneration: placement.generation, @@ -310,22 +330,15 @@ async function executeWorkerTurn(params: { signal: turn.abortSignal ? AbortSignal.any([turn.abortSignal, handoffAbort.signal]) : handoffAbort.signal, + onDispatchReady, }); - turn.onExecutionPhase?.({ phase: "process_spawned", backend: "cloud-worker" }); - let credentialDelivered: boolean; - try { - credentialDelivered = params.environments.acknowledgeCredentialDelivery(credential); - } catch (error) { - handoffAbort.abort(); - await processPromise.catch(() => undefined); - throw new Error("Cloud worker credential handoff failed", { cause: error }); - } - if (!credentialDelivered) { - handoffAbort.abort(); - await processPromise.catch(() => undefined); - throw new Error("Cloud worker credential owner changed during process handoff"); - } const processResult = await processPromise; + if (handoffError) { + throw handoffError; + } + if (!dispatchReady) { + throw new Error("Cloud worker launch completed before transport dispatch"); + } if (processResult.code !== 0 || processResult.signal !== null || processResult.killed) { // Boxes are destroyed on failure, so the redacted stderr tail is the only forensics. const detail = truncateUtf16Safe( @@ -613,6 +626,10 @@ export function createWorkerSessionTurnPlacementProvider( options.placements.handoffWorkspaceResultRecovery(turnClaim); throw error; } + if (error instanceof WorkerRunnerUnavailableError && !handedOff) { + await releaseClaimIfOwned(options.placements, turnClaim); + throw error; + } if (error instanceof WorkerWorkspaceReconciliationError && !handedOff) { // Recovery runs before remote launch. Preserve the journal's active // generation; only the new admission claim belongs to this attempt. diff --git a/src/gateway/worker-environments/workspace-sync.ts b/src/gateway/worker-environments/workspace-sync.ts index 6031ed1d2108..005ad645fdd4 100644 --- a/src/gateway/worker-environments/workspace-sync.ts +++ b/src/gateway/worker-environments/workspace-sync.ts @@ -137,10 +137,12 @@ export function createWorkerWorkspaceActions( // Exit 255 does not prove whether the remote command was accepted, so stateful // commands must stay pinned to one transport attempt. if (command.transportRetry === "never") { - return await runTask( + const operation = runTask( workerWorkspaceSshArgv(prepared, command.argv), commandOptions(timeoutMs), ); + command.onDispatchReady?.(); + return await operation; } return await runWorkerSshCandidates( prepared, diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 0461b7060a15..91d3af884632 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -5219,10 +5219,13 @@ server.listen(0, "127.0.0.1", () => writeFileSync(readyPath, String(server.addre expect(testStep.run).toContain( "swift_test_args=(--package-path apps/macos --enable-code-coverage)", ); - expect(testStep.run).toContain('if [[ "$SWIFT_TEST_EXECUTION" == "parallel" ]]'); - expect(testStep.run).toContain("swift_test_args+=(--parallel)"); - expect(testStep.run).toContain("else\n swift_test_args+=(--no-parallel)"); - expect(testStep.run).toContain('swift test "${swift_test_args[@]}"'); + expect(testStep.run).toContain('attempt_args=("${swift_test_args[@]}")'); + expect(testStep.run).toContain( + 'if [[ "$SWIFT_TEST_EXECUTION" == "parallel" && "$attempt" -eq 1 ]]', + ); + expect(testStep.run).toContain("attempt_args+=(--parallel)"); + expect(testStep.run).toContain("else\n attempt_args+=(--no-parallel)"); + expect(testStep.run).toContain('swift test "${attempt_args[@]}"'); expect(testStep.run).not.toContain( "swift test --package-path apps/macos --parallel --enable-code-coverage", ); diff --git a/ui/src/e2e/new-session-page.workspace-validation.e2e.test.ts b/ui/src/e2e/new-session-page.workspace-validation.e2e.test.ts index de493a7241f4..fbe2f19c9ec1 100644 --- a/ui/src/e2e/new-session-page.workspace-validation.e2e.test.ts +++ b/ui/src/e2e/new-session-page.workspace-validation.e2e.test.ts @@ -285,12 +285,13 @@ suite.define(() => { await expect.poll(() => start.isDisabled()).toBe(true); await whereTrigger.click(); const cloud = where.getByRole("button", { name: "Cloud · aws" }); - await detailTrigger.click(); - const worktree = detail.getByRole("button", { name: "Worktree" }); expect(await cloud.isDisabled()).toBe(true); expect(await cloud.getAttribute("title")).toBe( "Couldn't verify Git for this folder. Choose it again to retry.", ); + await page.keyboard.press("Escape"); + await detailTrigger.click(); + const worktree = detail.getByRole("button", { name: "Worktree" }); expect(await worktree.getAttribute("aria-pressed")).toBe("true"); expect(await worktree.isDisabled()).toBe(true); expect(await gateway.getRequests("sessions.create")).toHaveLength(0);