mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 21:07:01 -06:00
fix: nested worker delegation no longer times out (#130691)
* fix(workers): bind nested session tools to gateway lifecycle * fix(workers): fence child creation to live turn
This commit is contained in:
@@ -34,7 +34,7 @@ scenario:
|
||||
- Direct and group runs project a person invoker while room and route identifiers never become principals.
|
||||
- Senderless input is admitted with an unknown invoker, and an allowlist rejection creates no run, identity context, or decision fact.
|
||||
- Same and mixed participants remain correctly distinguished across collect-configured QA Channel ingress.
|
||||
- JSON and human CLI inspection remain stable after a Gateway replacement restart.
|
||||
- JSON and human CLI inspection remain stable after a Gateway replacement restart without exposing private generic decision facts as trusted owner evidence.
|
||||
docsRefs:
|
||||
- docs/gateway/audit.md
|
||||
- docs/cli/audit.md
|
||||
@@ -134,8 +134,8 @@ flow:
|
||||
- 60000
|
||||
- 250
|
||||
- assert:
|
||||
expr: "groupInspect.identity.context.invoker.state === 'present' && groupInspect.decisions.some((receipt) => receipt.action.family === 'channel' && receipt.enforcement.coverageState === 'enforced') && !JSON.stringify(groupInspect).includes('qa-group-room')"
|
||||
message: group allowlist admission must record enforced participant evidence without the room as principal
|
||||
expr: "groupInspect.identity.context.invoker.state === 'present' && groupInspect.coverage.state === 'unknown' && groupInspect.coverage.missingEvidence.includes('decision.display_provenance') && groupInspect.decisionDisplays.some((receipt) => receipt.action.family === 'decision' && receipt.provenance.state === 'unverified') && !JSON.stringify(groupInspect).includes('qa-group-room')"
|
||||
message: group inspection must retain participant identity without presenting private generic facts or the room as trusted owner evidence
|
||||
- set: senderlessRunsBefore
|
||||
value:
|
||||
expr: "[...new Set((await runQaCli(env, ['audit', '--kind', 'agent_run', '--limit', '500', '--json'], { timeoutMs: 60000, json: true })).events.map((event) => event.runId).filter((id) => typeof id === 'string'))]"
|
||||
@@ -164,7 +164,7 @@ flow:
|
||||
- 60000
|
||||
- 250
|
||||
- assert:
|
||||
expr: "senderlessInspect.identity.context.invoker.state === 'unknown' && senderlessInspect.decisions.some((receipt) => receipt.action.family === 'channel' && receipt.enforcement.coverageState === 'unknown')"
|
||||
expr: "senderlessInspect.identity.context.invoker.state === 'unknown' && senderlessInspect.coverage.state === 'unknown' && senderlessInspect.decisionDisplays.some((receipt) => receipt.action.family === 'decision' && receipt.provenance.state === 'unverified')"
|
||||
message: senderless admission must stay unknown
|
||||
detailsExpr: "`dm=${dmRunId}; group=${groupRunId}; senderless=${senderlessRunId}`"
|
||||
|
||||
|
||||
@@ -71,6 +71,35 @@ describe("trusted in-process Gateway session creation", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("uses an explicitly bound Gateway when worker creation has no ambient request scope", async () => {
|
||||
mocks.hasContext = false;
|
||||
const admitted = {} as GatewayRequestContext;
|
||||
const resolveGatewayContext = () => admitted;
|
||||
const sessionMutationCommitGuard = vi.fn();
|
||||
const creation = {
|
||||
via: "spawn" as const,
|
||||
actor: { type: "agent" as const, id: "main" },
|
||||
requesterSessionKey: "agent:main:dashboard:worker",
|
||||
inheritedToolPolicy: { version: 1 as const, allow: ["sessions_spawn"], deny: [] },
|
||||
};
|
||||
|
||||
await callInProcessGatewayToolWithCreation("sessions.create", { agentId: "main" }, creation, {
|
||||
resolveGatewayContext,
|
||||
sessionMutationCommitGuard,
|
||||
});
|
||||
|
||||
expect(mocks.dispatch).toHaveBeenCalledWith(
|
||||
"sessions.create",
|
||||
{ agentId: "main" },
|
||||
expect.objectContaining({
|
||||
resolveGatewayContext: expect.any(Function),
|
||||
sessionMutationCommitGuard,
|
||||
sessionCreation: creation,
|
||||
}),
|
||||
);
|
||||
expect(mocks.callGatewayTool).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("carries visible-spawn policy through signed identity on fallback dispatch", async () => {
|
||||
mocks.hasContext = false;
|
||||
const inheritedToolPolicy = {
|
||||
|
||||
@@ -21,6 +21,7 @@ import { callGatewayTool } from "./gateway.js";
|
||||
|
||||
type InProcessGatewayCallOptions = {
|
||||
resolveGatewayContext?: GatewayContextResolver;
|
||||
sessionMutationCommitGuard?: () => void;
|
||||
};
|
||||
|
||||
export type InProcessGatewayCaller = <T = Record<string, unknown>>(
|
||||
@@ -198,11 +199,16 @@ export const callAgentToolGatewayRequest: AgentToolGatewayRequestCaller = async
|
||||
);
|
||||
};
|
||||
|
||||
export const callInProcessGatewayTool: InProcessGatewayCaller = async <T>(
|
||||
async function callInProcessGatewayToolBound<T>(
|
||||
method: string,
|
||||
params: Record<string, unknown>,
|
||||
options: InProcessGatewayCallOptions = {},
|
||||
): Promise<T> => {
|
||||
options: InProcessGatewayCallOptions & {
|
||||
sessionCreation?: TrustedSessionCreation;
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number | null;
|
||||
},
|
||||
fallback: (scopes: ReturnType<typeof resolveLeastPrivilegeOperatorScopesForMethod>) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const scopes = resolveLeastPrivilegeOperatorScopesForMethod(method, params);
|
||||
const resolveGatewayContext = callerGatewayContextResolver(options.resolveGatewayContext);
|
||||
const boundGateway = resolveGatewayContext
|
||||
@@ -215,35 +221,10 @@ export const callInProcessGatewayTool: InProcessGatewayCaller = async <T>(
|
||||
await dispatchGatewayMethodInProcess<T>(method, params, {
|
||||
forceSyntheticClient: true,
|
||||
syntheticScopes: scopes,
|
||||
...(boundResolver ? { resolveGatewayContext: boundResolver } : {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (boundGateway) {
|
||||
throw new Error(`Gateway instance unavailable for ${method}`);
|
||||
}
|
||||
return await callGatewayTool<T>(method, {}, params, { scopes });
|
||||
};
|
||||
|
||||
export async function callInProcessGatewayToolWithCreation<T = Record<string, unknown>>(
|
||||
method: string,
|
||||
params: Record<string, unknown>,
|
||||
creation: TrustedSessionCreation,
|
||||
options: { signal?: AbortSignal; timeoutMs?: number | null } = {},
|
||||
): Promise<T> {
|
||||
const scopes = resolveLeastPrivilegeOperatorScopesForMethod(method, params);
|
||||
const resolveGatewayContext = callerGatewayContextResolver();
|
||||
const boundGateway = resolveGatewayContext
|
||||
? bindInProcessGatewayContext(method, resolveGatewayContext)
|
||||
: undefined;
|
||||
if (hasInProcessGatewayContext(boundGateway?.resolve)) {
|
||||
return await runBoundInProcessGatewayCall(
|
||||
boundGateway,
|
||||
async (boundResolver) =>
|
||||
await dispatchGatewayMethodInProcess<T>(method, params, {
|
||||
forceSyntheticClient: true,
|
||||
sessionCreation: creation,
|
||||
syntheticScopes: scopes,
|
||||
...(options.sessionCreation ? { sessionCreation: options.sessionCreation } : {}),
|
||||
...(options.sessionMutationCommitGuard
|
||||
? { sessionMutationCommitGuard: options.sessionMutationCommitGuard }
|
||||
: {}),
|
||||
...(options.signal ? { signal: options.signal } : {}),
|
||||
...(options.timeoutMs !== undefined && options.timeoutMs !== null
|
||||
? { timeoutMs: options.timeoutMs }
|
||||
@@ -255,28 +236,59 @@ export async function callInProcessGatewayToolWithCreation<T = Record<string, un
|
||||
if (boundGateway) {
|
||||
throw new Error(`Gateway instance unavailable for ${method}`);
|
||||
}
|
||||
// The fallback is a real local Gateway request. Carry spawn policy only in
|
||||
// the signed agent-runtime identity token, never in model-authored params.
|
||||
if (creation.via !== "spawn" || !creation.inheritedToolPolicy) {
|
||||
return await callGatewayTool<T>(method, {}, params, {
|
||||
scopes,
|
||||
...(options.signal ? { signal: options.signal } : {}),
|
||||
...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
|
||||
});
|
||||
}
|
||||
return await runWithGatewaySessionSpawnContext(
|
||||
{
|
||||
...(creation.completionOwnerSessionKey
|
||||
? { completionOwnerSessionKey: creation.completionOwnerSessionKey }
|
||||
: {}),
|
||||
inheritedToolPolicy: creation.inheritedToolPolicy,
|
||||
return await fallback(scopes);
|
||||
}
|
||||
|
||||
export const callInProcessGatewayTool: InProcessGatewayCaller = async <T>(
|
||||
method: string,
|
||||
params: Record<string, unknown>,
|
||||
options: InProcessGatewayCallOptions = {},
|
||||
): Promise<T> => {
|
||||
return await callInProcessGatewayToolBound(method, params, options, async (scopes) =>
|
||||
callGatewayTool<T>(method, {}, params, { scopes }),
|
||||
);
|
||||
};
|
||||
|
||||
export async function callInProcessGatewayToolWithCreation<T = Record<string, unknown>>(
|
||||
method: string,
|
||||
params: Record<string, unknown>,
|
||||
creation: TrustedSessionCreation,
|
||||
options: {
|
||||
resolveGatewayContext?: GatewayContextResolver;
|
||||
sessionMutationCommitGuard?: () => void;
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number | null;
|
||||
} = {},
|
||||
): Promise<T> {
|
||||
return await callInProcessGatewayToolBound(
|
||||
method,
|
||||
params,
|
||||
{ ...options, sessionCreation: creation },
|
||||
async (scopes) => {
|
||||
// The fallback is a real local Gateway request. Carry spawn policy only in
|
||||
// the signed agent-runtime identity token, never in model-authored params.
|
||||
if (creation.via !== "spawn" || !creation.inheritedToolPolicy) {
|
||||
return await callGatewayTool<T>(method, {}, params, {
|
||||
scopes,
|
||||
...(options.signal ? { signal: options.signal } : {}),
|
||||
...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
|
||||
});
|
||||
}
|
||||
return await runWithGatewaySessionSpawnContext(
|
||||
{
|
||||
...(creation.completionOwnerSessionKey
|
||||
? { completionOwnerSessionKey: creation.completionOwnerSessionKey }
|
||||
: {}),
|
||||
inheritedToolPolicy: creation.inheritedToolPolicy,
|
||||
},
|
||||
() =>
|
||||
callGatewayTool<T>(method, {}, params, {
|
||||
scopes,
|
||||
requireAgentRuntimeIdentity: true,
|
||||
...(options.signal ? { signal: options.signal } : {}),
|
||||
...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
|
||||
}),
|
||||
);
|
||||
},
|
||||
() =>
|
||||
callGatewayTool<T>(method, {}, params, {
|
||||
scopes,
|
||||
requireAgentRuntimeIdentity: true,
|
||||
...(options.signal ? { signal: options.signal } : {}),
|
||||
...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ type InProcessGatewayDispatchOptions = {
|
||||
onAccepted?: (payload: unknown) => void;
|
||||
onSignalAbort?: () => Promise<void> | void;
|
||||
requestIdPrefix?: string;
|
||||
sessionMutationCommitGuard?: () => void;
|
||||
timeoutMs?: number;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
@@ -182,6 +183,7 @@ export async function dispatchGatewayRequestInProcessRaw(
|
||||
},
|
||||
context: options.context,
|
||||
methodRegistry: options.methodRegistry,
|
||||
sessionMutationCommitGuard: options.sessionMutationCommitGuard,
|
||||
...(options.signal ? { signal: options.signal } : {}),
|
||||
})
|
||||
.then(() => {
|
||||
|
||||
@@ -702,6 +702,9 @@ export async function handleGatewayRequest(
|
||||
respond,
|
||||
context,
|
||||
...(signal ? { signal } : {}),
|
||||
...(opts.sessionMutationCommitGuard
|
||||
? { sessionMutationCommitGuard: opts.sessionMutationCommitGuard }
|
||||
: {}),
|
||||
...(authorization.sessionMutationAuthorization
|
||||
? { sessionMutationAuthorization: authorization.sessionMutationAuthorization }
|
||||
: {}),
|
||||
|
||||
@@ -67,6 +67,7 @@ export const sessionCreateHandlers: GatewayRequestHandlers = {
|
||||
context,
|
||||
client,
|
||||
isWebchatConnect,
|
||||
sessionMutationCommitGuard,
|
||||
sessionMutationAuthorization,
|
||||
}) => {
|
||||
if (!assertValidParams(params, validateSessionsCreateParams, "sessions.create", respond)) {
|
||||
@@ -78,8 +79,9 @@ export const sessionCreateHandlers: GatewayRequestHandlers = {
|
||||
const cfg = context.getRuntimeConfig();
|
||||
const authority = createAgentRuntimeAuthorityGuard(client, context, respond);
|
||||
const commitGuard =
|
||||
authority.commitGuard || sessionMutationAuthorization
|
||||
authority.commitGuard || sessionMutationCommitGuard || sessionMutationAuthorization
|
||||
? () => {
|
||||
sessionMutationCommitGuard?.();
|
||||
authority.commitGuard?.();
|
||||
sessionMutationAuthorization?.assertCurrent();
|
||||
}
|
||||
|
||||
@@ -451,6 +451,8 @@ export type GatewayRequestOptions = {
|
||||
respond: RespondFn;
|
||||
context: GatewayRequestContext;
|
||||
methodRegistry?: GatewayMethodRegistryView;
|
||||
/** In-process Gateway lifetime guard composed into durable session mutations. */
|
||||
sessionMutationCommitGuard?: () => void;
|
||||
/** In-process caller lifetime; never serialized into a Gateway request frame. */
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
@@ -469,6 +471,7 @@ export type GatewayRequestHandlerOptions = {
|
||||
isWebchatConnect: (params: ConnectParams | null | undefined) => boolean;
|
||||
respond: RespondFn;
|
||||
context: GatewayRequestContext;
|
||||
sessionMutationCommitGuard?: () => void;
|
||||
sessionMutationAuthorization?: SessionMutationAuthorization;
|
||||
/** In-process caller lifetime; absent for ordinary transport requests. */
|
||||
signal?: AbortSignal;
|
||||
|
||||
@@ -8,6 +8,7 @@ import { PROTOCOL_VERSION } from "../../packages/gateway-protocol/src/version.js
|
||||
import { createOperationalRunInstanceRef } from "../agents/admitted-run-context.js";
|
||||
import { upsertSessionEntryCore } from "../config/sessions/session-accessor.js";
|
||||
import { withPluginRuntimeGatewayRequestScope } from "../plugins/runtime/gateway-request-scope.js";
|
||||
import { createDeferredCore } from "../shared/deferred.js";
|
||||
import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
|
||||
import { createGatewayMethodRegistry } from "./methods/registry.js";
|
||||
import { resolveNodeInvokeRuntimeAuthorityError } from "./server-methods/nodes.invoke-authority.js";
|
||||
@@ -192,6 +193,79 @@ describe("typed in-process agent authorization", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("carries an explicit Gateway binding into the session mutation commit guard", async () => {
|
||||
const admitted = createContext();
|
||||
const replacement = createContext();
|
||||
let current = admitted;
|
||||
let committed = false;
|
||||
const setupStarted = createDeferredCore();
|
||||
const releaseSetup = createDeferredCore();
|
||||
admitted.getGatewayMethodRegistry = () =>
|
||||
createGatewayMethodRegistry([
|
||||
{
|
||||
name: "sessions.create",
|
||||
scope: "operator.write",
|
||||
owner: { kind: "core", area: "sessions" },
|
||||
handler: async ({
|
||||
respond,
|
||||
sessionMutationCommitGuard,
|
||||
}: GatewayRequestHandlerOptions) => {
|
||||
setupStarted.resolve();
|
||||
await releaseSetup.promise;
|
||||
sessionMutationCommitGuard?.();
|
||||
committed = true;
|
||||
respond(true, { key: "agent:main:dashboard:child" });
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const dispatch = dispatchGatewayMethodInProcess(
|
||||
"sessions.create",
|
||||
{ agentId: "main" },
|
||||
{
|
||||
forceSyntheticClient: true,
|
||||
resolveGatewayContext: () => current,
|
||||
syntheticScopes: ["operator.write"],
|
||||
},
|
||||
);
|
||||
await setupStarted.promise;
|
||||
current = replacement;
|
||||
releaseSetup.resolve();
|
||||
|
||||
await expect(dispatch).rejects.toThrow("current gateway instance binding");
|
||||
expect(committed).toBe(false);
|
||||
});
|
||||
|
||||
it("composes caller authority into the session mutation commit guard", async () => {
|
||||
const admitted = createContext();
|
||||
const assertCallerCurrent = vi.fn();
|
||||
admitted.getGatewayMethodRegistry = () =>
|
||||
createGatewayMethodRegistry([
|
||||
{
|
||||
name: "sessions.create",
|
||||
scope: "operator.write",
|
||||
owner: { kind: "core", area: "sessions" },
|
||||
handler: ({ respond, sessionMutationCommitGuard }: GatewayRequestHandlerOptions) => {
|
||||
sessionMutationCommitGuard?.();
|
||||
respond(true, { key: "agent:main:dashboard:child" });
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
await dispatchGatewayMethodInProcess(
|
||||
"sessions.create",
|
||||
{ agentId: "main" },
|
||||
{
|
||||
forceSyntheticClient: true,
|
||||
resolveGatewayContext: () => admitted,
|
||||
sessionMutationCommitGuard: assertCallerCurrent,
|
||||
syntheticScopes: ["operator.write"],
|
||||
},
|
||||
);
|
||||
|
||||
expect(assertCallerCurrent).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("preserves the scoped operator identity across synthetic model-initiated session creation", async () => {
|
||||
const owner = createOperatorClient({
|
||||
profileId: "model-spawn-owner",
|
||||
|
||||
@@ -84,6 +84,7 @@ type DispatchGatewayMethodInProcessOptions = {
|
||||
timeoutMs?: number;
|
||||
signal?: AbortSignal;
|
||||
resolveGatewayContext?: GatewayContextResolver;
|
||||
sessionMutationCommitGuard?: () => void;
|
||||
};
|
||||
|
||||
type ResolvedInProcessGatewayDispatch = {
|
||||
@@ -275,23 +276,37 @@ export async function dispatchGatewayMethodInProcessRaw(
|
||||
params: unknown,
|
||||
options?: DispatchGatewayMethodInProcessOptions,
|
||||
): Promise<GatewayMethodDispatchResponse> {
|
||||
return await withInProcessGatewayDispatch(
|
||||
method,
|
||||
options,
|
||||
async (resolved) =>
|
||||
await dispatchGatewayRequestInProcessRaw(method, params, {
|
||||
client: resolved.client,
|
||||
context: resolved.context,
|
||||
expectFinal: options?.expectFinal,
|
||||
isWebchatConnect: resolved.isWebchatConnect,
|
||||
methodRegistry: resolved.context.getGatewayMethodRegistry?.(),
|
||||
onAccepted: options?.onAccepted,
|
||||
onSignalAbort: options?.onSignalAbort,
|
||||
requestIdPrefix: "plugin-subagent",
|
||||
timeoutMs: options?.timeoutMs,
|
||||
...(options?.signal ? { signal: options.signal } : {}),
|
||||
}),
|
||||
);
|
||||
return await withInProcessGatewayDispatch(method, options, async (resolved) => {
|
||||
const assertGatewayContextCurrent = options?.resolveGatewayContext
|
||||
? () => {
|
||||
if (options.resolveGatewayContext?.() !== resolved.context) {
|
||||
throw new Error(
|
||||
`In-process gateway dispatch requires a current gateway instance binding (method: ${method}).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
: undefined;
|
||||
const sessionMutationCommitGuard =
|
||||
assertGatewayContextCurrent || options?.sessionMutationCommitGuard
|
||||
? () => {
|
||||
assertGatewayContextCurrent?.();
|
||||
options?.sessionMutationCommitGuard?.();
|
||||
}
|
||||
: undefined;
|
||||
return await dispatchGatewayRequestInProcessRaw(method, params, {
|
||||
client: resolved.client,
|
||||
context: resolved.context,
|
||||
expectFinal: options?.expectFinal,
|
||||
isWebchatConnect: resolved.isWebchatConnect,
|
||||
methodRegistry: resolved.context.getGatewayMethodRegistry?.(),
|
||||
onAccepted: options?.onAccepted,
|
||||
onSignalAbort: options?.onSignalAbort,
|
||||
requestIdPrefix: "plugin-subagent",
|
||||
...(sessionMutationCommitGuard ? { sessionMutationCommitGuard } : {}),
|
||||
timeoutMs: options?.timeoutMs,
|
||||
...(options?.signal ? { signal: options.signal } : {}),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** Live request context for trusted built-in tools that need direct runtime state. */
|
||||
|
||||
@@ -157,6 +157,7 @@ export async function prepareGatewayKernelState(params: {
|
||||
return await workerModule.createGatewayWorkerEnvironmentRuntime({
|
||||
getPluginRegistry: () => pluginRuntime.registry,
|
||||
getPortalRuntime: () => pluginGatewayContext.current,
|
||||
resolveGatewayContext: () => pluginGatewayContext.current,
|
||||
desktopSessionRegistry,
|
||||
...(nodeDesktopStreamBroker ? { nodeDesktopStreamBroker } : {}),
|
||||
startup: workerEnvironmentStartup,
|
||||
|
||||
@@ -47,6 +47,7 @@ describe("gateway worker environment startup", () => {
|
||||
const runtime = await createGatewayWorkerEnvironmentRuntime({
|
||||
getPluginRegistry: () => ({ workerProviders: new Map() }),
|
||||
getPortalRuntime: () => undefined,
|
||||
resolveGatewayContext: () => undefined,
|
||||
desktopSessionRegistry: createDesktopSessionRegistry({ lingerMs: 1 }),
|
||||
startup,
|
||||
log: { child: () => ({ warn: () => {} }) },
|
||||
@@ -108,6 +109,7 @@ describe("gateway worker environment startup", () => {
|
||||
const runtime = await createGatewayWorkerEnvironmentRuntime({
|
||||
getPluginRegistry: () => ({ workerProviders: new Map() }),
|
||||
getPortalRuntime: () => undefined,
|
||||
resolveGatewayContext: () => undefined,
|
||||
desktopSessionRegistry: createDesktopSessionRegistry({ lingerMs: 1 }),
|
||||
startup,
|
||||
log: { child: () => ({ warn: () => {} }) },
|
||||
@@ -205,6 +207,7 @@ describe("gateway worker environment startup", () => {
|
||||
const runtime = await createGatewayWorkerEnvironmentRuntime({
|
||||
getPluginRegistry: () => ({ workerProviders: new Map() }),
|
||||
getPortalRuntime: () => undefined,
|
||||
resolveGatewayContext: () => undefined,
|
||||
desktopSessionRegistry: createDesktopSessionRegistry({ lingerMs: 1 }),
|
||||
nodeDesktopStreamBroker: createNodeDesktopStreamBroker(),
|
||||
startup,
|
||||
|
||||
@@ -13,7 +13,7 @@ import type { NodeDesktopStreamBroker } from "./desktop/node-stream-broker.js";
|
||||
import type { DesktopSessionRegistry } from "./desktop/session-registry.js";
|
||||
import type { GitHubPublicationCoordinator } from "./github-publication.js";
|
||||
import type { NodeWorkerSupervisorTransport } from "./node-registry-private.js";
|
||||
import type { GatewayRequestContext } from "./server-methods/types.js";
|
||||
import type { GatewayContextResolver, GatewayRequestContext } from "./server-methods/types.js";
|
||||
import type { WorkerBundleProducer, WorkerNpmArtifact } from "./worker-environments/bundle.js";
|
||||
import {
|
||||
bindDeviceWorkerAvailability,
|
||||
@@ -110,6 +110,7 @@ export async function loadGatewayWorkerEnvironmentStartupState(): Promise<Gatewa
|
||||
export async function createGatewayWorkerEnvironmentRuntime(params: {
|
||||
getPluginRegistry: () => Pick<PluginRegistry, "workerProviders">;
|
||||
getPortalRuntime: () => Pick<GatewayRequestContext, "portalService" | "broadcast"> | undefined;
|
||||
resolveGatewayContext: GatewayContextResolver;
|
||||
desktopSessionRegistry: DesktopSessionRegistry;
|
||||
nodeDesktopStreamBroker?: NodeDesktopStreamBroker;
|
||||
startup: GatewayWorkerEnvironmentStartupState;
|
||||
@@ -382,6 +383,7 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
|
||||
return environmentIds;
|
||||
});
|
||||
executeSessionTool = createWorkerSessionToolExecutor({
|
||||
resolveGatewayContext: params.resolveGatewayContext,
|
||||
placements: params.startup.placementStore,
|
||||
environments: workerEnvironmentService,
|
||||
dispatchChild: (request) => dispatchChild(request),
|
||||
|
||||
@@ -39,7 +39,10 @@ import {
|
||||
openOpenClawAgentDatabase,
|
||||
resolveIncognitoOpenClawAgentSqlitePath,
|
||||
} from "../state/openclaw-agent-db.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import { ensureProfileForEmail, setUserProfileRole } from "../state/user-profiles.js";
|
||||
import { createOpenClawTestState } from "../test-utils/openclaw-test-state.js";
|
||||
import { GATEWAY_CLIENT_MODES, GATEWAY_CLIENT_NAMES } from "../utils/message-channel.js";
|
||||
@@ -76,6 +79,7 @@ import {
|
||||
seedSessionTranscript,
|
||||
threadBindingMocks,
|
||||
} from "./test/server-sessions.test-helpers.js";
|
||||
import { createWorkerSessionPlacementStore } from "./worker-environments/placement-store.js";
|
||||
|
||||
type EnsureSessionDiffBaseline =
|
||||
(typeof import("../sessions/session-diff-baseline.js"))["ensureSessionDiffBaseline"];
|
||||
@@ -3457,6 +3461,126 @@ test("sessions.create commits no session after delegated authority closes", asyn
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
test("sessions.create commits no child after its bound Gateway is replaced", async () => {
|
||||
const { storePath } = await createSessionStoreDir();
|
||||
const sessionKey = "agent:main:dashboard:gateway-replacement-race";
|
||||
const admitted = {};
|
||||
const replacement = {};
|
||||
let current = admitted;
|
||||
let guardCalls = 0;
|
||||
const firstGuard = createDeferredCore();
|
||||
const writerEntered = createDeferredCore();
|
||||
const releaseWriter = createDeferredCore();
|
||||
const resolvedStore = resolveSqliteStoreScope(storePath, { agentId: "main" });
|
||||
const heldWriter = runExclusiveSqliteSessionWrite(resolvedStore, async () => {
|
||||
writerEntered.resolve();
|
||||
await releaseWriter.promise;
|
||||
});
|
||||
await writerEntered.promise;
|
||||
const creating = directSessionReq(
|
||||
"sessions.create",
|
||||
{ agentId: "main", key: sessionKey },
|
||||
{
|
||||
sessionMutationAuthorization: {
|
||||
assertCurrent: () => {
|
||||
if (current !== admitted) {
|
||||
throw new Error("current gateway instance binding was replaced");
|
||||
}
|
||||
guardCalls += 1;
|
||||
if (guardCalls === 1) {
|
||||
firstGuard.resolve();
|
||||
}
|
||||
},
|
||||
assertTargetCurrent: vi.fn(),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
await firstGuard.promise;
|
||||
current = replacement;
|
||||
releaseWriter.resolve();
|
||||
await heldWriter;
|
||||
|
||||
await expect(creating).rejects.toThrow("current gateway instance binding was replaced");
|
||||
expect(loadSessionEntry({ agentId: "main", sessionKey, storePath })).toBeUndefined();
|
||||
});
|
||||
|
||||
test("sessions.create commits no child after its worker turn closes", async () => {
|
||||
const { storePath } = await createSessionStoreDir();
|
||||
const sessionKey = "agent:main:dashboard:worker-turn-race";
|
||||
const placements = createWorkerSessionPlacementStore({
|
||||
database: openOpenClawStateDatabase(),
|
||||
});
|
||||
let placement = placements.startDispatch({
|
||||
agentId: "main",
|
||||
sessionId: "worker-source-session",
|
||||
sessionKey: "agent:main:dashboard:worker-source",
|
||||
});
|
||||
for (const [from, to, patch] of [
|
||||
["requested", "provisioning", { environmentId: "worker-environment" }],
|
||||
["provisioning", "syncing", { workerBundleHash: "a".repeat(64) }],
|
||||
[
|
||||
"syncing",
|
||||
"starting",
|
||||
{ remoteWorkspaceDir: "/workspace/source", workspaceBaseManifestRef: "manifest-source" },
|
||||
],
|
||||
["starting", "active", { activeOwnerEpoch: 7 }],
|
||||
] as const) {
|
||||
placement = placements.transition({
|
||||
sessionId: placement.sessionId,
|
||||
from,
|
||||
to,
|
||||
expectedGeneration: placement.generation,
|
||||
patch,
|
||||
});
|
||||
}
|
||||
const turnClaim = placements.claimTurn({
|
||||
agentId: placement.agentId,
|
||||
sessionId: placement.sessionId,
|
||||
sessionKey: placement.sessionKey,
|
||||
claimId: "worker-claim",
|
||||
runId: "worker-run",
|
||||
owner: { kind: "worker", environmentId: "worker-environment", ownerEpoch: 7 },
|
||||
});
|
||||
let guardCalls = 0;
|
||||
const firstGuard = createDeferredCore();
|
||||
const writerEntered = createDeferredCore();
|
||||
const releaseWriter = createDeferredCore();
|
||||
const heldWriter = runExclusiveSqliteSessionWrite(
|
||||
resolveSqliteStoreScope(storePath, { agentId: "main" }),
|
||||
async () => {
|
||||
writerEntered.resolve();
|
||||
await releaseWriter.promise;
|
||||
},
|
||||
);
|
||||
await writerEntered.promise;
|
||||
const creating = directSessionReq(
|
||||
"sessions.create",
|
||||
{ agentId: "main", key: sessionKey },
|
||||
{
|
||||
sessionMutationAuthorization: {
|
||||
assertCurrent: () => {
|
||||
if (!placements.validateTurnClaim(turnClaim)) {
|
||||
throw new Error("worker turn authority changed");
|
||||
}
|
||||
if (++guardCalls === 1) {
|
||||
firstGuard.resolve();
|
||||
}
|
||||
},
|
||||
assertTargetCurrent: vi.fn(),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
await firstGuard.promise;
|
||||
placements.releaseTurn(turnClaim);
|
||||
releaseWriter.resolve();
|
||||
await heldWriter;
|
||||
|
||||
await expect(creating).rejects.toThrow("worker turn authority changed");
|
||||
expect(loadSessionEntry({ agentId: "main", sessionKey, storePath })).toBeUndefined();
|
||||
});
|
||||
|
||||
test("sessions.create starts no initial turn when authority closes after session commit", async () => {
|
||||
await createSessionStoreDir();
|
||||
const sessionKey = "agent:main:dashboard:authority-post-commit";
|
||||
|
||||
@@ -33,6 +33,7 @@ const dispatchChild = vi.hoisted(() => vi.fn());
|
||||
const spawnCallerIdentity = vi.hoisted(() => vi.fn());
|
||||
const spawnArgs = vi.hoisted(() => vi.fn());
|
||||
const githubPublicationRequest = vi.hoisted(() => vi.fn());
|
||||
const resolveGatewayContext = () => undefined;
|
||||
const scopedSessionAccess = vi.hoisted(() =>
|
||||
vi.fn(async (params: { run: () => Promise<unknown> }) => await params.run()),
|
||||
);
|
||||
@@ -97,7 +98,8 @@ vi.mock("../../agents/tools/in-process-gateway.js", () => ({
|
||||
method: string,
|
||||
params: Record<string, unknown>,
|
||||
creation: unknown,
|
||||
) => gatewayCreate({ creation, method, params }),
|
||||
options: unknown,
|
||||
) => gatewayCreate({ creation, method, options, params }),
|
||||
withAgentToolGatewayRuntimeIdentity: (request: unknown, identity: unknown) => {
|
||||
gatewayRuntimeIdentity(request, identity);
|
||||
return request;
|
||||
@@ -246,6 +248,7 @@ describe("worker session tool topology", () => {
|
||||
},
|
||||
);
|
||||
execute = createWorkerSessionToolExecutor({
|
||||
resolveGatewayContext,
|
||||
placements,
|
||||
dispatchChild,
|
||||
githubPublication: { requestForClaim: githubPublicationRequest },
|
||||
@@ -452,6 +455,11 @@ describe("worker session tool topology", () => {
|
||||
via: "spawn",
|
||||
}),
|
||||
method: "sessions.create",
|
||||
options: {
|
||||
resolveGatewayContext,
|
||||
sessionMutationCommitGuard: expect.any(Function),
|
||||
timeoutMs: null,
|
||||
},
|
||||
params: expect.not.objectContaining({ task: expect.anything() }),
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -26,6 +26,7 @@ import { sha256Base64Url, sha256HexPrefixCore } from "../../infra/crypto-digest.
|
||||
import { normalizeAgentId } from "../../routing/session-key.js";
|
||||
import { WORKER_TOOL_NAMES } from "../../worker/tool-authority.js";
|
||||
import type { GitHubPublicationCoordinator } from "../github-publication.js";
|
||||
import type { GatewayContextResolver } from "../server-methods/types.js";
|
||||
import { loadGatewaySessionEntryReadOnly } from "../session-utils.js";
|
||||
import type { WorkerConnectionIdentity } from "./connection-identity.js";
|
||||
import type { WorkerSessionPlacementStore } from "./placement-store.js";
|
||||
@@ -76,19 +77,15 @@ function operationKey(operationSeed: string, purpose: string): string {
|
||||
return sha256Base64Url(`openclaw.worker-session-tool-operation.v1\0${operationSeed}\0${purpose}`);
|
||||
}
|
||||
|
||||
function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||
signal?.throwIfAborted();
|
||||
}
|
||||
|
||||
function childSessionKey(params: { operationSeed: string; targetAgentId: string }): string {
|
||||
const suffix = sha256HexPrefixCore(
|
||||
`openclaw.worker-session-tool-operation.v1\0${params.operationSeed}\0child-session`,
|
||||
function childSessionKey(operationSeed: string, targetAgentId: string): string {
|
||||
return `agent:${targetAgentId}:dashboard:cloud-${sha256HexPrefixCore(
|
||||
`openclaw.worker-session-tool-operation.v1\0${operationSeed}\0child-session`,
|
||||
32,
|
||||
);
|
||||
return `agent:${params.targetAgentId}:dashboard:cloud-${suffix}`;
|
||||
)}`;
|
||||
}
|
||||
|
||||
export function createWorkerSessionToolExecutor(params: {
|
||||
resolveGatewayContext: GatewayContextResolver;
|
||||
placements: WorkerSessionPlacementStore;
|
||||
environments: Pick<WorkerEnvironmentService, "get">;
|
||||
dispatchChild: WorkerPlacementDispatchContract["dispatch"];
|
||||
@@ -106,14 +103,13 @@ export function createWorkerSessionToolExecutor(params: {
|
||||
childSessionKey: string;
|
||||
signal?: AbortSignal;
|
||||
}) => {
|
||||
throwIfAborted(operation.signal);
|
||||
operation.signal?.throwIfAborted();
|
||||
const sourceEnvironment = params.environments.get(operation.identity.environmentId);
|
||||
if (
|
||||
!sourceEnvironment ||
|
||||
sourceEnvironment.state !== "attached" ||
|
||||
sourceEnvironment.ownerEpoch !== operation.identity.ownerEpoch ||
|
||||
sourceEnvironment.attachedSessionIds.length !== 1 ||
|
||||
sourceEnvironment.attachedSessionIds[0] !== operation.source.sessionId
|
||||
!isDeepStrictEqual(sourceEnvironment.attachedSessionIds, [operation.source.sessionId])
|
||||
) {
|
||||
throw new Error("Worker source environment changed before child spawn");
|
||||
}
|
||||
@@ -138,7 +134,7 @@ export function createWorkerSessionToolExecutor(params: {
|
||||
timeoutMs: null,
|
||||
});
|
||||
}
|
||||
throwIfAborted(operation.signal);
|
||||
operation.signal?.throwIfAborted();
|
||||
exactSource({ identity: operation.identity, placements: params.placements });
|
||||
let loaded = loadGatewaySessionEntryReadOnly(operation.childSessionKey, {
|
||||
agentId: targetAgentId,
|
||||
@@ -183,6 +179,9 @@ export function createWorkerSessionToolExecutor(params: {
|
||||
inheritedToolPolicy: { version: 1, allow: authorizedTools, deny: [] },
|
||||
},
|
||||
{
|
||||
resolveGatewayContext: params.resolveGatewayContext,
|
||||
sessionMutationCommitGuard: () =>
|
||||
exactSource({ identity: operation.identity, placements: params.placements }),
|
||||
...(operation.signal ? { signal: operation.signal } : {}),
|
||||
timeoutMs: null,
|
||||
},
|
||||
@@ -244,7 +243,7 @@ export function createWorkerSessionToolExecutor(params: {
|
||||
}
|
||||
};
|
||||
const childPlacement = params.placements.get(childSessionId);
|
||||
throwIfAborted(operation.signal);
|
||||
operation.signal?.throwIfAborted();
|
||||
exactSource({ identity: operation.identity, placements: params.placements });
|
||||
if (childPlacement?.state !== "active") {
|
||||
try {
|
||||
@@ -269,7 +268,7 @@ export function createWorkerSessionToolExecutor(params: {
|
||||
}
|
||||
assertActiveChildPlacement();
|
||||
exactSource({ identity: operation.identity, placements: params.placements });
|
||||
throwIfAborted(operation.signal);
|
||||
operation.signal?.throwIfAborted();
|
||||
assertExactChild({
|
||||
childSessionKey: operation.childSessionKey,
|
||||
childSessionId,
|
||||
@@ -306,7 +305,7 @@ export function createWorkerSessionToolExecutor(params: {
|
||||
let sendResult: Record<string, unknown> | undefined;
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
try {
|
||||
throwIfAborted(operation.signal);
|
||||
operation.signal?.throwIfAborted();
|
||||
exactSource({ identity: operation.identity, placements: params.placements });
|
||||
assertExactChild({
|
||||
childSessionKey: operation.childSessionKey,
|
||||
@@ -424,6 +423,7 @@ export function createWorkerSessionToolExecutor(params: {
|
||||
{
|
||||
agentId: identity.agentId,
|
||||
sessionKey: identity.sessionKey,
|
||||
gatewayContextResolver: params.resolveGatewayContext,
|
||||
operationalRunInstance: identity.operationalRunInstance,
|
||||
executionIdentityToken: identity.executionIdentityToken,
|
||||
receiptAuthority: identity.receiptAuthority,
|
||||
@@ -436,7 +436,14 @@ export function createWorkerSessionToolExecutor(params: {
|
||||
workerIdentity = undefined;
|
||||
}
|
||||
})
|
||||
: await executeSpawn();
|
||||
: await withGatewayToolCallerIdentity(
|
||||
{
|
||||
agentId: operation.source.agentId,
|
||||
sessionKey: operation.source.sessionKey,
|
||||
gatewayContextResolver: params.resolveGatewayContext,
|
||||
},
|
||||
executeSpawn,
|
||||
);
|
||||
};
|
||||
|
||||
const send = async (operation: {
|
||||
@@ -447,7 +454,7 @@ export function createWorkerSessionToolExecutor(params: {
|
||||
idempotencyKey: string;
|
||||
signal?: AbortSignal;
|
||||
}) => {
|
||||
throwIfAborted(operation.signal);
|
||||
operation.signal?.throwIfAborted();
|
||||
exactSource({ identity: operation.identity, placements: params.placements });
|
||||
const config = getRuntimeConfig();
|
||||
const executeFencedSend = async () => {
|
||||
@@ -480,7 +487,7 @@ export function createWorkerSessionToolExecutor(params: {
|
||||
});
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
try {
|
||||
throwIfAborted(operation.signal);
|
||||
operation.signal?.throwIfAborted();
|
||||
exactSource({ identity: operation.identity, placements: params.placements });
|
||||
assertCurrentTarget();
|
||||
return await tool.execute(operation.request.toolCallId, {
|
||||
@@ -528,7 +535,7 @@ export function createWorkerSessionToolExecutor(params: {
|
||||
}
|
||||
};
|
||||
assertPublicationAuthority();
|
||||
throwIfAborted(request.signal);
|
||||
request.signal?.throwIfAborted();
|
||||
const publication = await params.githubPublication.requestForClaim({
|
||||
claim: source.turnClaim,
|
||||
sessionKey: source.sessionKey,
|
||||
@@ -618,10 +625,7 @@ export function createWorkerSessionToolExecutor(params: {
|
||||
let childKey = started.childSessionKey;
|
||||
if (request.toolName === "sessions_spawn" && !childKey) {
|
||||
const targetAgentId = normalizeAgentId(request.request.agentId ?? source.agentId);
|
||||
childKey = childSessionKey({
|
||||
operationSeed: started.operationSeed,
|
||||
targetAgentId,
|
||||
});
|
||||
childKey = childSessionKey(started.operationSeed, targetAgentId);
|
||||
if (
|
||||
!params.placements.bindWorkerSessionToolOperationChild({
|
||||
sourceSessionId: source.sessionId,
|
||||
|
||||
Reference in New Issue
Block a user