feat(agents): add tool-free isolated completion (#114343)

* feat: add isolated pure-inference completion

* fix(google): block ambient system prompt writes

* docs: refresh generated map
This commit is contained in:
Josh Avant
2026-07-30 13:24:45 -05:00
committed by GitHub
parent 3ffd3864ea
commit ce67ffb70e
67 changed files with 6188 additions and 1244 deletions
@@ -0,0 +1,423 @@
// Isolated runtime.llm.complete tests cover zero-tool dispatch and policy enforcement.
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { withPluginRuntimePluginIdScope } from "./gateway-request-scope.js";
import { createRuntimeLlm } from "./runtime-llm.runtime.js";
const hoisted = vi.hoisted(() => ({
prepareSimpleCompletionModelForAgent: vi.fn(),
completeWithPreparedSimpleCompletionModel: vi.fn(),
resolveSimpleCompletionSelectionForAgent: vi.fn(),
runIsolatedCompletion: vi.fn(),
}));
vi.mock("../../agents/isolated-completion.js", () => ({
runIsolatedCompletion: hoisted.runIsolatedCompletion,
}));
vi.mock("../../agents/simple-completion-runtime.js", () => ({
prepareSimpleCompletionModelForAgent: hoisted.prepareSimpleCompletionModelForAgent,
completeWithPreparedSimpleCompletionModel: hoisted.completeWithPreparedSimpleCompletionModel,
resolveSimpleCompletionSelectionForAgent: hoisted.resolveSimpleCompletionSelectionForAgent,
}));
const cfg = {
agents: {
defaults: {
model: "openai/gpt-5.5",
},
},
} satisfies OpenClawConfig;
function primeCompletionMocks() {
hoisted.resolveSimpleCompletionSelectionForAgent.mockImplementation(
(params: { modelRef?: string; agentId: string }) => {
const slash = params.modelRef?.indexOf("/") ?? -1;
return {
provider: slash > 0 ? params.modelRef?.slice(0, slash) : "openai",
modelId: slash > 0 ? params.modelRef?.slice(slash + 1) : (params.modelRef ?? "gpt-5.5"),
agentDir: `/tmp/${params.agentId}`,
};
},
);
hoisted.runIsolatedCompletion.mockResolvedValue({
text: "isolated",
provider: "openai",
model: "gpt-5.5",
owner: { kind: "harness", id: "openclaw" },
usage: {
input: 3,
output: 2,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 5,
},
});
}
function expectSingleCallFirstArg(mock: { mock: { calls: unknown[][] } }, expected: object) {
expect(mock.mock.calls).toHaveLength(1);
expect(mock.mock.calls[0]?.[0]).toEqual(expect.objectContaining(expected));
}
describe("runtime.llm.complete isolated agent runtime", () => {
beforeEach(() => {
hoisted.prepareSimpleCompletionModelForAgent.mockReset();
hoisted.completeWithPreparedSimpleCompletionModel.mockReset();
hoisted.resolveSimpleCompletionSelectionForAgent.mockReset();
hoisted.runIsolatedCompletion.mockReset();
primeCompletionMocks();
});
it("routes authorized isolated completion through the configured agent runtime", async () => {
hoisted.resolveSimpleCompletionSelectionForAgent.mockReturnValueOnce({
provider: "openai",
modelId: "gpt-5.5",
profileId: "openai:configured",
agentDir: "/tmp/main",
});
const llm = createRuntimeLlm({
getConfig: () => ({
...cfg,
plugins: {
entries: {
"llm-task": {
llm: {
allowAuthProfileOverride: true,
},
},
},
},
}),
authority: { allowComplete: true, preferredProfile: "openai:authority-bound" },
});
const result = await withPluginRuntimePluginIdScope("llm-task", () =>
llm.complete({
messages: [{ role: "user", content: "Return JSON" }],
systemPrompt: "JSON only",
reasoning: "high",
execution: {
mode: "isolated-agent-runtime",
authProfileId: "openai:work",
timeoutMs: 12_000,
},
}),
);
expectSingleCallFirstArg(hoisted.runIsolatedCompletion, {
config: expect.any(Object),
provider: "openai",
model: "gpt-5.5",
authProfileId: "openai:work",
agentId: "main",
systemPrompt: "JSON only",
prompt: "Return JSON",
timeoutMs: 12_000,
thinkLevel: "high",
streamParams: { maxTokens: undefined, temperature: undefined },
});
expect(result).toMatchObject({
text: "isolated",
execution: {
mode: "isolated-agent-runtime",
owner: { kind: "harness", id: "openclaw" },
},
usage: { inputTokens: 3, outputTokens: 2, totalTokens: 5 },
});
expect(hoisted.completeWithPreparedSimpleCompletionModel).not.toHaveBeenCalled();
});
it("uses the authority-bound profile before the agent-configured profile", async () => {
hoisted.resolveSimpleCompletionSelectionForAgent.mockReturnValueOnce({
provider: "openai",
modelId: "gpt-5.5",
profileId: "openai:configured",
agentDir: "/tmp/main",
});
const llm = createRuntimeLlm({
getConfig: () => cfg,
authority: { allowComplete: true, preferredProfile: "openai:authority-bound" },
});
await expect(
llm.complete({
messages: [{ role: "user", content: "Return JSON" }],
execution: { mode: "isolated-agent-runtime" },
}),
).resolves.toMatchObject({ text: "isolated" });
expect(hoisted.runIsolatedCompletion).toHaveBeenCalledWith(
expect.objectContaining({ authProfileId: "openai:authority-bound" }),
);
});
it("keeps an authorized model profile ahead of the authority-bound profile", async () => {
hoisted.resolveSimpleCompletionSelectionForAgent.mockReturnValueOnce({
provider: "openai",
modelId: "gpt-5.4",
profileId: "openai:model-profile",
agentDir: "/tmp/main",
});
const llm = createRuntimeLlm({
getConfig: () => ({
...cfg,
plugins: {
entries: {
"model-plugin": {
llm: {
allowModelOverride: true,
allowAuthProfileOverride: true,
allowedModels: ["openai/gpt-5.4"],
},
},
},
},
}),
authority: { allowComplete: true, preferredProfile: "openai:authority-bound" },
});
await expect(
withPluginRuntimePluginIdScope("model-plugin", () =>
llm.complete({
model: "openai/gpt-5.4@openai:model-profile",
messages: [{ role: "user", content: "Return JSON" }],
execution: { mode: "isolated-agent-runtime" },
}),
),
).resolves.toMatchObject({ text: "isolated" });
expect(hoisted.runIsolatedCompletion).toHaveBeenCalledWith(
expect.objectContaining({ authProfileId: "openai:model-profile" }),
);
});
it("validates isolated reasoning against the host-resolved model and runtime", async () => {
const llm = createRuntimeLlm({ getConfig: () => cfg, authority: { allowComplete: true } });
await expect(
llm.complete({
messages: [{ role: "user", content: "Return JSON" }],
reasoning: "ultra",
execution: { mode: "isolated-agent-runtime" },
}),
).rejects.toMatchObject({ code: "LLM_ISOLATED_INPUT_REJECTED" });
expect(hoisted.runIsolatedCompletion).not.toHaveBeenCalled();
});
it("denies request-level auth profiles without host policy", async () => {
const llm = createRuntimeLlm({ getConfig: () => cfg, authority: { allowComplete: true } });
await expect(
withPluginRuntimePluginIdScope("plain-plugin", () =>
llm.complete({
messages: [{ role: "user", content: "Return JSON" }],
execution: {
mode: "isolated-agent-runtime",
authProfileId: "openai:work",
},
}),
),
).rejects.toMatchObject({ code: "LLM_COMPLETION_NOT_AUTHORIZED" });
expect(hoisted.runIsolatedCompletion).not.toHaveBeenCalled();
});
it("denies auth profiles selected through a model override without host policy", async () => {
hoisted.resolveSimpleCompletionSelectionForAgent.mockReturnValueOnce({
provider: "openai",
modelId: "gpt-5.4",
profileId: "openai:work",
agentDir: "/tmp/main",
});
const llm = createRuntimeLlm({
getConfig: () => ({
...cfg,
plugins: {
entries: {
"plain-plugin": {
llm: { allowModelOverride: true, allowedModels: ["openai/gpt-5.4"] },
},
},
},
}),
authority: { allowComplete: true },
});
await expect(
withPluginRuntimePluginIdScope("plain-plugin", () =>
llm.complete({
model: "openai/gpt-5.4@openai:work",
messages: [{ role: "user", content: "Return JSON" }],
execution: { mode: "isolated-agent-runtime" },
}),
),
).rejects.toMatchObject({ code: "LLM_COMPLETION_NOT_AUTHORIZED" });
expect(hoisted.runIsolatedCompletion).not.toHaveBeenCalled();
});
it("uses the agent-configured auth profile without treating it as an override", async () => {
hoisted.resolveSimpleCompletionSelectionForAgent.mockReturnValueOnce({
provider: "openai",
modelId: "gpt-5.5",
profileId: "openai:configured",
agentDir: "/tmp/main",
});
const llm = createRuntimeLlm({ getConfig: () => cfg, authority: { allowComplete: true } });
await expect(
llm.complete({
messages: [{ role: "user", content: "Return JSON" }],
execution: { mode: "isolated-agent-runtime" },
}),
).resolves.toMatchObject({ text: "isolated" });
expect(hoisted.runIsolatedCompletion).toHaveBeenCalledWith(
expect.objectContaining({ authProfileId: "openai:configured" }),
);
});
it("does not require profile authority for a model-only override", async () => {
hoisted.resolveSimpleCompletionSelectionForAgent.mockReturnValueOnce({
provider: "openai",
modelId: "gpt-5.4",
profileId: "openai:configured",
agentDir: "/tmp/main",
});
const llm = createRuntimeLlm({
getConfig: () => ({
...cfg,
plugins: {
entries: {
"model-plugin": {
llm: { allowModelOverride: true, allowedModels: ["openai/gpt-5.4"] },
},
},
},
}),
authority: { allowComplete: true },
});
await expect(
withPluginRuntimePluginIdScope("model-plugin", () =>
llm.complete({
model: "openai/gpt-5.4",
messages: [{ role: "user", content: "Return JSON" }],
execution: { mode: "isolated-agent-runtime" },
}),
),
).resolves.toMatchObject({ text: "isolated" });
});
it("rejects chat histories before isolated runtime dispatch", async () => {
const llm = createRuntimeLlm({ getConfig: () => cfg, authority: { allowComplete: true } });
await expect(
llm.complete({
messages: [
{ role: "user", content: "first" },
{ role: "assistant", content: "second" },
],
execution: { mode: "isolated-agent-runtime" },
} as unknown as Parameters<typeof llm.complete>[0]),
).rejects.toMatchObject({ code: "LLM_ISOLATED_INPUT_REJECTED" });
expect(hoisted.runIsolatedCompletion).not.toHaveBeenCalled();
});
it("rejects a missing isolated messages container with the stable input code", async () => {
const llm = createRuntimeLlm({ getConfig: () => cfg, authority: { allowComplete: true } });
await expect(
llm.complete({
execution: { mode: "isolated-agent-runtime" },
} as unknown as Parameters<typeof llm.complete>[0]),
).rejects.toMatchObject({ code: "LLM_ISOLATED_INPUT_REJECTED" });
expect(hoisted.runIsolatedCompletion).not.toHaveBeenCalled();
});
it("rejects unknown execution modes instead of falling through to direct inference", async () => {
const llm = createRuntimeLlm({ getConfig: () => cfg, authority: { allowComplete: true } });
await expect(
llm.complete({
messages: [{ role: "user", content: "Return JSON" }],
execution: { mode: "isoltaed-agent-runtime" },
} as unknown as Parameters<typeof llm.complete>[0]),
).rejects.toMatchObject({ code: "LLM_ISOLATED_INPUT_REJECTED" });
expect(hoisted.runIsolatedCompletion).not.toHaveBeenCalled();
expect(hoisted.completeWithPreparedSimpleCompletionModel).not.toHaveBeenCalled();
});
it.each([2_147_483_648, Number.NaN])("rejects invalid isolated timeout %s", async (timeoutMs) => {
const llm = createRuntimeLlm({ getConfig: () => cfg, authority: { allowComplete: true } });
await expect(
llm.complete({
messages: [{ role: "user", content: "Return JSON" }],
execution: { mode: "isolated-agent-runtime", timeoutMs },
}),
).rejects.toMatchObject({ code: "LLM_ISOLATED_INPUT_REJECTED" });
});
it("settles at the deadline when the isolated runtime ignores cancellation", async () => {
hoisted.runIsolatedCompletion.mockReturnValueOnce(new Promise(() => {}));
const llm = createRuntimeLlm({ getConfig: () => cfg, authority: { allowComplete: true } });
await expect(
llm.complete({
messages: [{ role: "user", content: "Return JSON" }],
execution: { mode: "isolated-agent-runtime", timeoutMs: 5 },
}),
).rejects.toMatchObject({ code: "LLM_COMPLETION_TIMEOUT" });
});
it("settles on caller abort when the isolated runtime ignores cancellation", async () => {
let markStarted: (() => void) | undefined;
const started = new Promise<void>((resolve) => {
markStarted = resolve;
});
hoisted.runIsolatedCompletion.mockImplementationOnce(() => {
markStarted?.();
return new Promise(() => {});
});
const controller = new AbortController();
const llm = createRuntimeLlm({ getConfig: () => cfg, authority: { allowComplete: true } });
const completion = llm.complete({
messages: [{ role: "user", content: "Return JSON" }],
signal: controller.signal,
execution: { mode: "isolated-agent-runtime" },
});
await started;
controller.abort();
await expect(completion).rejects.toMatchObject({ code: "LLM_COMPLETION_ABORTED" });
});
it("maps unsupported isolated runtimes to a stable public error code", async () => {
hoisted.runIsolatedCompletion.mockRejectedValueOnce(
Object.assign(new Error("Agent harness external does not support isolated completion."), {
code: "unsupported",
}),
);
const llm = createRuntimeLlm({ getConfig: () => cfg, authority: { allowComplete: true } });
await expect(
llm.complete({
messages: [{ role: "user", content: "Return JSON" }],
execution: { mode: "isolated-agent-runtime" },
}),
).rejects.toMatchObject({ code: "LLM_ISOLATED_UNSUPPORTED" });
});
it.each([
["input-rejected", "LLM_ISOLATED_INPUT_REJECTED"],
["output-rejected", "LLM_COMPLETION_OUTPUT_REJECTED"],
] as const)("maps %s adapter failures to %s", async (adapterCode, publicCode) => {
hoisted.runIsolatedCompletion.mockRejectedValueOnce(
Object.assign(new Error(`adapter ${adapterCode}`), { code: adapterCode }),
);
const llm = createRuntimeLlm({ getConfig: () => cfg, authority: { allowComplete: true } });
await expect(
llm.complete({
messages: [{ role: "user", content: "Return JSON" }],
execution: { mode: "isolated-agent-runtime" },
}),
).rejects.toMatchObject({ code: publicCode });
});
});
+233
View File
@@ -0,0 +1,233 @@
// Isolated plugin LLM completion policy validates and dispatches the zero-tool runtime mode.
import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion";
import type { IsolatedCompletionResult } from "../../agents/isolated-completion.js";
import { buildConfiguredModelCatalog } from "../../agents/model-selection-shared.js";
import { resolveEffectiveAgentRuntime } from "../../agents/thinking-runtime.js";
import { resolveThinkingProfile } from "../../auto-reply/thinking.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import type {
LlmCompleteErrorCode,
LlmCompleteParams,
LlmIsolatedAgentRuntimeCompleteParams,
} from "./types-core.js";
const MAX_TIMER_DELAY_MS = 2_147_483_647;
function completionError(
code: LlmCompleteErrorCode,
message: string,
cause?: unknown,
): Error & { code: LlmCompleteErrorCode } {
const error = new Error(message, cause === undefined ? undefined : { cause }) as Error & {
code: LlmCompleteErrorCode;
};
error.name = "LlmCompleteError";
error.code = code;
return error;
}
function requireIsolatedUserPrompt(params: LlmCompleteParams): string {
if (
params.execution?.mode !== "isolated-agent-runtime" ||
!Array.isArray(params.messages) ||
params.messages.length !== 1 ||
params.messages[0]?.role !== "user" ||
typeof params.messages[0].content !== "string"
) {
throw completionError(
"LLM_ISOLATED_INPUT_REJECTED",
"Isolated agent-runtime completion requires exactly one user message; pass system instructions through systemPrompt.",
);
}
return params.messages[0].content;
}
export function isIsolatedAgentRuntimeRequest(
params: LlmCompleteParams,
): params is LlmIsolatedAgentRuntimeCompleteParams {
return params.execution?.mode === "isolated-agent-runtime";
}
export function assertSupportedExecutionMode(params: LlmCompleteParams): void {
const execution = (params as { execution?: unknown }).execution;
if (execution === undefined) {
return;
}
if (
!execution ||
typeof execution !== "object" ||
Array.isArray(execution) ||
(execution as { mode?: unknown }).mode !== "isolated-agent-runtime"
) {
throw completionError(
"LLM_ISOLATED_INPUT_REJECTED",
'Plugin LLM completion execution.mode must be "isolated-agent-runtime" when execution is provided.',
);
}
}
function resolveIsolatedTimeoutMs(value: number | undefined): number {
if (value === undefined) {
return 30_000;
}
const timeoutMs = asFiniteNumber(value);
if (
timeoutMs === undefined ||
!Number.isSafeInteger(timeoutMs) ||
timeoutMs <= 0 ||
timeoutMs > MAX_TIMER_DELAY_MS
) {
throw completionError(
"LLM_ISOLATED_INPUT_REJECTED",
`Isolated agent-runtime completion timeoutMs must be an integer from 1 through ${MAX_TIMER_DELAY_MS}.`,
);
}
return timeoutMs;
}
function assertIsolatedReasoningSupported(params: {
cfg: OpenClawConfig;
agentId: string;
provider: string;
model: string;
reasoning: LlmCompleteParams["reasoning"];
}): void {
if (params.reasoning === undefined) {
return;
}
const catalog = buildConfiguredModelCatalog({ cfg: params.cfg });
const profile = resolveThinkingProfile({
provider: params.provider,
model: params.model,
agentRuntime: resolveEffectiveAgentRuntime({
cfg: params.cfg,
agentId: params.agentId,
provider: params.provider,
modelId: params.model,
}),
...(catalog.length > 0 ? { catalog } : {}),
});
if (profile.levels.some((level) => level.id === params.reasoning)) {
return;
}
throw completionError(
"LLM_ISOLATED_INPUT_REJECTED",
`Thinking level "${params.reasoning}" is not supported for ${params.provider}/${params.model}. Use one of: ${profile.levels.map((level) => level.label).join(", ")}.`,
);
}
export async function runIsolatedAgentRuntimeCompletion(params: {
request: LlmIsolatedAgentRuntimeCompleteParams;
cfg: OpenClawConfig;
agentId: string;
provider: string;
model: string;
authProfileId?: string;
}): Promise<IsolatedCompletionResult> {
const prompt = requireIsolatedUserPrompt(params.request);
const timeoutMs = resolveIsolatedTimeoutMs(params.request.execution.timeoutMs);
assertIsolatedReasoningSupported({
cfg: params.cfg,
agentId: params.agentId,
provider: params.provider,
model: params.model,
reasoning: params.request.reasoning,
});
const controller = new AbortController();
let timedOut = false;
const abortFromCaller = () => controller.abort(params.request.signal?.reason);
if (params.request.signal?.aborted) {
throw completionError("LLM_COMPLETION_ABORTED", "Plugin LLM completion was aborted.");
}
params.request.signal?.addEventListener("abort", abortFromCaller, { once: true });
const timer = setTimeout(() => {
timedOut = true;
controller.abort(new Error(`Isolated completion timed out after ${timeoutMs}ms.`));
}, timeoutMs);
timer.unref?.();
let rejectOnAbort: (() => void) | undefined;
const abortPromise = new Promise<never>((_resolve, reject) => {
rejectOnAbort = () => {
const reason = controller.signal.reason;
reject(reason instanceof Error ? reason : new Error("Isolated completion was aborted."));
};
controller.signal.addEventListener("abort", rejectOnAbort, { once: true });
});
try {
const operation = (async () => {
const { runIsolatedCompletion } = await import("../../agents/isolated-completion.js");
return await runIsolatedCompletion({
config: params.cfg,
provider: params.provider,
model: params.model,
authProfileId: params.authProfileId,
agentId: params.agentId,
systemPrompt: params.request.systemPrompt ?? "",
prompt,
timeoutMs,
abortSignal: controller.signal,
thinkLevel: params.request.reasoning,
streamParams: {
maxTokens: asFiniteNumber(params.request.maxTokens),
temperature: asFiniteNumber(params.request.temperature),
},
});
})();
return await Promise.race([operation, abortPromise]);
} catch (error) {
if (timedOut) {
throw completionError(
"LLM_COMPLETION_TIMEOUT",
`Plugin LLM completion timed out after ${timeoutMs}ms.`,
error,
);
}
if (params.request.signal?.aborted) {
throw completionError("LLM_COMPLETION_ABORTED", "Plugin LLM completion was aborted.", error);
}
const isolatedError = error as { code?: unknown; message?: unknown };
if (isolatedError.code === "unsupported") {
throw completionError(
"LLM_ISOLATED_UNSUPPORTED",
typeof isolatedError.message === "string"
? isolatedError.message
: "Configured agent runtime does not support isolated completion.",
error,
);
}
if (isolatedError.code === "runtime-unavailable") {
throw completionError(
"LLM_RUNTIME_UNAVAILABLE",
typeof isolatedError.message === "string"
? isolatedError.message
: "Configured agent runtime is unavailable.",
error,
);
}
if (isolatedError.code === "input-rejected") {
throw completionError(
"LLM_ISOLATED_INPUT_REJECTED",
typeof isolatedError.message === "string"
? isolatedError.message
: "Isolated completion input was rejected.",
error,
);
}
if (isolatedError.code === "output-rejected") {
throw completionError(
"LLM_COMPLETION_OUTPUT_REJECTED",
typeof isolatedError.message === "string"
? isolatedError.message
: "Isolated completion output was rejected.",
error,
);
}
throw completionError("LLM_COMPLETION_FAILED", "Plugin LLM completion failed.", error);
} finally {
clearTimeout(timer);
if (rejectOnAbort) {
controller.signal.removeEventListener("abort", rejectOnAbort);
}
params.request.signal?.removeEventListener("abort", abortFromCaller);
}
}
+194 -2
View File
@@ -549,6 +549,41 @@ describe("runtime.llm.complete", () => {
).rejects.toThrow('model override "openai/gpt-5.5" is not allowlisted');
});
it("requires model overrides to satisfy host and plugin allowlists", async () => {
const llm = createRuntimeLlm({
getConfig: () => ({
...cfg,
plugins: {
entries: {
"restricted-plugin": {
llm: {
allowModelOverride: true,
allowedModels: ["openai/gpt-5.4"],
},
},
},
},
}),
authority: {
allowComplete: true,
allowModelOverride: true,
allowedModels: ["openai/gpt-5.5"],
},
});
await expect(
withPluginRuntimePluginIdScope("restricted-plugin", () =>
llm.complete({
model: "openai/gpt-5.5",
messages: [{ role: "user", content: "Ping" }],
}),
),
).rejects.toThrow(
'model override "openai/gpt-5.5" is not allowlisted for plugin "restricted-plugin"',
);
expect(hoisted.prepareSimpleCompletionModelForAgent).not.toHaveBeenCalled();
});
it("uses runtime-scoped config and the host preparation/dispatch path", async () => {
const logger = createLogger();
const llm = createRuntimeLlm({
@@ -761,11 +796,168 @@ describe("runtime.llm.complete", () => {
await expect(
withPluginRuntimePluginIdScope("trusted-plugin", () =>
llm.complete({
model: "openai/gpt-5.5",
model: "openai/gpt-5.6",
messages: [{ role: "user", content: "Ping" }],
}),
),
).rejects.toThrow('model override "openai/gpt-5.5" is not allowlisted');
).rejects.toThrow('model override "openai/gpt-5.6" is not allowlisted');
});
it("preserves direct model-profile overrides under model authority", async () => {
hoisted.resolveSimpleCompletionSelectionForAgent.mockReturnValueOnce({
provider: "openai",
modelId: "gpt-5.4",
profileId: "openai:work",
agentDir: "/tmp/main",
});
const llm = createRuntimeLlm({
getConfig: () => ({
...cfg,
plugins: {
entries: {
"trusted-plugin": {
llm: {
allowModelOverride: true,
allowedModels: ["openai/gpt-5.4"],
},
},
},
},
}),
authority: { allowComplete: true },
});
await expect(
withPluginRuntimePluginIdScope("trusted-plugin", () =>
llm.complete({
model: "openai/gpt-5.4@openai:work",
messages: [{ role: "user", content: "Ping" }],
}),
),
).resolves.toMatchObject({ text: "done" });
expectSingleCallFirstArg(hoisted.prepareSimpleCompletionModelForAgent, {
agentId: "main",
modelRef: "openai/gpt-5.4@openai:work",
});
});
it("keeps the shipped model allowlist scoped to explicit overrides", async () => {
const llm = createRuntimeLlm({
getConfig: () => ({
...cfg,
plugins: {
entries: {
"restricted-plugin": {
llm: { allowedModels: ["anthropic/claude-haiku-4-5"] },
},
},
},
}),
authority: { allowComplete: true },
});
await expect(
withPluginRuntimePluginIdScope("restricted-plugin", () =>
llm.complete({ messages: [{ role: "user", content: "Ping" }] }),
),
).resolves.toMatchObject({ text: "done" });
});
it("applies a completion model allowlist to the host-resolved default", async () => {
const llm = createRuntimeLlm({
getConfig: () => ({
...cfg,
plugins: {
entries: {
"restricted-plugin": {
llm: { allowedCompletionModels: ["anthropic/claude-haiku-4-5"] },
},
},
},
}),
authority: { allowComplete: true },
});
await expect(
withPluginRuntimePluginIdScope("restricted-plugin", () =>
llm.complete({ messages: [{ role: "user", content: "Ping" }] }),
),
).rejects.toThrow('model "openai/gpt-5.5" is not allowlisted for completions');
expect(hoisted.prepareSimpleCompletionModelForAgent).not.toHaveBeenCalled();
});
it("applies the completion model allowlist to explicit overrides too", async () => {
const llm = createRuntimeLlm({
getConfig: () => ({
...cfg,
plugins: {
entries: {
"restricted-plugin": {
llm: {
allowModelOverride: true,
allowedModels: ["*"],
allowedCompletionModels: ["openai/gpt-5.4"],
},
},
},
},
}),
authority: { allowComplete: true },
});
await expect(
withPluginRuntimePluginIdScope("restricted-plugin", () =>
llm.complete({
model: "openai/gpt-5.6",
messages: [{ role: "user", content: "Ping" }],
}),
),
).rejects.toThrow('model "openai/gpt-5.6" is not allowlisted for completions');
expect(hoisted.prepareSimpleCompletionModelForAgent).not.toHaveBeenCalled();
});
it.each([[[]], [["not-a-canonical-model-ref"]]])(
"fails closed for an unusable completion allowlist %j",
async (allowedCompletionModels) => {
const llm = createRuntimeLlm({
getConfig: () => ({
...cfg,
plugins: {
entries: {
"restricted-plugin": { llm: { allowedCompletionModels } },
},
},
}),
authority: { allowComplete: true },
});
await expect(
withPluginRuntimePluginIdScope("restricted-plugin", () =>
llm.complete({ messages: [{ role: "user", content: "Ping" }] }),
),
).rejects.toThrow("completion model allowlist has no valid models");
expect(hoisted.prepareSimpleCompletionModelForAgent).not.toHaveBeenCalled();
},
);
it("accepts an explicit wildcard completion allowlist", async () => {
const llm = createRuntimeLlm({
getConfig: () => ({
...cfg,
plugins: {
entries: {
"restricted-plugin": { llm: { allowedCompletionModels: ["*"] } },
},
},
}),
authority: { allowComplete: true },
});
await expect(
withPluginRuntimePluginIdScope("restricted-plugin", () =>
llm.complete({ messages: [{ role: "user", content: "Ping" }] }),
),
).resolves.toMatchObject({ text: "done" });
});
it("denies completions when runtime authority disables the capability", async () => {
+284 -67
View File
@@ -2,6 +2,7 @@
import { parseModelCatalogRef } from "@openclaw/model-catalog-core/model-catalog-refs";
import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { splitTrailingAuthProfile } from "../../agents/model-ref-profile.js";
import { modelKey } from "../../agents/model-ref-shared.js";
import { normalizeModelRef } from "../../agents/model-selection.js";
import type { NormalizedUsage, UsageLike } from "../../agents/usage.js";
@@ -13,8 +14,14 @@ import { normalizeAgentId } from "../../routing/session-key.js";
import { estimateUsageCost, resolveModelCostConfig } from "../../utils/usage-format.js";
import { normalizePluginsConfig } from "../config-state.js";
import { getPluginRuntimeGatewayRequestScope } from "./gateway-request-scope.js";
import {
assertSupportedExecutionMode,
isIsolatedAgentRuntimeRequest,
runIsolatedAgentRuntimeCompletion,
} from "./runtime-llm-isolated.js";
import type {
LlmCompleteCaller,
LlmCompleteErrorCode,
LlmCompleteParams,
LlmCompleteResult,
LlmCompleteUsage,
@@ -33,6 +40,8 @@ export type RuntimeLlmAuthority = {
allowAgentIdOverride?: boolean;
allowModelOverride?: boolean;
allowedModels?: readonly string[];
allowedCompletionModels?: readonly string[];
allowAuthProfileOverride?: boolean;
allowComplete?: boolean;
denyReason?: string;
};
@@ -43,12 +52,18 @@ export type CreateRuntimeLlmOptions = {
logger?: RuntimeLogger;
};
type RuntimeLlmOverridePolicy = {
type RuntimeModelAllowlist = {
configured: boolean;
allowAny: boolean;
models: Set<string>;
};
type RuntimeLlmPolicy = {
allowAgentIdOverride: boolean;
allowModelOverride: boolean;
hasConfiguredAllowedModels: boolean;
allowAnyModel: boolean;
allowedModels: Set<string>;
allowAuthProfileOverride: boolean;
overrideModels: RuntimeModelAllowlist;
completionModels: RuntimeModelAllowlist;
};
const defaultLogger = getChildLogger({ capability: "runtime.llm" });
@@ -77,6 +92,19 @@ function normalizeCaller(
};
}
function completionError(
code: LlmCompleteErrorCode,
message: string,
cause?: unknown,
): Error & { code: LlmCompleteErrorCode } {
const error = new Error(message, cause === undefined ? undefined : { cause }) as Error & {
code: LlmCompleteErrorCode;
};
error.name = "LlmCompleteError";
error.code = code;
return error;
}
function resolveTrustedCaller(authority?: RuntimeLlmAuthority): LlmCompleteCaller {
if (authority?.caller?.kind === "context-engine") {
return normalizeCaller(authority.caller);
@@ -108,17 +136,26 @@ async function resolveAgentId(params: {
const authorityAgentId = authorityAgentIdRaw ? normalizeAgentId(authorityAgentIdRaw) : undefined;
const requestedAgentId = requestedAgentIdRaw ? normalizeAgentId(requestedAgentIdRaw) : undefined;
if (params.authority?.requiresBoundAgent && !authorityAgentId) {
throw new Error("Plugin LLM completion is not bound to an active session agent.");
throw completionError(
"LLM_COMPLETION_NOT_AUTHORIZED",
"Plugin LLM completion is not bound to an active session agent.",
);
}
if (authorityAgentId) {
if (requestedAgentId && requestedAgentId !== authorityAgentId && !params.allowAgentIdOverride) {
throw new Error("Plugin LLM completion cannot override the active session agent.");
throw completionError(
"LLM_COMPLETION_NOT_AUTHORIZED",
"Plugin LLM completion cannot override the active session agent.",
);
}
return authorityAgentId;
}
if (requestedAgentId) {
if (!params.allowAgentIdOverride) {
throw new Error("Plugin LLM completion cannot override the target agent.");
throw completionError(
"LLM_COMPLETION_NOT_AUTHORIZED",
"Plugin LLM completion cannot override the target agent.",
);
}
return requestedAgentId;
}
@@ -238,31 +275,47 @@ function normalizeAllowedModelRef(raw: string): string | null {
return modelKey(normalized.provider, normalized.model);
}
function buildPolicyFromEntry(entry: {
allowAgentIdOverride?: boolean;
allowModelOverride?: boolean;
hasAllowedModelsConfig?: boolean;
allowedModels?: readonly string[];
}): RuntimeLlmOverridePolicy {
const allowedModels = new Set<string>();
let allowAnyModel = false;
for (const modelRef of entry.allowedModels ?? []) {
function normalizeModelAllowlist(params: {
configured: boolean;
values?: readonly string[];
}): RuntimeModelAllowlist {
const models = new Set<string>();
let allowAny = false;
for (const modelRef of params.values ?? []) {
const normalizedModelRef = normalizeAllowedModelRef(modelRef);
if (!normalizedModelRef) {
continue;
}
if (normalizedModelRef === "*") {
allowAnyModel = true;
allowAny = true;
continue;
}
allowedModels.add(normalizedModelRef);
models.add(normalizedModelRef);
}
return { configured: params.configured, allowAny, models };
}
function buildPolicyFromEntry(entry: {
allowAgentIdOverride?: boolean;
allowModelOverride?: boolean;
allowAuthProfileOverride?: boolean;
hasAllowedModelsConfig?: boolean;
allowedModels?: readonly string[];
hasAllowedCompletionModelsConfig?: boolean;
allowedCompletionModels?: readonly string[];
}): RuntimeLlmPolicy {
return {
allowAgentIdOverride: entry.allowAgentIdOverride === true,
allowModelOverride: entry.allowModelOverride === true,
hasConfiguredAllowedModels: entry.hasAllowedModelsConfig === true,
allowAnyModel,
allowedModels,
allowAuthProfileOverride: entry.allowAuthProfileOverride === true,
overrideModels: normalizeModelAllowlist({
configured: entry.hasAllowedModelsConfig === true,
values: entry.allowedModels,
}),
completionModels: normalizeModelAllowlist({
configured: entry.hasAllowedCompletionModelsConfig === true,
values: entry.allowedCompletionModels,
}),
};
}
@@ -281,10 +334,10 @@ function resolvePluginPolicyId(
return pluginId;
}
function resolvePluginLlmOverridePolicy(
function resolvePluginLlmPolicy(
cfg: OpenClawConfig,
pluginId: string | undefined,
): RuntimeLlmOverridePolicy | undefined {
): RuntimeLlmPolicy | undefined {
if (!pluginId) {
return undefined;
}
@@ -294,57 +347,138 @@ function resolvePluginLlmOverridePolicy(
function resolveAuthorityModelPolicy(
authority?: RuntimeLlmAuthority,
): RuntimeLlmOverridePolicy | undefined {
): RuntimeLlmPolicy | undefined {
if (
authority?.allowAgentIdOverride !== true &&
authority?.allowModelOverride !== true &&
authority?.allowedModels === undefined
authority?.allowAuthProfileOverride !== true &&
authority?.allowedModels === undefined &&
authority?.allowedCompletionModels === undefined
) {
return undefined;
}
return buildPolicyFromEntry({
allowAgentIdOverride: authority.allowAgentIdOverride,
allowModelOverride: authority.allowModelOverride,
allowAuthProfileOverride: authority.allowAuthProfileOverride,
hasAllowedModelsConfig: authority.allowedModels !== undefined,
allowedModels: authority.allowedModels,
hasAllowedCompletionModelsConfig: authority.allowedCompletionModels !== undefined,
allowedCompletionModels: authority.allowedCompletionModels,
});
}
function assertAllowedAuthProfileOverride(params: {
authProfileId: string | undefined;
authorityPolicy: RuntimeLlmPolicy | undefined;
pluginPolicy: RuntimeLlmPolicy | undefined;
}): void {
if (!params.authProfileId) {
return;
}
if (
params.authorityPolicy?.allowAuthProfileOverride === true ||
params.pluginPolicy?.allowAuthProfileOverride === true
) {
return;
}
throw completionError(
"LLM_COMPLETION_NOT_AUTHORIZED",
"Plugin LLM completion cannot override the auth profile. Enable plugins.entries.<id>.llm.allowAuthProfileOverride to authorize it.",
);
}
function assertOverrideModelAllowed(params: {
resolvedModelRef: string | null;
policy: RuntimeLlmPolicy | undefined;
policyOwnerPluginId?: string;
}): void {
const allowlist = params.policy?.overrideModels;
if (!allowlist?.configured) {
return;
}
if (allowlist.allowAny) {
return;
}
if (allowlist.models.size === 0) {
throw completionError(
"LLM_COMPLETION_NOT_AUTHORIZED",
"Plugin LLM completion model override allowlist has no valid models.",
);
}
if (!params.resolvedModelRef) {
throw completionError(
"LLM_COMPLETION_NOT_AUTHORIZED",
"Plugin LLM completion model override allowlist requires a resolvable provider/model target.",
);
}
if (!allowlist.models.has(params.resolvedModelRef)) {
const owner = params.policyOwnerPluginId ? ` for plugin "${params.policyOwnerPluginId}"` : "";
throw completionError(
"LLM_COMPLETION_NOT_AUTHORIZED",
`Plugin LLM completion model override "${params.resolvedModelRef}" is not allowlisted${owner}.`,
);
}
}
function assertAllowedModelOverride(params: {
resolvedModelRef: string | null;
pluginPolicyId: string | undefined;
authorityPolicy: RuntimeLlmOverridePolicy | undefined;
pluginPolicy: RuntimeLlmOverridePolicy | undefined;
authorityPolicy: RuntimeLlmPolicy | undefined;
pluginPolicy: RuntimeLlmPolicy | undefined;
}): void {
let policy: RuntimeLlmOverridePolicy | undefined;
let policyOwnerPluginId: string | undefined;
if (params.authorityPolicy?.allowModelOverride) {
policy = params.authorityPolicy;
} else if (params.pluginPolicy?.allowModelOverride) {
policy = params.pluginPolicy;
policyOwnerPluginId = params.pluginPolicyId;
}
if (!policy) {
throw new Error("Plugin LLM completion cannot override the target model.");
}
if (policy.allowAnyModel) {
return;
}
if (policy.hasConfiguredAllowedModels && policy.allowedModels.size === 0) {
throw new Error("Plugin LLM completion model override allowlist has no valid models.");
}
if (policy.allowedModels.size === 0) {
return;
}
if (!params.resolvedModelRef) {
throw new Error(
"Plugin LLM completion model override allowlist requires a resolvable provider/model target.",
if (
params.authorityPolicy?.allowModelOverride !== true &&
params.pluginPolicy?.allowModelOverride !== true
) {
throw completionError(
"LLM_COMPLETION_NOT_AUTHORIZED",
"Plugin LLM completion cannot override the target model.",
);
}
if (!policy.allowedModels.has(params.resolvedModelRef)) {
const owner = policyOwnerPluginId ? ` for plugin "${policyOwnerPluginId}"` : "";
throw new Error(
`Plugin LLM completion model override "${params.resolvedModelRef}" is not allowlisted${owner}.`,
// Host and operator policy are independent trust boundaries. When both
// configure a restriction, an override must satisfy their intersection.
assertOverrideModelAllowed({
resolvedModelRef: params.resolvedModelRef,
policy: params.authorityPolicy,
});
assertOverrideModelAllowed({
resolvedModelRef: params.resolvedModelRef,
policy: params.pluginPolicy,
policyOwnerPluginId: params.pluginPolicyId,
});
}
function assertCompletionModelAllowed(params: {
resolvedModelRef: string | null;
policy: RuntimeLlmPolicy | undefined;
policyOwnerPluginId?: string;
}): void {
const policy = params.policy;
const allowlist = policy?.completionModels;
if (!allowlist?.configured) {
return;
}
if (allowlist.allowAny) {
return;
}
if (allowlist.models.size === 0) {
throw completionError(
"LLM_COMPLETION_NOT_AUTHORIZED",
"Plugin LLM completion model allowlist has no valid models.",
);
}
if (!params.resolvedModelRef) {
throw completionError(
"LLM_COMPLETION_NOT_AUTHORIZED",
"Plugin LLM completion model allowlist requires a resolvable provider/model target.",
);
}
if (!allowlist.models.has(params.resolvedModelRef)) {
const owner = params.policyOwnerPluginId ? ` for plugin "${params.policyOwnerPluginId}"` : "";
throw completionError(
"LLM_COMPLETION_NOT_AUTHORIZED",
`Plugin LLM completion model "${params.resolvedModelRef}" is not allowlisted for completions${owner}.`,
);
}
}
@@ -366,8 +500,12 @@ export function createRuntimeLlm(
purpose: params.purpose,
reason,
});
throw new Error(`Plugin LLM completion denied: ${reason}`);
throw completionError(
"LLM_COMPLETION_NOT_AUTHORIZED",
`Plugin LLM completion denied: ${reason}`,
);
}
assertSupportedExecutionMode(params);
const [
{
@@ -381,7 +519,7 @@ export function createRuntimeLlm(
Promise.resolve(resolveRuntimeConfig(options)),
]);
const pluginPolicyId = resolvePluginPolicyId(options.authority, caller);
const pluginPolicy = resolvePluginLlmOverridePolicy(cfg, pluginPolicyId);
const pluginPolicy = resolvePluginLlmPolicy(cfg, pluginPolicyId);
const authorityPolicy = resolveAuthorityModelPolicy(options.authority);
const preferredProfile = normalizeOptionalString(options.authority?.preferredProfile);
const agentId = await resolveAgentId({
@@ -395,18 +533,26 @@ export function createRuntimeLlm(
pluginPolicy?.allowAgentIdOverride === true,
});
const requestedModel = normalizeOptionalString(params.model);
const requestedModelProfile = requestedModel
? normalizeOptionalString(splitTrailingAuthProfile(requestedModel).profile)
: undefined;
const selection = resolveSimpleCompletionSelectionForAgent({
cfg,
agentId,
modelRef: requestedModel,
});
if (!selection) {
throw completionError("LLM_COMPLETION_FAILED", `No model configured for agent ${agentId}.`);
}
const normalizedSelection = normalizeModelRef(selection.provider, selection.modelId);
const resolvedModelRef = modelKey(normalizedSelection.provider, normalizedSelection.model);
assertCompletionModelAllowed({ resolvedModelRef, policy: authorityPolicy });
assertCompletionModelAllowed({
resolvedModelRef,
policy: pluginPolicy,
policyOwnerPluginId: pluginPolicyId,
});
if (requestedModel) {
const selection = resolveSimpleCompletionSelectionForAgent({
cfg,
agentId,
modelRef: requestedModel,
});
const normalizedSelection = selection
? normalizeModelRef(selection.provider, selection.modelId)
: null;
const resolvedModelRef = normalizedSelection
? modelKey(normalizedSelection.provider, normalizedSelection.model)
: null;
assertAllowedModelOverride({
resolvedModelRef,
pluginPolicyId,
@@ -415,6 +561,71 @@ export function createRuntimeLlm(
});
}
const isolatedRequest = isIsolatedAgentRuntimeRequest(params);
const executionProfile = isolatedRequest
? normalizeOptionalString(params.execution.authProfileId)
: undefined;
const modelProfile = normalizeOptionalString(selection.profileId);
if (executionProfile && requestedModelProfile && executionProfile !== requestedModelProfile) {
throw completionError(
"LLM_ISOLATED_INPUT_REJECTED",
"Isolated completion received conflicting auth profiles in model and execution.authProfileId.",
);
}
if (isolatedRequest) {
// Direct completions preserve the shipped model@profile contract under model
// override authority. Isolated credential routing requires separate authority.
assertAllowedAuthProfileOverride({
authProfileId: executionProfile ?? requestedModelProfile,
authorityPolicy,
pluginPolicy,
});
const result = await runIsolatedAgentRuntimeCompletion({
request: params,
cfg,
agentId,
provider: selection.provider,
model: selection.modelId,
// Request-authorized profiles win, then the host/session binding. Only
// an unbound call may fall back to the agent's configured selection.
authProfileId:
executionProfile ?? requestedModelProfile ?? preferredProfile ?? modelProfile,
});
const normalizedUsage = normalizeUsage(result.usage as UsageLike | undefined);
const usage = buildUsage({
rawUsage: result.usage,
normalized: normalizedUsage,
cfg,
provider: result.provider,
model: result.model,
});
logger.info("plugin llm completion", {
caller,
purpose: params.purpose,
sessionKey: options.authority?.sessionKey,
agentId,
provider: result.provider,
model: result.model,
executionMode: params.execution.mode,
executionOwner: result.owner,
usage,
});
return {
text: result.text,
provider: result.provider,
model: result.model,
agentId,
usage,
execution: { mode: params.execution.mode, owner: result.owner },
audit: {
caller,
...(params.purpose ? { purpose: params.purpose } : {}),
...(options.authority?.sessionKey ? { sessionKey: options.authority.sessionKey } : {}),
},
};
}
const prepared = await prepareSimpleCompletionModelForAgent({
cfg,
agentId,
@@ -472,6 +683,8 @@ export function createRuntimeLlm(
agentId,
provider: prepared.selection.provider,
model: prepared.selection.modelId,
executionMode: "direct-provider",
executionOwner: { kind: "provider", id: prepared.selection.provider },
usage,
});
@@ -481,6 +694,10 @@ export function createRuntimeLlm(
model: prepared.selection.modelId,
agentId,
usage,
execution: {
mode: "direct-provider",
owner: { kind: "provider", id: prepared.selection.provider },
},
audit: {
caller,
...(params.purpose ? { purpose: params.purpose } : {}),
+43 -2
View File
@@ -224,11 +224,12 @@ export type LlmCompleteUsage = {
costUsd?: number;
};
export type LlmCompleteParams = {
messages: LlmCompleteMessage[];
type LlmCompleteCommonParams = {
/** Model ref (e.g. "anthropic/claude-sonnet-4-6"); defaults to the target agent's configured model. */
model?: string;
/** Advisory output limit; runtime owners without an equivalent control may ignore it. */
maxTokens?: number;
/** Advisory sampling hint; runtime owners without an equivalent control may ignore it. */
temperature?: number;
/** Requested reasoning effort; the host normalizes it for the selected model. */
reasoning?: import("../../auto-reply/thinking.js").ThinkLevel;
@@ -240,12 +241,52 @@ export type LlmCompleteParams = {
agentId?: string;
};
type LlmDirectCompleteParams = LlmCompleteCommonParams & {
messages: LlmCompleteMessage[];
execution?: undefined;
};
export type LlmIsolatedAgentRuntimeCompleteParams = LlmCompleteCommonParams & {
/** Isolated runtimes currently accept one fresh user prompt, not a replayed chat history. */
messages: [{ role: "user"; content: string }];
execution: {
/** Fresh, literal-zero-tool completion through the configured agent runtime. */
mode: "isolated-agent-runtime";
/** Exact credential owner. Requires host-granted plugin policy. */
authProfileId?: string;
timeoutMs?: number;
};
};
export type LlmCompleteParams = LlmDirectCompleteParams | LlmIsolatedAgentRuntimeCompleteParams;
export type LlmCompleteErrorCode =
| "LLM_COMPLETION_NOT_AUTHORIZED"
| "LLM_ISOLATED_INPUT_REJECTED"
| "LLM_ISOLATED_UNSUPPORTED"
| "LLM_RUNTIME_UNAVAILABLE"
| "LLM_COMPLETION_ABORTED"
| "LLM_COMPLETION_TIMEOUT"
| "LLM_COMPLETION_OUTPUT_REJECTED"
| "LLM_COMPLETION_FAILED";
type LlmCompleteExecution =
| {
mode: "direct-provider";
owner: { kind: "provider"; id: string };
}
| {
mode: "isolated-agent-runtime";
owner: { kind: "cli" | "harness"; id: string };
};
export type LlmCompleteResult = {
text: string;
provider: string;
model: string;
agentId: string;
usage: LlmCompleteUsage;
execution: LlmCompleteExecution;
audit: {
caller: LlmCompleteCaller;
purpose?: string;