fix(llama-cpp): clarify local model setup

This commit is contained in:
Vincent Koc
2026-08-02 18:04:25 +08:00
parent 0bc32f0f5c
commit bce957fe61
15 changed files with 63 additions and 44 deletions
+1 -1
View File
@@ -75,7 +75,7 @@ describe("llama.cpp provider plugin", () => {
expect(registerProvider).toHaveBeenCalledWith(
expect.objectContaining({
id: "llama-cpp",
label: "Local model (llama.cpp)",
label: "llama.cpp",
createStreamFn: expect.any(Function),
normalizeToolSchemas: expect.any(Function),
inspectToolSchemas: expect.any(Function),
+4 -4
View File
@@ -24,7 +24,7 @@ export default definePluginEntry({
{
id: "local",
label: LLAMA_CPP_PROVIDER_LABEL,
hint: "In-process local GGUF model (about 5.0 GB download; requires 16 GB RAM)",
hint: "Run one private GGUF model directly inside this Gateway",
kind: "custom",
appGuidedSetup: {
detect: detectLlamaCppSetup,
@@ -59,15 +59,15 @@ export default definePluginEntry({
setup: {
choiceId: LLAMA_CPP_PROVIDER_ID,
choiceLabel: LLAMA_CPP_PROVIDER_LABEL,
choiceHint: "In-process local model (about 5.0 GB download; requires 16 GB RAM)",
choiceHint: "Run one private GGUF model directly inside this Gateway",
groupId: LLAMA_CPP_PROVIDER_ID,
groupLabel: "Local llama.cpp",
groupHint: "No API key required",
methodId: "local",
},
modelPicker: {
label: "llama.cpp (local GGUF)",
hint: "Run a GGUF model in the OpenClaw process",
label: "llama.cpp",
hint: "Run a GGUF model directly inside OpenClaw",
methodId: "local",
},
},
+3 -3
View File
@@ -29,9 +29,9 @@
"method": "local",
"choiceId": "llama-cpp",
"appGuidedDiscovery": true,
"appGuidedActionLabel": "Review download",
"choiceLabel": "Local model (llama.cpp)",
"choiceHint": "Downloads an approximately 5.0 GB local model; requires 16 GB RAM",
"appGuidedActionLabel": "Set up model",
"choiceLabel": "llama.cpp",
"choiceHint": "Run one private GGUF model directly inside this Gateway",
"groupId": "llama-cpp",
"groupLabel": "Local llama.cpp",
"groupHint": "No API key required"
+1 -1
View File
@@ -6,7 +6,7 @@ import type {
} from "openclaw/plugin-sdk/provider-model-shared";
export const LLAMA_CPP_PROVIDER_ID = "llama-cpp";
export const LLAMA_CPP_PROVIDER_LABEL = "Local model (llama.cpp)";
export const LLAMA_CPP_PROVIDER_LABEL = "llama.cpp";
const LLAMA_CPP_LOCAL_AUTH_MARKER = "llama-cpp-local";
const LLAMA_CPP_LOCAL_BASE_URL = "local://llama-cpp";
+6 -4
View File
@@ -125,7 +125,7 @@ describe("llama.cpp setup", () => {
await expect(detectLlamaCppSetup({ config: configWithCache(), env: {} })).resolves.toEqual({
modelRef: DEFAULT_LLAMA_CPP_MODEL_REF,
detail: "gemma-4-e4b-it-q4_k_m (downloaded)",
detail: "Ready locally",
});
expect(nodeLlamaMocks.createModelDownloader).not.toHaveBeenCalled();
expect(nodeLlamaMocks.resolveModelFile).toHaveBeenCalledWith(
@@ -157,7 +157,7 @@ describe("llama.cpp setup", () => {
await expect(detectLlamaCppSetup({ config, env: {} })).resolves.toEqual({
modelRef: "llama-cpp/custom",
detail: "custom (downloaded)",
detail: "Ready locally",
});
expect(nodeLlamaMocks.resolveModelFile).toHaveBeenCalledWith(
"hf:org/repo/model.gguf#release",
@@ -215,7 +215,7 @@ describe("llama.cpp setup", () => {
expect(ctx.prompter.confirm).not.toHaveBeenCalled();
expect(ctx.prompter.note).toHaveBeenCalledWith(
"This machine has 8 GB RAM; the bundled local model needs 16 GB+. Use Ollama/LM Studio with a smaller model, or a cloud provider.",
"This Gateway has 8 GB RAM; the recommended model needs 16 GB+. Use Ollama or LM Studio with a smaller model, configure an existing GGUF, or choose a cloud provider.",
"Setup skipped",
);
expect(nodeLlamaMocks.createModelDownloader).not.toHaveBeenCalled();
@@ -240,7 +240,9 @@ describe("llama.cpp setup", () => {
await expect(runLlamaCppSetup(ctx)).resolves.toEqual({ profiles: [] });
expect(ctx.prompter.confirm).toHaveBeenCalledWith(
expect.objectContaining({ message: expect.stringContaining("about 5.0 GB") }),
expect.objectContaining({
message: expect.stringContaining("run it directly inside this Gateway"),
}),
);
expect(nodeLlamaMocks.createModelDownloader).not.toHaveBeenCalled();
});
+4 -3
View File
@@ -98,7 +98,7 @@ export async function detectLlamaCppSetup(ctx: ProviderAppGuidedSetupContext) {
}
return {
modelRef: `${LLAMA_CPP_PROVIDER_ID}/${candidate.model.id}`,
detail: `${candidate.model.id} (downloaded)`,
detail: "Ready locally",
};
} catch {
// Discovery is read-only: a missing model or native module is not a setup error.
@@ -148,13 +148,14 @@ export async function runLlamaCppSetup(ctx: ProviderAuthContext): Promise<Provid
const totalmemBytes = os.totalmem();
if (!meetsLlamaCppDefaultModelRamFloor(totalmemBytes)) {
await ctx.prompter.note(
`This machine has ${formatRamGb(totalmemBytes)} GB RAM; the bundled local model needs 16 GB+. Use Ollama/LM Studio with a smaller model, or a cloud provider.`,
`This Gateway has ${formatRamGb(totalmemBytes)} GB RAM; the recommended model needs 16 GB+. Use Ollama or LM Studio with a smaller model, configure an existing GGUF, or choose a cloud provider.`,
"Setup skipped",
);
return { profiles: [] };
}
const consent = await ctx.prompter.confirm({
message: "Download Gemma 4 E4B IT Q4_K_M (about 5.0 GB) for local llama.cpp inference?",
message:
"OpenClaw will download Gemma 4 E4B IT Q4_K_M (about 5.0 GB) and run it directly inside this Gateway. Continue?",
initialValue: false,
});
if (!consent) {