From a5237fe925a86507c5b04e4a70711fe63c5be236 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 22:17:57 -0700 Subject: [PATCH] feat(llama-cpp): gate Gemma default by RAM (#109585) --- docs/docs_map.md | 1 + docs/plugins/reference/llama-cpp.md | 15 +++ extensions/llama-cpp/README.md | 10 +- extensions/llama-cpp/index.ts | 4 +- extensions/llama-cpp/openclaw.plugin.json | 2 +- extensions/llama-cpp/src/defaults.ts | 22 +++-- extensions/llama-cpp/src/setup.test.ts | 109 +++++++++++++++++++++- extensions/llama-cpp/src/setup.ts | 74 +++++++++++++-- 8 files changed, 215 insertions(+), 22 deletions(-) diff --git a/docs/docs_map.md b/docs/docs_map.md index 99f393980e2f..90ca95484b85 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -6581,6 +6581,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H1: Llama Cpp plugin - H2: Distribution - H2: Surface + - H2: Default text model - H2: Related docs ## plugins/reference/llm-task.md diff --git a/docs/plugins/reference/llama-cpp.md b/docs/plugins/reference/llama-cpp.md index 6870cbbac00a..48167641b609 100644 --- a/docs/plugins/reference/llama-cpp.md +++ b/docs/plugins/reference/llama-cpp.md @@ -18,6 +18,21 @@ Local GGUF text inference and embeddings through node-llama-cpp. providers: `llama-cpp`; contracts: `embeddingProviders` + + +## Default text model + +During interactive setup, OpenClaw offers Gemma 4 E4B IT Q4_K_M as an +approximately 5.0 GB bundled download. The offer requires at least 16 GiB of +total RAM. Existing cached models are still detected on smaller machines. + +To use another model, set `params.modelPath` to any custom GGUF. Custom models +are not subject to the bundled-download RAM requirement. On machines below the +requirement, you can also run a smaller model through Ollama or LM Studio, or +choose a cloud provider. + + + ## Related docs - [llama-cpp](/plugins/llama-cpp) diff --git a/extensions/llama-cpp/README.md b/extensions/llama-cpp/README.md index 671cceab1354..d2f6487aae44 100644 --- a/extensions/llama-cpp/README.md +++ b/extensions/llama-cpp/README.md @@ -17,8 +17,14 @@ native installs and updates. ## Configure text inference Choose **Local model (llama.cpp)** during onboarding. After explicit consent, -OpenClaw downloads the approximately 2.5 GB Qwen3 4B Instruct 2507 Q4_K_M -default. Discovery never downloads a model. +OpenClaw downloads Gemma 4 E4B IT Q4_K_M (approximately 5.0 GB) as the default. +The bundled download is offered only on machines with at least 16 GiB of RAM. +Discovery never downloads a model. + +On smaller machines, use Ollama or LM Studio with a smaller model, use a cloud +provider, or configure any custom GGUF through `params.modelPath`. The 16 GiB +gate applies only to OpenClaw's bundled default download; custom GGUF models +remain available on any machine. See the [llama.cpp provider guide](https://docs.openclaw.ai/plugins/llama-cpp) for custom GGUF model configuration and hardware guidance. diff --git a/extensions/llama-cpp/index.ts b/extensions/llama-cpp/index.ts index bac2d67f0917..4fa7fe4b3419 100644 --- a/extensions/llama-cpp/index.ts +++ b/extensions/llama-cpp/index.ts @@ -23,7 +23,7 @@ export default definePluginEntry({ { id: "local", label: LLAMA_CPP_PROVIDER_LABEL, - hint: "In-process local GGUF model (about 2.5 GB download)", + hint: "In-process local GGUF model (about 5.0 GB download; requires 16 GB RAM)", kind: "custom", appGuidedSetup: { detect: detectLlamaCppSetup, @@ -57,7 +57,7 @@ export default definePluginEntry({ setup: { choiceId: LLAMA_CPP_PROVIDER_ID, choiceLabel: LLAMA_CPP_PROVIDER_LABEL, - choiceHint: "In-process local model (about 2.5 GB download)", + choiceHint: "In-process local model (about 5.0 GB download; requires 16 GB RAM)", groupId: LLAMA_CPP_PROVIDER_ID, groupLabel: "Local llama.cpp", groupHint: "No API key required", diff --git a/extensions/llama-cpp/openclaw.plugin.json b/extensions/llama-cpp/openclaw.plugin.json index 400fc4a720de..86b208113382 100644 --- a/extensions/llama-cpp/openclaw.plugin.json +++ b/extensions/llama-cpp/openclaw.plugin.json @@ -30,7 +30,7 @@ "choiceId": "llama-cpp", "appGuidedDiscovery": true, "choiceLabel": "Local model (llama.cpp)", - "choiceHint": "Downloads an approximately 2.5 GB local model", + "choiceHint": "Downloads an approximately 5.0 GB local model; requires 16 GB RAM", "groupId": "llama-cpp", "groupLabel": "Local llama.cpp", "groupHint": "No API key required" diff --git a/extensions/llama-cpp/src/defaults.ts b/extensions/llama-cpp/src/defaults.ts index 1d3c06ff45fa..349a269f148a 100644 --- a/extensions/llama-cpp/src/defaults.ts +++ b/extensions/llama-cpp/src/defaults.ts @@ -14,17 +14,25 @@ export function resolveLlamaCppSyntheticApiKey(): string { return LLAMA_CPP_LOCAL_AUTH_MARKER; } -export const DEFAULT_LLAMA_CPP_MODEL_ID = "qwen3-4b-instruct-2507-q4_k_m"; +export const DEFAULT_LLAMA_CPP_MODEL_ID = "gemma-4-e4b-it-q4_k_m"; export const DEFAULT_LLAMA_CPP_MODEL_REF = `${LLAMA_CPP_PROVIDER_ID}/${DEFAULT_LLAMA_CPP_MODEL_ID}`; -// Verified 2026-07-16: 2,497,280,736 bytes (about 2.5 GB) from the public -// bartowski mirror. Qwen does not publish an official Instruct-2507 GGUF repo. +// Verified 2026-07-16: 4,977,169,568 bytes (about 5.0 GB) from the public +// Unsloth Hugging Face repository metadata and response headers. export const DEFAULT_LLAMA_CPP_MODEL_URI = - "hf:bartowski/Qwen_Qwen3-4B-Instruct-2507-GGUF/Qwen_Qwen3-4B-Instruct-2507-Q4_K_M.gguf"; + "hf:unsloth/gemma-4-E4B-it-GGUF/gemma-4-E4B-it-Q4_K_M.gguf"; export const DEFAULT_LLAMA_CPP_MODEL_CACHE_FILE = - "hf_bartowski_Qwen_Qwen3-4B-Instruct-2507-Q4_K_M.gguf"; -export const DEFAULT_LLAMA_CPP_MODEL_SIZE_BYTES = 2_497_280_736; + "hf_unsloth_gemma-4-E4B-it-GGUF_gemma-4-E4B-it-Q4_K_M.gguf"; +export const DEFAULT_LLAMA_CPP_MODEL_SIZE_BYTES = 4_977_169_568; export const DEFAULT_LLAMA_CPP_CONTEXT_SIZE = 8192; +// 5 GB weights + KV cache + OS headroom. Below 16 GiB the bundled default +// thrashes, so the owner decision for 2026-07 is to omit that offer entirely. +const LLAMA_CPP_DEFAULT_MODEL_RAM_FLOOR_BYTES = 16 * 1024 ** 3; + +export function meetsLlamaCppDefaultModelRamFloor(totalmemBytes = os.totalmem()): boolean { + return totalmemBytes >= LLAMA_CPP_DEFAULT_MODEL_RAM_FLOOR_BYTES; +} + export function resolveLlamaCppModelCacheDir(provider?: ModelProviderConfig): string { const configured = provider?.params?.modelCacheDir; return typeof configured === "string" && configured.trim() @@ -77,7 +85,7 @@ export function resolveCachedLlamaCppModelPath(params: { function buildDefaultLlamaCppModel(): ModelDefinitionConfig { return { id: DEFAULT_LLAMA_CPP_MODEL_ID, - name: "Qwen3 4B Instruct 2507 (Q4_K_M)", + name: "Gemma 4 E4B (Q4_K_M)", api: "openai-completions", reasoning: false, input: ["text"], diff --git a/extensions/llama-cpp/src/setup.test.ts b/extensions/llama-cpp/src/setup.test.ts index 3b44cf5b75d7..b8c5a5fcd4d6 100644 --- a/extensions/llama-cpp/src/setup.test.ts +++ b/extensions/llama-cpp/src/setup.test.ts @@ -9,7 +9,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_LLAMA_CPP_MODEL_CACHE_FILE, DEFAULT_LLAMA_CPP_MODEL_REF, + DEFAULT_LLAMA_CPP_MODEL_SIZE_BYTES, + DEFAULT_LLAMA_CPP_MODEL_URI, LLAMA_CPP_PROVIDER_ID, + meetsLlamaCppDefaultModelRamFloor, } from "./defaults.js"; const nodeLlamaMocks = vi.hoisted(() => ({ @@ -27,10 +30,23 @@ vi.mock("node-llama-cpp", () => ({ import { detectLlamaCppSetup, prepareLlamaCppSetup, runLlamaCppSetup } from "./setup.js"; +const { formatLlamaCppDownloadProgress } = (globalThis as Record)[ + Symbol.for("openclaw.llamaCppSetupTestApi") +] as { + formatLlamaCppDownloadProgress: (params: { + downloadedSize: number; + totalSize: number; + bytesPerSecond: number; + }) => string; +}; + +const GIB = 1024 ** 3; + let tempRoot: string; let cacheDir: string; beforeEach(async () => { + vi.spyOn(os, "totalmem").mockReturnValue(16 * GIB); tempRoot = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "llama-cpp-setup-"))); cacheDir = path.join(tempRoot, "models"); await fs.mkdir(cacheDir); @@ -46,6 +62,7 @@ beforeEach(async () => { }); afterEach(async () => { + vi.restoreAllMocks(); await fs.rm(tempRoot, { recursive: true, force: true }); }); @@ -77,6 +94,28 @@ function createAuthContext(confirm: boolean): ProviderAuthContext { } describe("llama.cpp setup", () => { + it("uses the verified Gemma 4 default artifact", () => { + expect(DEFAULT_LLAMA_CPP_MODEL_URI).toBe( + "hf:unsloth/gemma-4-E4B-it-GGUF/gemma-4-E4B-it-Q4_K_M.gguf", + ); + expect(DEFAULT_LLAMA_CPP_MODEL_SIZE_BYTES).toBe(4_977_169_568); + }); + + it("requires 16 GiB for the bundled default offer", () => { + expect(meetsLlamaCppDefaultModelRamFloor(16 * GIB - 1)).toBe(false); + expect(meetsLlamaCppDefaultModelRamFloor(16 * GIB)).toBe(true); + }); + + it("formats percent, decimal GB, and transfer rate", () => { + expect( + formatLlamaCppDownloadProgress({ + downloadedSize: 2_100_000_000, + totalSize: 5_000_000_000, + bytesPerSecond: 38_000_000, + }), + ).toBe("Downloading Gemma 4 E4B… 42% (2.1/5.0 GB, 38 MB/s)"); + }); + it("returns null when the configured model is not cached", async () => { await expect(detectLlamaCppSetup({ config: configWithCache(), env: {} })).resolves.toBeNull(); }); @@ -86,7 +125,7 @@ describe("llama.cpp setup", () => { await expect(detectLlamaCppSetup({ config: configWithCache(), env: {} })).resolves.toEqual({ modelRef: DEFAULT_LLAMA_CPP_MODEL_REF, - detail: "qwen3-4b-instruct-2507-q4_k_m (downloaded)", + detail: "gemma-4-e4b-it-q4_k_m (downloaded)", }); expect(nodeLlamaMocks.createModelDownloader).not.toHaveBeenCalled(); expect(nodeLlamaMocks.resolveModelFile).toHaveBeenCalledWith( @@ -96,6 +135,7 @@ describe("llama.cpp setup", () => { }); it("uses node-llama-cpp cache resolution for a configured HF branch", async () => { + vi.mocked(os.totalmem).mockReturnValue(8 * GIB); const cachedPath = path.join(cacheDir, "hf_org_repo_release_model.gguf"); await fs.writeFile(cachedPath, "fixture"); nodeLlamaMocks.resolveModelFile.mockResolvedValueOnce(cachedPath); @@ -150,7 +190,16 @@ describe("llama.cpp setup", () => { providers: { [LLAMA_CPP_PROVIDER_ID]: { baseUrl: "local://llama-cpp", - models: [expect.objectContaining({ id: "qwen3-4b-instruct-2507-q4_k_m" })], + models: [ + expect.objectContaining({ + id: "gemma-4-e4b-it-q4_k_m", + name: "Gemma 4 E4B (Q4_K_M)", + contextWindow: 8192, + contextTokens: 8192, + maxTokens: 2048, + compat: expect.objectContaining({ supportsTools: true }), + }), + ], }, }, }, @@ -158,13 +207,40 @@ describe("llama.cpp setup", () => { }); }); - it("exits without config or download when consent is declined", async () => { + it("skips the bundled offer below the RAM floor", async () => { + vi.mocked(os.totalmem).mockReturnValue(8 * GIB); + const ctx = createAuthContext(true); + + await expect(runLlamaCppSetup(ctx)).resolves.toEqual({ profiles: [] }); + + expect(ctx.prompter.confirm).not.toHaveBeenCalled(); + expect(ctx.prompter.note).toHaveBeenCalledWith( + "This machine has 8 GB RAM; the bundled local model needs 16 GB+. Use Ollama/LM Studio with a smaller model, or a cloud provider.", + "Setup skipped", + ); + expect(nodeLlamaMocks.createModelDownloader).not.toHaveBeenCalled(); + }); + + it("honors a cached default below the RAM floor", async () => { + vi.mocked(os.totalmem).mockReturnValue(8 * GIB); + await fs.writeFile(path.join(cacheDir, DEFAULT_LLAMA_CPP_MODEL_CACHE_FILE), "fixture"); + const ctx = createAuthContext(true); + + await expect(runLlamaCppSetup(ctx)).resolves.toMatchObject({ + defaultModel: DEFAULT_LLAMA_CPP_MODEL_REF, + }); + + expect(ctx.prompter.confirm).not.toHaveBeenCalled(); + expect(ctx.prompter.note).not.toHaveBeenCalled(); + }); + + it("keeps the consent path at the RAM floor", async () => { const ctx = createAuthContext(false); await expect(runLlamaCppSetup(ctx)).resolves.toEqual({ profiles: [] }); expect(ctx.prompter.confirm).toHaveBeenCalledWith( - expect.objectContaining({ message: expect.stringContaining("about 2.5 GB") }), + expect.objectContaining({ message: expect.stringContaining("about 5.0 GB") }), ); expect(nodeLlamaMocks.createModelDownloader).not.toHaveBeenCalled(); }); @@ -195,4 +271,29 @@ describe("llama.cpp setup", () => { ); expect(nodeLlamaMocks.download).toHaveBeenCalledTimes(1); }); + + it("calculates rolling rate from download deltas without counting resumed bytes", async () => { + const update = vi.fn(); + const ctx = createAuthContext(true); + vi.mocked(ctx.prompter.progress).mockReturnValue({ update, stop: vi.fn() }); + vi.spyOn(Date, "now").mockReturnValueOnce(1_000).mockReturnValueOnce(2_000); + nodeLlamaMocks.createModelDownloader.mockImplementationOnce( + async (options: { + onProgress?: (status: { downloadedSize: number; totalSize: number }) => void; + }) => ({ + download: vi.fn(async () => { + options.onProgress?.({ downloadedSize: 2_000_000_000, totalSize: 5_000_000_000 }); + options.onProgress?.({ downloadedSize: 2_100_000_000, totalSize: 5_000_000_000 }); + }), + }), + ); + + await runLlamaCppSetup(ctx); + + expect(update).toHaveBeenNthCalledWith(1, "Downloading Gemma 4 E4B… 40% (2.0/5.0 GB, 0 MB/s)"); + expect(update).toHaveBeenNthCalledWith( + 2, + "Downloading Gemma 4 E4B… 42% (2.1/5.0 GB, 100 MB/s)", + ); + }); }); diff --git a/extensions/llama-cpp/src/setup.ts b/extensions/llama-cpp/src/setup.ts index f4c92dd0b672..f25139196d36 100644 --- a/extensions/llama-cpp/src/setup.ts +++ b/extensions/llama-cpp/src/setup.ts @@ -1,4 +1,5 @@ import fs from "node:fs/promises"; +import os from "node:os"; import type { ProviderAppGuidedSetupContext, ProviderAuthContext, @@ -16,6 +17,7 @@ import { DEFAULT_LLAMA_CPP_MODEL_URI, LLAMA_CPP_PROVIDER_ID, buildLlamaCppProviderConfig, + meetsLlamaCppDefaultModelRamFloor, resolveCachedLlamaCppModelPath, resolveLlamaCppModelCacheDir, resolveLlamaCppModelSource, @@ -26,6 +28,27 @@ import { type NodeLlamaCppModule, } from "./node-llama.runtime.js"; +const BYTES_PER_GB = 1_000_000_000; +const BYTES_PER_MB = 1_000_000; + +function formatLlamaCppDownloadProgress(params: { + downloadedSize: number; + totalSize: number; + bytesPerSecond: number; +}): string { + const downloadedSize = Math.max(0, params.downloadedSize); + const totalSize = Math.max(1, params.totalSize); + const percent = Math.min(100, Math.floor((downloadedSize / totalSize) * 100)); + const downloadedGb = (downloadedSize / BYTES_PER_GB).toFixed(1); + const totalGb = (totalSize / BYTES_PER_GB).toFixed(1); + const rateMb = Math.max(0, Math.round(params.bytesPerSecond / BYTES_PER_MB)); + return `Downloading Gemma 4 E4B… ${percent}% (${downloadedGb}/${totalGb} GB, ${rateMb} MB/s)`; +} + +function formatRamGb(totalmemBytes: number): string { + return (totalmemBytes / 1024 ** 3).toFixed(1).replace(/\.0$/, ""); +} + function readPrimaryModel(config: ProviderAppGuidedSetupContext["config"]): string | undefined { const model = config.agents?.defaults?.model; return typeof model === "string" ? model : model?.primary; @@ -122,31 +145,64 @@ export async function runLlamaCppSetup(ctx: ProviderAuthContext): Promise { + const now = Date.now(); + if ( + previousDownloadedSize !== undefined && + previousProgressAtMs !== undefined && + downloadedSize >= previousDownloadedSize && + now > previousProgressAtMs + ) { + const elapsedSeconds = (now - previousProgressAtMs) / 1000; + const currentBytesPerSecond = + (downloadedSize - previousDownloadedSize) / elapsedSeconds; + // Four-sample EWMA: a small rolling window without per-update allocations. + rollingBytesPerSecond = + rollingBytesPerSecond === 0 + ? currentBytesPerSecond + : rollingBytesPerSecond * 0.75 + currentBytesPerSecond * 0.25; + } + previousDownloadedSize = downloadedSize; + previousProgressAtMs = now; const expectedSize = totalSize || DEFAULT_LLAMA_CPP_MODEL_SIZE_BYTES; - const percent = Math.min(100, Math.floor((downloadedSize / expectedSize) * 100)); - progress.update(`Downloading Qwen3 4B model… ${percent}%`); + progress.update( + formatLlamaCppDownloadProgress({ + downloadedSize, + totalSize: expectedSize, + bytesPerSecond: rollingBytesPerSecond, + }), + ); }, }); await downloader.download({ signal: ctx.signal }); - progress.stop("Qwen3 4B model downloaded"); + progress.stop("Gemma 4 E4B model downloaded"); } catch (error) { progress.stop("Model download failed"); throw new Error(formatLlamaCppSetupError(error), { cause: error }); @@ -154,3 +210,9 @@ export async function runLlamaCppSetup(ctx: ProviderAuthContext): Promise)[Symbol.for("openclaw.llamaCppSetupTestApi")] = { + formatLlamaCppDownloadProgress, + }; +}