feat(llama-cpp): gate Gemma default by RAM (#109585)

2026-08-12 21:53:00 -06:00 · 2026-07-16 22:17:57 -07:00
parent 5199bfafea
commit a5237fe925
8 changed files with 215 additions and 22 deletions
@@ -6581,6 +6581,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
  - H1: Llama Cpp plugin
  - H2: Distribution
  - H2: Surface
+  - H2: Default text model
  - H2: Related docs

 ## plugins/reference/llm-task.md
@@ -18,6 +18,21 @@ Local GGUF text inference and embeddings through node-llama-cpp.

 providers: `llama-cpp`; contracts: `embeddingProviders`

+<!-- openclaw-plugin-reference:manual-start -->
+
+## Default text model
+
+During interactive setup, OpenClaw offers Gemma 4 E4B IT Q4_K_M as an
+approximately 5.0 GB bundled download. The offer requires at least 16 GiB of
+total RAM. Existing cached models are still detected on smaller machines.
+
+To use another model, set `params.modelPath` to any custom GGUF. Custom models
+are not subject to the bundled-download RAM requirement. On machines below the
+requirement, you can also run a smaller model through Ollama or LM Studio, or
+choose a cloud provider.
+
+<!-- openclaw-plugin-reference:manual-end -->
+
 ## Related docs

 - [llama-cpp](/plugins/llama-cpp)
@@ -17,8 +17,14 @@ native installs and updates.
 ## Configure text inference

 Choose **Local model (llama.cpp)** during onboarding. After explicit consent,
-OpenClaw downloads the approximately 2.5 GB Qwen3 4B Instruct 2507 Q4_K_M
-default. Discovery never downloads a model.
+OpenClaw downloads Gemma 4 E4B IT Q4_K_M (approximately 5.0 GB) as the default.
+The bundled download is offered only on machines with at least 16 GiB of RAM.
+Discovery never downloads a model.
+
+On smaller machines, use Ollama or LM Studio with a smaller model, use a cloud
+provider, or configure any custom GGUF through `params.modelPath`. The 16 GiB
+gate applies only to OpenClaw's bundled default download; custom GGUF models
+remain available on any machine.

 See the [llama.cpp provider guide](https://docs.openclaw.ai/plugins/llama-cpp)
 for custom GGUF model configuration and hardware guidance.
@@ -23,7 +23,7 @@ export default definePluginEntry({
        {
          id: "local",
          label: LLAMA_CPP_PROVIDER_LABEL,
-          hint: "In-process local GGUF model (about 2.5 GB download)",
+          hint: "In-process local GGUF model (about 5.0 GB download; requires 16 GB RAM)",
          kind: "custom",
          appGuidedSetup: {
            detect: detectLlamaCppSetup,
@@ -57,7 +57,7 @@ export default definePluginEntry({
        setup: {
          choiceId: LLAMA_CPP_PROVIDER_ID,
          choiceLabel: LLAMA_CPP_PROVIDER_LABEL,
-          choiceHint: "In-process local model (about 2.5 GB download)",
+          choiceHint: "In-process local model (about 5.0 GB download; requires 16 GB RAM)",
          groupId: LLAMA_CPP_PROVIDER_ID,
          groupLabel: "Local llama.cpp",
          groupHint: "No API key required",
@@ -30,7 +30,7 @@
      "choiceId": "llama-cpp",
      "appGuidedDiscovery": true,
      "choiceLabel": "Local model (llama.cpp)",
-      "choiceHint": "Downloads an approximately 2.5 GB local model",
+      "choiceHint": "Downloads an approximately 5.0 GB local model; requires 16 GB RAM",
      "groupId": "llama-cpp",
      "groupLabel": "Local llama.cpp",
      "groupHint": "No API key required"
@@ -14,17 +14,25 @@ export function resolveLlamaCppSyntheticApiKey(): string {
  return LLAMA_CPP_LOCAL_AUTH_MARKER;
 }

-export const DEFAULT_LLAMA_CPP_MODEL_ID = "qwen3-4b-instruct-2507-q4_k_m";
+export const DEFAULT_LLAMA_CPP_MODEL_ID = "gemma-4-e4b-it-q4_k_m";
 export const DEFAULT_LLAMA_CPP_MODEL_REF = `${LLAMA_CPP_PROVIDER_ID}/${DEFAULT_LLAMA_CPP_MODEL_ID}`;
-// Verified 2026-07-16: 2,497,280,736 bytes (about 2.5 GB) from the public
-// bartowski mirror. Qwen does not publish an official Instruct-2507 GGUF repo.
+// Verified 2026-07-16: 4,977,169,568 bytes (about 5.0 GB) from the public
+// Unsloth Hugging Face repository metadata and response headers.
 export const DEFAULT_LLAMA_CPP_MODEL_URI =
-  "hf:bartowski/Qwen_Qwen3-4B-Instruct-2507-GGUF/Qwen_Qwen3-4B-Instruct-2507-Q4_K_M.gguf";
+  "hf:unsloth/gemma-4-E4B-it-GGUF/gemma-4-E4B-it-Q4_K_M.gguf";
 export const DEFAULT_LLAMA_CPP_MODEL_CACHE_FILE =
-  "hf_bartowski_Qwen_Qwen3-4B-Instruct-2507-Q4_K_M.gguf";
-export const DEFAULT_LLAMA_CPP_MODEL_SIZE_BYTES = 2_497_280_736;
+  "hf_unsloth_gemma-4-E4B-it-GGUF_gemma-4-E4B-it-Q4_K_M.gguf";
+export const DEFAULT_LLAMA_CPP_MODEL_SIZE_BYTES = 4_977_169_568;
 export const DEFAULT_LLAMA_CPP_CONTEXT_SIZE = 8192;

+// 5 GB weights + KV cache + OS headroom. Below 16 GiB the bundled default
+// thrashes, so the owner decision for 2026-07 is to omit that offer entirely.
+const LLAMA_CPP_DEFAULT_MODEL_RAM_FLOOR_BYTES = 16 * 1024 ** 3;
+
+export function meetsLlamaCppDefaultModelRamFloor(totalmemBytes = os.totalmem()): boolean {
+  return totalmemBytes >= LLAMA_CPP_DEFAULT_MODEL_RAM_FLOOR_BYTES;
+}
+
 export function resolveLlamaCppModelCacheDir(provider?: ModelProviderConfig): string {
  const configured = provider?.params?.modelCacheDir;
  return typeof configured === "string" && configured.trim()
@@ -77,7 +85,7 @@ export function resolveCachedLlamaCppModelPath(params: {
 function buildDefaultLlamaCppModel(): ModelDefinitionConfig {
  return {
    id: DEFAULT_LLAMA_CPP_MODEL_ID,
-    name: "Qwen3 4B Instruct 2507 (Q4_K_M)",
+    name: "Gemma 4 E4B (Q4_K_M)",
    api: "openai-completions",
    reasoning: false,
    input: ["text"],
@@ -9,7 +9,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import {
  DEFAULT_LLAMA_CPP_MODEL_CACHE_FILE,
  DEFAULT_LLAMA_CPP_MODEL_REF,
+  DEFAULT_LLAMA_CPP_MODEL_SIZE_BYTES,
+  DEFAULT_LLAMA_CPP_MODEL_URI,
  LLAMA_CPP_PROVIDER_ID,
+  meetsLlamaCppDefaultModelRamFloor,
 } from "./defaults.js";

 const nodeLlamaMocks = vi.hoisted(() => ({
@@ -27,10 +30,23 @@ vi.mock("node-llama-cpp", () => ({

 import { detectLlamaCppSetup, prepareLlamaCppSetup, runLlamaCppSetup } from "./setup.js";

+const { formatLlamaCppDownloadProgress } = (globalThis as Record<PropertyKey, unknown>)[
+  Symbol.for("openclaw.llamaCppSetupTestApi")
+] as {
+  formatLlamaCppDownloadProgress: (params: {
+    downloadedSize: number;
+    totalSize: number;
+    bytesPerSecond: number;
+  }) => string;
+};
+
+const GIB = 1024 ** 3;
+
 let tempRoot: string;
 let cacheDir: string;

 beforeEach(async () => {
+  vi.spyOn(os, "totalmem").mockReturnValue(16 * GIB);
  tempRoot = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "llama-cpp-setup-")));
  cacheDir = path.join(tempRoot, "models");
  await fs.mkdir(cacheDir);
@@ -46,6 +62,7 @@ beforeEach(async () => {
 });

 afterEach(async () => {
+  vi.restoreAllMocks();
  await fs.rm(tempRoot, { recursive: true, force: true });
 });

@@ -77,6 +94,28 @@ function createAuthContext(confirm: boolean): ProviderAuthContext {
 }

 describe("llama.cpp setup", () => {
+  it("uses the verified Gemma 4 default artifact", () => {
+    expect(DEFAULT_LLAMA_CPP_MODEL_URI).toBe(
+      "hf:unsloth/gemma-4-E4B-it-GGUF/gemma-4-E4B-it-Q4_K_M.gguf",
+    );
+    expect(DEFAULT_LLAMA_CPP_MODEL_SIZE_BYTES).toBe(4_977_169_568);
+  });
+
+  it("requires 16 GiB for the bundled default offer", () => {
+    expect(meetsLlamaCppDefaultModelRamFloor(16 * GIB - 1)).toBe(false);
+    expect(meetsLlamaCppDefaultModelRamFloor(16 * GIB)).toBe(true);
+  });
+
+  it("formats percent, decimal GB, and transfer rate", () => {
+    expect(
+      formatLlamaCppDownloadProgress({
+        downloadedSize: 2_100_000_000,
+        totalSize: 5_000_000_000,
+        bytesPerSecond: 38_000_000,
+      }),
+    ).toBe("Downloading Gemma 4 E4B… 42% (2.1/5.0 GB, 38 MB/s)");
+  });
+
  it("returns null when the configured model is not cached", async () => {
    await expect(detectLlamaCppSetup({ config: configWithCache(), env: {} })).resolves.toBeNull();
  });
@@ -86,7 +125,7 @@ describe("llama.cpp setup", () => {

    await expect(detectLlamaCppSetup({ config: configWithCache(), env: {} })).resolves.toEqual({
      modelRef: DEFAULT_LLAMA_CPP_MODEL_REF,
-      detail: "qwen3-4b-instruct-2507-q4_k_m (downloaded)",
+      detail: "gemma-4-e4b-it-q4_k_m (downloaded)",
    });
    expect(nodeLlamaMocks.createModelDownloader).not.toHaveBeenCalled();
    expect(nodeLlamaMocks.resolveModelFile).toHaveBeenCalledWith(
@@ -96,6 +135,7 @@ describe("llama.cpp setup", () => {
  });

  it("uses node-llama-cpp cache resolution for a configured HF branch", async () => {
+    vi.mocked(os.totalmem).mockReturnValue(8 * GIB);
    const cachedPath = path.join(cacheDir, "hf_org_repo_release_model.gguf");
    await fs.writeFile(cachedPath, "fixture");
    nodeLlamaMocks.resolveModelFile.mockResolvedValueOnce(cachedPath);
@@ -150,7 +190,16 @@ describe("llama.cpp setup", () => {
          providers: {
            [LLAMA_CPP_PROVIDER_ID]: {
              baseUrl: "local://llama-cpp",
-              models: [expect.objectContaining({ id: "qwen3-4b-instruct-2507-q4_k_m" })],
+              models: [
+                expect.objectContaining({
+                  id: "gemma-4-e4b-it-q4_k_m",
+                  name: "Gemma 4 E4B (Q4_K_M)",
+                  contextWindow: 8192,
+                  contextTokens: 8192,
+                  maxTokens: 2048,
+                  compat: expect.objectContaining({ supportsTools: true }),
+                }),
+              ],
            },
          },
        },
@@ -158,13 +207,40 @@ describe("llama.cpp setup", () => {
    });
  });

-  it("exits without config or download when consent is declined", async () => {
+  it("skips the bundled offer below the RAM floor", async () => {
+    vi.mocked(os.totalmem).mockReturnValue(8 * GIB);
+    const ctx = createAuthContext(true);
+
+    await expect(runLlamaCppSetup(ctx)).resolves.toEqual({ profiles: [] });
+
+    expect(ctx.prompter.confirm).not.toHaveBeenCalled();
+    expect(ctx.prompter.note).toHaveBeenCalledWith(
+      "This machine has 8 GB RAM; the bundled local model needs 16 GB+. Use Ollama/LM Studio with a smaller model, or a cloud provider.",
+      "Setup skipped",
+    );
+    expect(nodeLlamaMocks.createModelDownloader).not.toHaveBeenCalled();
+  });
+
+  it("honors a cached default below the RAM floor", async () => {
+    vi.mocked(os.totalmem).mockReturnValue(8 * GIB);
+    await fs.writeFile(path.join(cacheDir, DEFAULT_LLAMA_CPP_MODEL_CACHE_FILE), "fixture");
+    const ctx = createAuthContext(true);
+
+    await expect(runLlamaCppSetup(ctx)).resolves.toMatchObject({
+      defaultModel: DEFAULT_LLAMA_CPP_MODEL_REF,
+    });
+
+    expect(ctx.prompter.confirm).not.toHaveBeenCalled();
+    expect(ctx.prompter.note).not.toHaveBeenCalled();
+  });
+
+  it("keeps the consent path at the RAM floor", async () => {
    const ctx = createAuthContext(false);

    await expect(runLlamaCppSetup(ctx)).resolves.toEqual({ profiles: [] });

    expect(ctx.prompter.confirm).toHaveBeenCalledWith(
-      expect.objectContaining({ message: expect.stringContaining("about 2.5 GB") }),
+      expect.objectContaining({ message: expect.stringContaining("about 5.0 GB") }),
    );
    expect(nodeLlamaMocks.createModelDownloader).not.toHaveBeenCalled();
  });
@@ -195,4 +271,29 @@ describe("llama.cpp setup", () => {
    );
    expect(nodeLlamaMocks.download).toHaveBeenCalledTimes(1);
  });
+
+  it("calculates rolling rate from download deltas without counting resumed bytes", async () => {
+    const update = vi.fn();
+    const ctx = createAuthContext(true);
+    vi.mocked(ctx.prompter.progress).mockReturnValue({ update, stop: vi.fn() });
+    vi.spyOn(Date, "now").mockReturnValueOnce(1_000).mockReturnValueOnce(2_000);
+    nodeLlamaMocks.createModelDownloader.mockImplementationOnce(
+      async (options: {
+        onProgress?: (status: { downloadedSize: number; totalSize: number }) => void;
+      }) => ({
+        download: vi.fn(async () => {
+          options.onProgress?.({ downloadedSize: 2_000_000_000, totalSize: 5_000_000_000 });
+          options.onProgress?.({ downloadedSize: 2_100_000_000, totalSize: 5_000_000_000 });
+        }),
+      }),
+    );
+
+    await runLlamaCppSetup(ctx);
+
+    expect(update).toHaveBeenNthCalledWith(1, "Downloading Gemma 4 E4B… 40% (2.0/5.0 GB, 0 MB/s)");
+    expect(update).toHaveBeenNthCalledWith(
+      2,
+      "Downloading Gemma 4 E4B… 42% (2.1/5.0 GB, 100 MB/s)",
+    );
+  });
 });
@@ -1,4 +1,5 @@
 import fs from "node:fs/promises";
+import os from "node:os";
 import type {
  ProviderAppGuidedSetupContext,
  ProviderAuthContext,
@@ -16,6 +17,7 @@ import {
  DEFAULT_LLAMA_CPP_MODEL_URI,
  LLAMA_CPP_PROVIDER_ID,
  buildLlamaCppProviderConfig,
+  meetsLlamaCppDefaultModelRamFloor,
  resolveCachedLlamaCppModelPath,
  resolveLlamaCppModelCacheDir,
  resolveLlamaCppModelSource,
@@ -26,6 +28,27 @@ import {
  type NodeLlamaCppModule,
 } from "./node-llama.runtime.js";

+const BYTES_PER_GB = 1_000_000_000;
+const BYTES_PER_MB = 1_000_000;
+
+function formatLlamaCppDownloadProgress(params: {
+  downloadedSize: number;
+  totalSize: number;
+  bytesPerSecond: number;
+}): string {
+  const downloadedSize = Math.max(0, params.downloadedSize);
+  const totalSize = Math.max(1, params.totalSize);
+  const percent = Math.min(100, Math.floor((downloadedSize / totalSize) * 100));
+  const downloadedGb = (downloadedSize / BYTES_PER_GB).toFixed(1);
+  const totalGb = (totalSize / BYTES_PER_GB).toFixed(1);
+  const rateMb = Math.max(0, Math.round(params.bytesPerSecond / BYTES_PER_MB));
+  return `Downloading Gemma 4 E4B… ${percent}% (${downloadedGb}/${totalGb} GB, ${rateMb} MB/s)`;
+}
+
+function formatRamGb(totalmemBytes: number): string {
+  return (totalmemBytes / 1024 ** 3).toFixed(1).replace(/\.0$/, "");
+}
+
 function readPrimaryModel(config: ProviderAppGuidedSetupContext["config"]): string | undefined {
  const model = config.agents?.defaults?.model;
  return typeof model === "string" ? model : model?.primary;
@@ -122,31 +145,64 @@ export async function runLlamaCppSetup(ctx: ProviderAuthContext): Promise<Provid
    provider: existing,
  });
  if (!cachedPath || !(await isFile(cachedPath))) {
+    const totalmemBytes = os.totalmem();
+    if (!meetsLlamaCppDefaultModelRamFloor(totalmemBytes)) {
+      await ctx.prompter.note(
+        `This machine has ${formatRamGb(totalmemBytes)} GB RAM; the bundled local model needs 16 GB+. Use Ollama/LM Studio with a smaller model, or a cloud provider.`,
+        "Setup skipped",
+      );
+      return { profiles: [] };
+    }
    const consent = await ctx.prompter.confirm({
-      message:
-        "Download Qwen3 4B Instruct 2507 Q4_K_M (about 2.5 GB) for local llama.cpp inference?",
+      message: "Download Gemma 4 E4B IT Q4_K_M (about 5.0 GB) for local llama.cpp inference?",
      initialValue: false,
    });
    if (!consent) {
      await ctx.prompter.note("Local model download skipped.", "Setup skipped");
      return { profiles: [] };
    }
-    const progress = ctx.prompter.progress("Preparing Qwen3 4B model download…");
+    const progress = ctx.prompter.progress("Preparing Gemma 4 E4B model download…");
    try {
      const runtime = await importNodeLlamaCpp();
+      let previousDownloadedSize: number | undefined;
+      let previousProgressAtMs: number | undefined;
+      let rollingBytesPerSecond = 0;
      const downloader = await runtime.createModelDownloader({
        modelUri: DEFAULT_LLAMA_CPP_MODEL_URI,
        dirPath: cacheDir,
        fileName: DEFAULT_LLAMA_CPP_MODEL_CACHE_FILE,
        showCliProgress: false,
        onProgress: ({ downloadedSize, totalSize }) => {
+          const now = Date.now();
+          if (
+            previousDownloadedSize !== undefined &&
+            previousProgressAtMs !== undefined &&
+            downloadedSize >= previousDownloadedSize &&
+            now > previousProgressAtMs
+          ) {
+            const elapsedSeconds = (now - previousProgressAtMs) / 1000;
+            const currentBytesPerSecond =
+              (downloadedSize - previousDownloadedSize) / elapsedSeconds;
+            // Four-sample EWMA: a small rolling window without per-update allocations.
+            rollingBytesPerSecond =
+              rollingBytesPerSecond === 0
+                ? currentBytesPerSecond
+                : rollingBytesPerSecond * 0.75 + currentBytesPerSecond * 0.25;
+          }
+          previousDownloadedSize = downloadedSize;
+          previousProgressAtMs = now;
          const expectedSize = totalSize || DEFAULT_LLAMA_CPP_MODEL_SIZE_BYTES;
-          const percent = Math.min(100, Math.floor((downloadedSize / expectedSize) * 100));
-          progress.update(`Downloading Qwen3 4B model… ${percent}%`);
+          progress.update(
+            formatLlamaCppDownloadProgress({
+              downloadedSize,
+              totalSize: expectedSize,
+              bytesPerSecond: rollingBytesPerSecond,
+            }),
+          );
        },
      });
      await downloader.download({ signal: ctx.signal });
-      progress.stop("Qwen3 4B model downloaded");
+      progress.stop("Gemma 4 E4B model downloaded");
    } catch (error) {
      progress.stop("Model download failed");
      throw new Error(formatLlamaCppSetupError(error), { cause: error });
@@ -154,3 +210,9 @@ export async function runLlamaCppSetup(ctx: ProviderAuthContext): Promise<Provid
  }
  return buildSetupResult(ctx.config);
 }
+
+if (process.env.VITEST || process.env.NODE_ENV === "test") {
+  (globalThis as Record<PropertyKey, unknown>)[Symbol.for("openclaw.llamaCppSetupTestApi")] = {
+    formatLlamaCppDownloadProgress,
+  };
+}