ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-25 20:05:46 -06:00
Code Issues Packages Projects Releases Wiki Activity

test(llama-cpp): consolidate provider fixtures (#118426)

Browse Source
This commit is contained in:
Peter Steinberger
2026-08-02 20:59:50 -07:00
committed by GitHub
parent 7aaf3023fc
commit 1cc374b2f0
2 changed files with 196 additions and 405 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/llama-cpp/index.test.ts
+90 -208
View File
@@ -32,6 +32,8 @@ import { llamaCppEmbeddingProviderAdapter } from "./src/embedding-provider.js";
const DEFAULT_LLAMA_CPP_EMBEDDING_MODEL =
"hf:ggml-org/embeddinggemma-300m-qat-q8_0-GGUF/embeddinggemma-300m-qat-Q8_0.gguf";
const DEFAULT_LLAMA_CPP_EMBEDDING_CACHE_FILE = "hf_ggml-org_embeddinggemma-300m-qat-Q8_0.gguf";
const DEFAULT_LLAMA_CPP_EMBEDDING_CACHE_DIR = path.join(os.homedir(), ".node-llama-cpp", "models");
type AdapterCreateOptions = Parameters<typeof llamaCppEmbeddingProviderAdapter.create>[0];
type MemoryCreateTestOptions = AdapterCreateOptions & {
fallback?: "none";
@@ -67,6 +69,45 @@ async function createLlamaCppMemoryEmbeddingProvider(options: MemoryCreateTestOp
});
}
function mockLocalEmbeddingProvider(model = DEFAULT_LLAMA_CPP_EMBEDDING_MODEL) {
memoryHostEmbeddingMocks.createLocalEmbeddingProvider.mockResolvedValue({
id: "local",
model,
embedQuery: vi.fn(),
embedBatch: vi.fn(),
});
}
async function createMemoryProvider(
model: string,
local: NonNullable<AdapterCreateOptions["local"]> = { modelPath: model },
) {
return await createLlamaCppMemoryEmbeddingProvider({
config: {},
provider: "local",
fallback: "none",
model,
local,
});
}
function cacheKeyData(model = DEFAULT_LLAMA_CPP_EMBEDDING_MODEL) {
return { provider: "local", model };
}
function identityAliases(...models: string[]) {
return models.map((model) => ({ model, cacheKeyData: cacheKeyData(model) }));
}
function resolveIndexIdentity(modelPath: string, modelCacheDir?: string) {
return llamaCppEmbeddingProviderAdapter.resolveIndexIdentity?.({
config: {},
provider: "local",
model: modelPath,
local: { modelPath, ...(modelCacheDir ? { modelCacheDir } : {}) },
});
}
afterEach(() => {
clearEmbeddingProviders();
clearMemoryEmbeddingProviders();
@@ -223,12 +264,7 @@ describe("llama.cpp provider plugin", () => {
});
it("includes output dimensionality in local cache and index identities", async () => {
memoryHostEmbeddingMocks.createLocalEmbeddingProvider.mockResolvedValue({
id: "local",
model: DEFAULT_LLAMA_CPP_EMBEDDING_MODEL,
embedQuery: vi.fn(),
embedBatch: vi.fn(),
});
mockLocalEmbeddingProvider();
const result = await createLlamaCppMemoryEmbeddingProvider({
config: {},
@@ -264,76 +300,22 @@ describe("llama.cpp provider plugin", () => {
it("keeps the default model identity when configured with its exact cache artifact path", async () => {
const modelPath = path.join(
os.homedir(),
".node-llama-cpp",
"models",
"hf_ggml-org_embeddinggemma-300m-qat-Q8_0.gguf",
DEFAULT_LLAMA_CPP_EMBEDDING_CACHE_DIR,
DEFAULT_LLAMA_CPP_EMBEDDING_CACHE_FILE,
);
memoryHostEmbeddingMocks.createLocalEmbeddingProvider.mockResolvedValue({
id: "local",
model: modelPath,
embedQuery: vi.fn(),
embedBatch: vi.fn(),
});
mockLocalEmbeddingProvider(modelPath);
const result = await createLlamaCppMemoryEmbeddingProvider({
config: {},
provider: "local",
fallback: "none",
model: modelPath,
local: { modelPath },
});
const result = await createMemoryProvider(modelPath);
expect(result.provider?.model).toBe(DEFAULT_LLAMA_CPP_EMBEDDING_MODEL);
expect(result.runtime?.cacheKeyData).toEqual({
provider: "local",
expect(result.runtime?.cacheKeyData).toEqual(cacheKeyData());
expect(result.runtime?.indexIdentityAliases).toEqual(
identityAliases(modelPath, DEFAULT_LLAMA_CPP_EMBEDDING_CACHE_FILE),
);
expect(resolveIndexIdentity(modelPath)).toEqual({
model: DEFAULT_LLAMA_CPP_EMBEDDING_MODEL,
});
expect(result.runtime?.indexIdentityAliases).toEqual([
{
model: modelPath,
cacheKeyData: {
provider: "local",
model: modelPath,
},
},
{
model: "hf_ggml-org_embeddinggemma-300m-qat-Q8_0.gguf",
cacheKeyData: {
provider: "local",
model: "hf_ggml-org_embeddinggemma-300m-qat-Q8_0.gguf",
},
},
]);
expect(
llamaCppEmbeddingProviderAdapter.resolveIndexIdentity?.({
config: {},
provider: "local",
model: modelPath,
local: { modelPath },
}),
).toEqual({
model: DEFAULT_LLAMA_CPP_EMBEDDING_MODEL,
cacheKeyData: {
provider: "local",
model: DEFAULT_LLAMA_CPP_EMBEDDING_MODEL,
},
aliases: [
{
model: modelPath,
cacheKeyData: {
provider: "local",
model: modelPath,
},
},
{
model: "hf_ggml-org_embeddinggemma-300m-qat-Q8_0.gguf",
cacheKeyData: {
provider: "local",
model: "hf_ggml-org_embeddinggemma-300m-qat-Q8_0.gguf",
},
},
],
cacheKeyData: cacheKeyData(),
aliases: identityAliases(modelPath, DEFAULT_LLAMA_CPP_EMBEDDING_CACHE_FILE),
});
expect(memoryHostEmbeddingMocks.createLocalEmbeddingProvider).toHaveBeenCalledWith(
expect.objectContaining({
@@ -346,70 +328,35 @@ describe("llama.cpp provider plugin", () => {
);
});
it("keeps an arbitrary same-basename model path as a distinct identity", async () => {
const modelPath = path.join(
os.tmpdir(),
"custom-models",
DEFAULT_LLAMA_CPP_EMBEDDING_MODEL.split("/").at(-1)!,
);
memoryHostEmbeddingMocks.createLocalEmbeddingProvider.mockResolvedValue({
id: "local",
model: modelPath,
embedQuery: vi.fn(),
embedBatch: vi.fn(),
});
const result = await createLlamaCppMemoryEmbeddingProvider({
config: {},
provider: "local",
fallback: "none",
model: modelPath,
local: { modelPath },
});
expect(result.provider?.model).toBe(modelPath);
expect(result.runtime?.cacheKeyData).toEqual({
provider: "local",
model: modelPath,
});
expect(result.runtime).not.toHaveProperty("indexIdentityAliases");
});
it("keeps a bare same-basename file in the default cache as a distinct identity", async () => {
const modelPath = path.join(
os.homedir(),
".node-llama-cpp",
"models",
DEFAULT_LLAMA_CPP_EMBEDDING_MODEL.split("/").at(-1)!,
);
memoryHostEmbeddingMocks.createLocalEmbeddingProvider.mockResolvedValue({
id: "local",
model: modelPath,
embedQuery: vi.fn(),
embedBatch: vi.fn(),
});
const result = await createLlamaCppMemoryEmbeddingProvider({
config: {},
provider: "local",
fallback: "none",
model: modelPath,
local: { modelPath },
});
it.each([
[
"keeps an arbitrary same-basename model path as a distinct identity",
path.join(os.tmpdir(), "custom-models", DEFAULT_LLAMA_CPP_EMBEDDING_MODEL.split("/").at(-1)!),
true,
],
[
"keeps a bare same-basename file in the default cache as a distinct identity",
path.join(
DEFAULT_LLAMA_CPP_EMBEDDING_CACHE_DIR,
DEFAULT_LLAMA_CPP_EMBEDDING_MODEL.split("/").at(-1)!,
),
false,
],
])("%s", async (_name, modelPath, checksCacheKey) => {
mockLocalEmbeddingProvider(modelPath);
const result = await createMemoryProvider(modelPath);
expect(result.provider?.model).toBe(modelPath);
if (checksCacheKey) {
expect(result.runtime?.cacheKeyData).toEqual(cacheKeyData(modelPath));
}
expect(result.runtime).not.toHaveProperty("indexIdentityAliases");
});
it("keeps the default model identity with a custom cache directory", async () => {
const modelCacheDir = path.join(os.tmpdir(), "llama-cpp-model-cache");
const modelPath = path.join(modelCacheDir, "hf_ggml-org_embeddinggemma-300m-qat-Q8_0.gguf");
memoryHostEmbeddingMocks.createLocalEmbeddingProvider.mockResolvedValue({
id: "local",
model: modelPath,
embedQuery: vi.fn(),
embedBatch: vi.fn(),
});
const modelPath = path.join(modelCacheDir, DEFAULT_LLAMA_CPP_EMBEDDING_CACHE_FILE);
mockLocalEmbeddingProvider(modelPath);
const result = await createLlamaCppMemoryEmbeddingProvider({
config: {},
@@ -420,26 +367,10 @@ describe("llama.cpp provider plugin", () => {
});
expect(result.provider?.model).toBe(DEFAULT_LLAMA_CPP_EMBEDDING_MODEL);
expect(result.runtime?.cacheKeyData).toEqual({
provider: "local",
model: DEFAULT_LLAMA_CPP_EMBEDDING_MODEL,
});
expect(result.runtime?.indexIdentityAliases).toEqual([
{
model: modelPath,
cacheKeyData: {
provider: "local",
model: modelPath,
},
},
{
model: "hf_ggml-org_embeddinggemma-300m-qat-Q8_0.gguf",
cacheKeyData: {
provider: "local",
model: "hf_ggml-org_embeddinggemma-300m-qat-Q8_0.gguf",
},
},
]);
expect(result.runtime?.cacheKeyData).toEqual(cacheKeyData());
expect(result.runtime?.indexIdentityAliases).toEqual(
identityAliases(modelPath, DEFAULT_LLAMA_CPP_EMBEDDING_CACHE_FILE),
);
});
it.each([
@@ -449,81 +380,32 @@ describe("llama.cpp provider plugin", () => {
},
{
direction: "exact relative cache artifact to default URI",
modelPath: "hf_ggml-org_embeddinggemma-300m-qat-Q8_0.gguf",
modelPath: DEFAULT_LLAMA_CPP_EMBEDDING_CACHE_FILE,
},
])("keeps $direction compatible", ({ modelPath }) => {
const modelCacheDir = path.join(os.tmpdir(), "llama-cpp-relative-model-cache");
const relativeModelPath = "hf_ggml-org_embeddinggemma-300m-qat-Q8_0.gguf";
const relativeModelPath = DEFAULT_LLAMA_CPP_EMBEDDING_CACHE_FILE;
const resolvedModelPath = path.join(modelCacheDir, relativeModelPath);
expect(
llamaCppEmbeddingProviderAdapter.resolveIndexIdentity?.({
config: {},
provider: "local",
model: modelPath,
local: { modelPath, modelCacheDir },
}),
).toEqual({
expect(resolveIndexIdentity(modelPath, modelCacheDir)).toEqual({
model: DEFAULT_LLAMA_CPP_EMBEDDING_MODEL,
cacheKeyData: {
provider: "local",
model: DEFAULT_LLAMA_CPP_EMBEDDING_MODEL,
},
aliases: [
{
model: resolvedModelPath,
cacheKeyData: {
provider: "local",
model: resolvedModelPath,
},
},
{
model: relativeModelPath,
cacheKeyData: {
provider: "local",
model: relativeModelPath,
},
},
],
cacheKeyData: cacheKeyData(),
aliases: identityAliases(resolvedModelPath, relativeModelPath),
});
});
it("keeps the default model identity for its exact relative cache artifact", async () => {
const modelCacheDir = path.join(os.tmpdir(), "llama-cpp-relative-model-cache");
const modelPath = "hf_ggml-org_embeddinggemma-300m-qat-Q8_0.gguf";
const modelPath = DEFAULT_LLAMA_CPP_EMBEDDING_CACHE_FILE;
const resolvedModelPath = path.join(modelCacheDir, modelPath);
memoryHostEmbeddingMocks.createLocalEmbeddingProvider.mockResolvedValue({
id: "local",
model: modelPath,
embedQuery: vi.fn(),
embedBatch: vi.fn(),
});
mockLocalEmbeddingProvider(modelPath);
const result = await createLlamaCppMemoryEmbeddingProvider({
config: {},
provider: "local",
fallback: "none",
model: modelPath,
local: { modelPath, modelCacheDir },
});
const result = await createMemoryProvider(modelPath, { modelPath, modelCacheDir });
expect(result.provider?.model).toBe(DEFAULT_LLAMA_CPP_EMBEDDING_MODEL);
expect(result.runtime?.indexIdentityAliases).toEqual([
{
model: resolvedModelPath,
cacheKeyData: {
provider: "local",
model: resolvedModelPath,
},
},
{
model: modelPath,
cacheKeyData: {
provider: "local",
model: modelPath,
},
},
]);
expect(result.runtime?.indexIdentityAliases).toEqual(
identityAliases(resolvedModelPath, modelPath),
);
});
it("formats missing runtime errors with the plugin install command", () => {
extensions/llama-cpp/src/inference-provider.test.ts
+106 -197
View File
@@ -83,6 +83,22 @@ const model: Model = {
params: { modelPath: "test.gguf" },
};
const weatherTool = {
name: "weather",
description: "Weather",
parameters: { type: "object" },
} satisfies NonNullable<Context["tools"]>[number];
const weatherToolWithCity = {
name: "weather",
description: "Get weather",
parameters: { type: "object", properties: { city: { type: "string" } } },
} satisfies NonNullable<Context["tools"]>[number];
const calendarTool = {
name: "calendar",
description: "Calendar",
parameters: { type: "object" },
} satisfies NonNullable<Context["tools"]>[number];
async function collectEvents(
stream: AsyncIterable<AssistantMessageEvent>,
): Promise<AssistantMessageEvent[]> {
@@ -93,6 +109,28 @@ async function collectEvents(
return events;
}
type TestStreamParams = {
selectedModel?: Model;
prompt?: string;
tools?: Context["tools"];
options?: Parameters<ReturnType<typeof createLlamaCppStreamFn>>[2];
};
async function createTestStream(params: TestStreamParams = {}) {
return await createLlamaCppStreamFn({})(
params.selectedModel ?? model,
{
messages: [{ role: "user", content: params.prompt ?? "Hi", timestamp: 1 }],
...(params.tools ? { tools: params.tools } : {}),
},
params.options,
);
}
async function collectTestEvents(params: TestStreamParams = {}) {
return await collectEvents(await createTestStream(params));
}
beforeEach(async () => {
await clearLlamaCppInferenceCacheForTests();
vi.clearAllMocks();
@@ -207,13 +245,7 @@ describe("llama.cpp inference provider", () => {
metadata: { stopReason: "eogToken" },
};
});
const stream = await createLlamaCppStreamFn({})(
model,
{ messages: [{ role: "user", content: "Hi", timestamp: 1 }] },
{ stop: ["END"] },
);
const events = await collectEvents(stream);
const events = await collectTestEvents({ prompt: "Hi", options: { stop: ["END"] } });
expect(events.map((event) => event.type)).toEqual([
"start",
@@ -260,15 +292,11 @@ describe("llama.cpp inference provider", () => {
};
});
const events = await collectEvents(
await createLlamaCppStreamFn({})(
{ ...model, reasoning: true },
{
messages: [{ role: "user", content: "Why?", timestamp: 1 }],
tools: [{ name: "weather", description: "Weather", parameters: { type: "object" } }],
},
),
);
const events = await collectTestEvents({
selectedModel: { ...model, reasoning: true },
prompt: "Why?",
tools: [weatherTool],
});
expect(events.map((event) => event.type)).toEqual([
"start",
@@ -324,15 +352,11 @@ describe("llama.cpp inference provider", () => {
};
});
const events = await collectEvents(
await createLlamaCppStreamFn({})(
{ ...model, reasoning: true },
{
messages: [{ role: "user", content: "Weather?", timestamp: 1 }],
tools: [{ name: "weather", description: "Weather", parameters: { type: "object" } }],
},
),
);
const events = await collectTestEvents({
selectedModel: { ...model, reasoning: true },
prompt: "Weather?",
tools: [weatherTool],
});
expect(events.map((event) => event.type)).toEqual([
"start",
@@ -384,12 +408,10 @@ describe("llama.cpp inference provider", () => {
};
});
const events = await collectEvents(
await createLlamaCppStreamFn({})(
{ ...model, reasoning: true },
{ messages: [{ role: "user", content: "Reason twice", timestamp: 1 }] },
),
);
const events = await collectTestEvents({
selectedModel: { ...model, reasoning: true },
prompt: "Reason twice",
});
expect(events.map((event) => event.type)).toEqual([
"start",
@@ -432,13 +454,7 @@ describe("llama.cpp inference provider", () => {
required: ["reply"],
additionalProperties: false,
};
const stream = await createLlamaCppStreamFn({})(
model,
{ messages: [{ role: "user", content: "Hi", timestamp: 1 }] },
{ responseFormat: schema },
);
await collectEvents(stream);
await collectTestEvents({ options: { responseFormat: schema } });
expect(mocks.llama.createGrammarForJsonSchema).toHaveBeenCalledWith(schema);
expect(mocks.generateResponse.mock.calls[0]?.[1]).toMatchObject({
@@ -454,31 +470,21 @@ describe("llama.cpp inference provider", () => {
required: ["reply"],
additionalProperties: false,
};
const stream = await createLlamaCppStreamFn({})(
model,
{ messages: [{ role: "user", content: "Hi", timestamp: 1 }] },
{
await collectTestEvents({
options: {
responseFormat: {
type: "json_schema",
json_schema: { name: "planner", schema },
},
},
);
await collectEvents(stream);
});
expect(mocks.llama.createGrammarForJsonSchema).toHaveBeenCalledWith(schema);
expect(mocks.generateResponse.mock.calls[0]?.[1]).toMatchObject({ grammar: { schema } });
});
it("maps provider-shaped json_object response formats to the JSON grammar", async () => {
const stream = await createLlamaCppStreamFn({})(
model,
{ messages: [{ role: "user", content: "Hi", timestamp: 1 }] },
{ responseFormat: { type: "json_object" } },
);
await collectEvents(stream);
await collectTestEvents({ options: { responseFormat: { type: "json_object" } } });
expect(mocks.llama.getGrammarFor).toHaveBeenCalledWith("json");
expect(mocks.generateResponse.mock.calls[0]?.[1]).toMatchObject({
@@ -487,13 +493,7 @@ describe("llama.cpp inference provider", () => {
});
it("maps an empty JSON Schema to the generic JSON grammar", async () => {
const stream = await createLlamaCppStreamFn({})(
model,
{ messages: [{ role: "user", content: "Hi", timestamp: 1 }] },
{ responseFormat: {} },
);
await collectEvents(stream);
await collectTestEvents({ options: { responseFormat: {} } });
expect(mocks.llama.getGrammarFor).toHaveBeenCalledWith("json");
expect(mocks.generateResponse.mock.calls[0]?.[1]).toMatchObject({
@@ -502,13 +502,7 @@ describe("llama.cpp inference provider", () => {
});
it("keeps provider-shaped text response formats unconstrained", async () => {
const stream = await createLlamaCppStreamFn({})(
model,
{ messages: [{ role: "user", content: "Hi", timestamp: 1 }] },
{ responseFormat: { type: "text" } },
);
await collectEvents(stream);
await collectTestEvents({ options: { responseFormat: { type: "text" } } });
expect(mocks.llama.getGrammarFor).not.toHaveBeenCalled();
expect(mocks.llama.createGrammarForJsonSchema).not.toHaveBeenCalled();
@@ -521,19 +515,11 @@ describe("llama.cpp inference provider", () => {
functionCalls: [{ functionName: "weather", params: { city: "Paris" }, raw: [] }],
metadata: { stopReason: "functionCalls" },
});
const stream = await createLlamaCppStreamFn({})(model, {
messages: [{ role: "user", content: "Weather?", timestamp: 1 }],
tools: [
{
name: "weather",
description: "Get weather",
parameters: { type: "object", properties: { city: { type: "string" } } },
},
],
const events = await collectTestEvents({
prompt: "Weather?",
tools: [weatherToolWithCity],
});
const events = await collectEvents(stream);
expect(events.map((event) => event.type)).toEqual([
"start",
"toolcall_start",
@@ -607,14 +593,10 @@ describe("llama.cpp inference provider", () => {
};
});
const stream = await createLlamaCppStreamFn({})(model, {
messages: [{ role: "user", content: "Check both", timestamp: 1 }],
tools: [
{ name: "weather", description: "Weather", parameters: { type: "object" } },
{ name: "calendar", description: "Calendar", parameters: { type: "object" } },
],
const events = await collectTestEvents({
prompt: "Check both",
tools: [weatherTool, calendarTool],
});
const events = await collectEvents(stream);
expect(events.map((event) => event.type)).toEqual([
"start",
@@ -675,13 +657,24 @@ describe("llama.cpp inference provider", () => {
}
});
it("never completes or executes an interrupted native call at the token limit", async () => {
it.each([
[
"never completes or executes an interrupted native call at the token limit",
'{"city":',
false,
],
[
"never completes a native call when its final argument reaches the token limit",
'{"city":"Paris"}',
true,
],
])("%s", async (_name, paramsChunk, done) => {
mocks.generateResponse.mockImplementationOnce(async (_history, options) => {
options.onFunctionCallParamsChunk({
callIndex: 0,
functionName: "weather",
paramsChunk: '{"city":',
done: false,
paramsChunk,
done,
});
return {
response: "",
@@ -690,47 +683,7 @@ describe("llama.cpp inference provider", () => {
};
});
const events = await collectEvents(
await createLlamaCppStreamFn({})(model, {
messages: [{ role: "user", content: "Weather?", timestamp: 1 }],
tools: [{ name: "weather", description: "Weather", parameters: { type: "object" } }],
}),
);
expect(events.map((event) => event.type)).toEqual([
"start",
"toolcall_start",
"toolcall_delta",
"done",
]);
expect(events.at(-1)).toMatchObject({
type: "done",
reason: "length",
message: { content: [], stopReason: "length" },
});
});
it("never completes a native call when its final argument reaches the token limit", async () => {
mocks.generateResponse.mockImplementationOnce(async (_history, options) => {
options.onFunctionCallParamsChunk({
callIndex: 0,
functionName: "weather",
paramsChunk: '{"city":"Paris"}',
done: true,
});
return {
response: "",
functionCalls: undefined,
metadata: { stopReason: "maxTokens" },
};
});
const events = await collectEvents(
await createLlamaCppStreamFn({})(model, {
messages: [{ role: "user", content: "Weather?", timestamp: 1 }],
tools: [{ name: "weather", description: "Weather", parameters: { type: "object" } }],
}),
);
const events = await collectTestEvents({ prompt: "Weather?", tools: [weatherTool] });
expect(events.map((event) => event.type)).toEqual([
"start",
@@ -766,15 +719,10 @@ describe("llama.cpp inference provider", () => {
};
});
const events = await collectEvents(
await createLlamaCppStreamFn({})(model, {
messages: [{ role: "user", content: "Check both", timestamp: 1 }],
tools: [
{ name: "weather", description: "Weather", parameters: { type: "object" } },
{ name: "calendar", description: "Calendar", parameters: { type: "object" } },
],
}),
);
const events = await collectTestEvents({
prompt: "Check both",
tools: [weatherTool, calendarTool],
});
expect(events.filter((event) => event.type === "toolcall_end")).toHaveLength(0);
expect(events.at(-1)).toMatchObject({
@@ -807,19 +755,11 @@ describe("llama.cpp inference provider", () => {
};
});
const stream = await createLlamaCppStreamFn({})(model, {
messages: [{ role: "user", content: "Weather?", timestamp: 1 }],
tools: [
{
name: "weather",
description: "Get weather",
parameters: { type: "object", properties: { city: { type: "string" } } },
},
],
const events = await collectTestEvents({
prompt: "Weather?",
tools: [weatherToolWithCity],
});
const events = await collectEvents(stream);
expect(events.map((event) => event.type)).toEqual([
"start",
"toolcall_start",
@@ -854,19 +794,11 @@ describe("llama.cpp inference provider", () => {
};
});
const stream = await createLlamaCppStreamFn({})(model, {
messages: [{ role: "user", content: "Weather?", timestamp: 1 }],
tools: [
{
name: "weather",
description: "Get weather",
parameters: { type: "object", properties: { city: { type: "string" } } },
},
],
const events = await collectTestEvents({
prompt: "Weather?",
tools: [weatherToolWithCity],
});
const events = await collectEvents(stream);
expect(events.map((event) => event.type)).toEqual([
"start",
"text_start",
@@ -882,19 +814,10 @@ describe("llama.cpp inference provider", () => {
});
it("lets tools win when responseFormat is also present", async () => {
const stream = await createLlamaCppStreamFn({})(
model,
{
messages: [{ role: "user", content: "Weather?", timestamp: 1 }],
tools: [
{
name: "weather",
description: "Get weather",
parameters: { type: "object", properties: { city: { type: "string" } } },
},
],
},
{
await collectTestEvents({
prompt: "Weather?",
tools: [weatherToolWithCity],
options: {
responseFormat: {
type: "object",
properties: { reply: { type: "string" } },
@@ -902,9 +825,7 @@ describe("llama.cpp inference provider", () => {
additionalProperties: false,
},
},
);
await collectEvents(stream);
});
expect(mocks.llama.createGrammarForJsonSchema).not.toHaveBeenCalled();
expect(mocks.generateResponse.mock.calls[0]?.[1]).toMatchObject({
@@ -970,11 +891,7 @@ describe("llama.cpp inference provider", () => {
expectedGpuFit: 2048,
},
])("bounds native context and GPU allocation by $scenario", async (scenario) => {
await collectEvents(
await createLlamaCppStreamFn({})(scenario.model, {
messages: [{ role: "user", content: "Hi", timestamp: 1 }],
}),
);
await collectTestEvents({ selectedModel: scenario.model });
expect(mocks.model.createContext).toHaveBeenCalledWith(
expect.objectContaining({ contextSize: scenario.expectedContextSize }),
);
@@ -986,12 +903,9 @@ describe("llama.cpp inference provider", () => {
});
it("expands home-relative local model paths before resolving the file", async () => {
const stream = await createLlamaCppStreamFn({})(
{ ...model, params: { modelPath: "~/Models/test.gguf" } },
{ messages: [{ role: "user", content: "Hi", timestamp: 1 }] },
);
await collectEvents(stream);
await collectTestEvents({
selectedModel: { ...model, params: { modelPath: "~/Models/test.gguf" } },
});
expect(mocks.resolveModelFile).toHaveBeenCalledWith(
path.join(os.homedir(), "Models", "test.gguf"),
@@ -1004,9 +918,7 @@ describe("llama.cpp inference provider", () => {
options.onTextChunk("Partial");
throw new Error("generation failed");
});
const stream = await createLlamaCppStreamFn({})(model, {
messages: [{ role: "user", content: "Hi", timestamp: 1 }],
});
const stream = await createTestStream();
await expect(stream.result()).resolves.toMatchObject({
stopReason: "error",
@@ -1018,11 +930,10 @@ describe("llama.cpp inference provider", () => {
it("returns an aborted stream error when the signal is cancelled", async () => {
const controller = new AbortController();
controller.abort();
const stream = await createLlamaCppStreamFn({})(
model,
{ messages: [{ role: "user", content: "stop", timestamp: 1 }] },
{ signal: controller.signal },
);
const stream = await createTestStream({
prompt: "stop",
options: { signal: controller.signal },
});
await expect(stream.result()).resolves.toMatchObject({
stopReason: "aborted",
@@ -1037,9 +948,7 @@ describe("llama.cpp inference provider", () => {
functionCalls: undefined,
metadata: { stopReason: "abort" },
});
const stream = await createLlamaCppStreamFn({})(model, {
messages: [{ role: "user", content: "stop", timestamp: 1 }],
});
const stream = await createTestStream({ prompt: "stop" });
await expect(stream.result()).resolves.toMatchObject({
stopReason: "aborted",