ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(llama-cpp): detach retired model state

Browse Source
This commit is contained in:
Vincent Koc
2026-08-04 23:03:19 +08:00
parent 889cd316c7
commit a9d768ced6
2 changed files with 10 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/llama-cpp/src/inference-provider.test.ts
+9 -14
View File
@@ -848,21 +848,16 @@ describe("llama.cpp inference provider", () => {
expect(mocks.generateResponse.mock.calls[0]?.[1]).not.toHaveProperty("grammar");
});
it("disposes the previous model and context when the model changes", async () => {
const streamFn = inferenceRuntime.createStreamFn({});
await collectEvents(
await streamFn(model, { messages: [{ role: "user", content: "one", timestamp: 1 }] }),
);
await collectEvents(
await streamFn(
{ ...model, id: "other.gguf", params: { modelPath: "other.gguf" } },
{ messages: [{ role: "user", content: "two", timestamp: 2 }] },
),
);
expect(mocks.contextDispose).toHaveBeenCalledTimes(1);
it("disposes changed models without reusing retired state", async () => {
const otherModel = { ...model, id: "other.gguf", params: { modelPath: "other.gguf" } };
await collectTestEvents({ prompt: "one" });
await collectTestEvents({ selectedModel: otherModel, prompt: "two" });
mocks.contextDispose.mockRejectedValueOnce(new Error("context cleanup failed"));
await collectTestEvents({ prompt: "three" });
await collectTestEvents({ selectedModel: otherModel, prompt: "four" });
expect(mocks.contextDispose).toHaveBeenCalledTimes(2);
expect(mocks.modelDispose).toHaveBeenCalledTimes(1);
expect(mocks.llama.loadModel).toHaveBeenCalledTimes(2);
expect(mocks.llama.loadModel).toHaveBeenCalledTimes(3);
});
it("reuses one context sequence across serialized requests for the same model", async () => {
extensions/llama-cpp/src/inference-provider.ts
+1 -3
View File
@@ -247,11 +247,9 @@ async function disposeLoadedModel(state: LlamaCppInferenceRuntimeState): Promise
return;
}
const previous = state.loadedModel;
state.loadedModel = undefined;
await previous.context.dispose();
await previous.model.dispose();
if (state.loadedModel === previous) {
state.loadedModel = undefined;
}
}
async function getLoadedModel(params: {