fix(llama-cpp): reject queued work after cleanup failure

extensions/llama-cpp/src/inference-provider.test.ts

@@ -858,14 +858,16 @@ describe("llama.cpp inference provider", () => {
     const otherModel = { ...model, id: "other.gguf", params: { modelPath: "other.gguf" } };
     await collectTestEvents({ prompt: "one" });
     await collectTestEvents({ selectedModel: otherModel, prompt: "two" });
-    const cleanup = Promise.withResolvers<void>();
-    mocks.contextDispose.mockImplementationOnce(async () => await cleanup.promise);
-    const failedSwitch = await createTestStream({ prompt: "three" });
+    let rejectCleanup!: (error: Error) => void;
+    const cleanup = new Promise<void>((_resolve, reject) => {
+      rejectCleanup = reject;
+    });
+    mocks.contextDispose.mockImplementationOnce(async () => await cleanup);
+    await createTestStream({ prompt: "three" });
     await vi.waitFor(() => expect(mocks.contextDispose).toHaveBeenCalledTimes(2));
-    const disposing = inferenceRuntime.dispose();
-    cleanup.reject(new Error("context cleanup failed"));
-    await failedSwitch.result();
     const unavailable = await createTestStream({ selectedModel: otherModel, prompt: "four" });
+    const disposing = inferenceRuntime.dispose();
+    rejectCleanup(new Error("context cleanup failed"));
     await expect(unavailable.result()).resolves.toMatchObject({
       errorMessage: "llama.cpp runtime stopped after cleanup failed",
     });

extensions/llama-cpp/src/inference-provider.ts

@@ -46,7 +46,7 @@ type LlamaCppInferenceRuntimeState = {
   llamaInstance?: Llama;
   operationQueue: Promise<void>;
   lifecycle: "open" | "closing" | "closed";
-  cleanupFailure?: { error: unknown };
+  cleanupFailure?: { error: Error };
   disposePromise?: Promise<void>;
 };
 
@@ -86,6 +86,20 @@ function buildMessage(params: {
   };
 }
 
+function runtimeUnavailableMessage(
+  state: LlamaCppInferenceRuntimeState,
+  model: Parameters<StreamFn>[0],
+): AssistantMessage {
+  return buildMessage({
+    model,
+    content: [],
+    stopReason: "error",
+    errorMessage: state.cleanupFailure
+      ? "llama.cpp runtime stopped after cleanup failed"
+      : "llama.cpp runtime is stopping",
+  });
+}
+
 function extractText(content: unknown): string {
   if (typeof content === "string") {
     return content;
@@ -259,7 +273,7 @@ async function disposeLoadedModel(state: LlamaCppInferenceRuntimeState): Promise
 }
 
 function recordCleanupFailure(state: LlamaCppInferenceRuntimeState, error: unknown): void {
-  state.cleanupFailure ??= { error };
+  state.cleanupFailure ??= { error: error instanceof Error ? error : new Error(String(error)) };
   state.lifecycle = "closed";
 }
 
@@ -358,14 +372,7 @@ function createLlamaCppStreamFnForRuntime(
       stream.push({
         type: "error",
         reason: "error",
-        error: buildMessage({
-          model,
-          content: [],
-          stopReason: "error",
-          errorMessage: state.cleanupFailure
-            ? "llama.cpp runtime stopped after cleanup failed"
-            : "llama.cpp runtime is stopping",
-        }),
+        error: runtimeUnavailableMessage(state, model),
       });
       stream.end();
       return stream;
@@ -404,6 +411,14 @@ function createLlamaCppStreamFnForRuntime(
       started = true;
       signal?.removeEventListener("abort", abortWhileQueued);
       try {
+        if (state.lifecycle !== "open") {
+          stream.push({
+            type: "error",
+            reason: "error",
+            error: runtimeUnavailableMessage(state, model),
+          });
+          return;
+        }
         const runtime = await importNodeLlamaCpp();
         const loaded = await getLoadedModel({
           state,