mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
Fix local embedding worker safety (#85348)
Summary: - The PR routes local GGUF memory embeddings through a bundled worker sidecar, adds structured degradation and fallback handling, updates memory tests/build output, and keeps the local config contract unchanged. - PR surface: Source +831, Tests +503, Docs +1, Other +2. Total +1337 across 23 files. - Reproducibility: Do we have a high-confidence way to reproduce the issue? Source and report evidence are str ... cludes native crash logs; the exact Metal teardown abort was not reproduced in this review or the PR proof. Automerge notes: - PR branch already contained follow-up commit before automerge: fix(memory): keep local embedding config unchanged - PR branch already contained follow-up commit before automerge: fix(memory): type local embedding degradation - PR branch already contained follow-up commit before automerge: fix(memory): refresh keywords after embedding fallback - PR branch already contained follow-up commit before automerge: fix(memory): keep worker errors internal - PR branch already contained follow-up commit before automerge: test: satisfy memory provider lifecycle harnesses - PR branch already contained follow-up commit before automerge: fix: harden local embedding worker fallback Validation: - ClawSweeper review passed for head1d1fe41c4e. - Required merge gates passed before the squash merge. Prepared head SHA:1d1fe41c4eReview: https://github.com/openclaw/openclaw/pull/85348#issuecomment-4518516047 Co-authored-by: Onur Solmaz <onur@Onurs-MacBook-Pro.local> Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com> Approved-by: osolmaz Co-authored-by: osolmaz <2453968+osolmaz@users.noreply.github.com>
This commit is contained in:
@@ -14,6 +14,7 @@ Docs: https://docs.openclaw.ai
|
||||
|
||||
### Fixes
|
||||
|
||||
- Memory/local embeddings: run local GGUF embeddings in an isolated worker sidecar and degrade to configured fallback or keyword search on worker failure so native embedding crashes do not take down the Gateway. (#85348) Thanks @osolmaz.
|
||||
- Agents/heartbeat: stop heartbeat turns after the first valid `heartbeat_respond` so repeated response loops do not burn tokens. (#86357) Thanks @udaymanish6.
|
||||
- Memory-core: keep REM dreaming focused on live light-staged memories and mark staged entries as considered so old recall history no longer dominates fresh candidates. (#86302) Thanks @SebTardif.
|
||||
- Telegram: propagate forum topic names through the account-scoped topic cache for native command context and topic create/edit actions. (#86299) Thanks @SebTardif.
|
||||
|
||||
@@ -12,6 +12,7 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi }
|
||||
import "./test-runtime-mocks.js";
|
||||
import type { MemoryIndexManager } from "./index.js";
|
||||
import { closeAllMemorySearchManagers, getMemorySearchManager } from "./index.js";
|
||||
import { LOCAL_EMBEDDING_WORKER_ERROR_CODES } from "./manager-local-worker-errors.js";
|
||||
import { closeMemoryIndexManagersForAgent, EMBEDDING_PROBE_CACHE_TTL_MS } from "./manager.js";
|
||||
import {
|
||||
DEFAULT_LOCAL_MODEL,
|
||||
@@ -34,6 +35,14 @@ let providerCloseGate: Promise<void> | null = null;
|
||||
let providerCalls: Array<{ provider?: string; model?: string; outputDimensionality?: number }> = [];
|
||||
let forceNoProvider = false;
|
||||
|
||||
function createLocalWorkerExitError(): Error {
|
||||
return Object.assign(new Error("Local embedding worker exited unexpectedly (exit code 134)"), {
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.exited,
|
||||
reason: "exit",
|
||||
exitCode: 134,
|
||||
});
|
||||
}
|
||||
|
||||
vi.mock("./embeddings.js", () => {
|
||||
const embedText = (text: string) => {
|
||||
const lower = text.toLowerCase();
|
||||
@@ -44,6 +53,10 @@ vi.mock("./embeddings.js", () => {
|
||||
return [alpha, beta, image, audio];
|
||||
};
|
||||
return {
|
||||
resolveEmbeddingProviderFallbackModel: (providerId: string, fallbackSourceModel: string) =>
|
||||
providerId === "gemini" || providerId === "fallback-provider"
|
||||
? `${providerId}-embed`
|
||||
: fallbackSourceModel,
|
||||
createEmbeddingProvider: async (options: {
|
||||
provider?: string;
|
||||
model?: string;
|
||||
@@ -61,7 +74,10 @@ vi.mock("./embeddings.js", () => {
|
||||
providerUnavailableReason: "No API key found for provider",
|
||||
};
|
||||
}
|
||||
const providerId = options.provider === "gemini" ? "gemini" : "mock";
|
||||
const providerId =
|
||||
options.provider === "gemini" || options.provider === "fallback-provider"
|
||||
? options.provider
|
||||
: "mock";
|
||||
const model = options.model ?? "mock-embed";
|
||||
return {
|
||||
requestedProvider: options.provider ?? "openai",
|
||||
@@ -81,7 +97,7 @@ vi.mock("./embeddings.js", () => {
|
||||
embedBatchCalls += 1;
|
||||
return texts.map(embedText);
|
||||
},
|
||||
...(providerId === "gemini"
|
||||
...(providerId === "gemini" || providerId === "fallback-provider"
|
||||
? {
|
||||
embedBatchInputs: async (
|
||||
inputs: Array<{
|
||||
@@ -112,12 +128,12 @@ vi.mock("./embeddings.js", () => {
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
...(providerId === "gemini"
|
||||
...(providerId === "gemini" || providerId === "fallback-provider"
|
||||
? {
|
||||
runtime: {
|
||||
id: "gemini",
|
||||
id: providerId,
|
||||
cacheKeyData: {
|
||||
provider: "gemini",
|
||||
provider: providerId,
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
||||
model,
|
||||
outputDimensionality: options.outputDimensionality,
|
||||
@@ -242,7 +258,8 @@ describe("memory index", () => {
|
||||
extraPaths?: string[];
|
||||
sources?: Array<"memory" | "sessions">;
|
||||
sessionMemory?: boolean;
|
||||
provider?: "openai" | "gemini";
|
||||
provider?: "openai" | "gemini" | "fallback-provider";
|
||||
fallback?: "none" | "gemini" | "fallback-provider";
|
||||
model?: string;
|
||||
outputDimensionality?: number;
|
||||
multimodal?: {
|
||||
@@ -263,6 +280,7 @@ describe("memory index", () => {
|
||||
memorySearch: {
|
||||
provider: params.provider ?? "openai",
|
||||
model: params.model ?? "mock-embed",
|
||||
fallback: params.fallback,
|
||||
outputDimensionality: params.outputDimensionality,
|
||||
store: { path: params.storePath, vector: { enabled: params.vectorEnabled ?? false } },
|
||||
// Perf: keep test indexes to a single chunk to reduce sqlite work.
|
||||
@@ -577,6 +595,144 @@ describe("memory index", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("clears cached embedding probe readiness when local embeddings degrade", async () => {
|
||||
const cfg = createCfg({ storePath: path.join(workspaceDir, "index-probe-degraded.sqlite") });
|
||||
const manager = await getPersistentManager(cfg);
|
||||
|
||||
await expect(manager.probeEmbeddingAvailability()).resolves.toEqual({ ok: true });
|
||||
expect(manager.getCachedEmbeddingAvailability()?.ok).toBe(true);
|
||||
(
|
||||
manager as unknown as {
|
||||
provider: {
|
||||
id: string;
|
||||
model: string;
|
||||
embedQuery: (text: string) => Promise<number[]>;
|
||||
embedBatch: (texts: string[]) => Promise<number[][]>;
|
||||
close: () => Promise<void>;
|
||||
};
|
||||
}
|
||||
).provider = {
|
||||
id: "local",
|
||||
model: "local-model",
|
||||
embedQuery: async () => [1, 0],
|
||||
embedBatch: async (texts: string[]) => texts.map(() => [1, 0]),
|
||||
close: async () => {},
|
||||
};
|
||||
|
||||
(
|
||||
manager as unknown as {
|
||||
markLocalEmbeddingProviderDegraded: (err: unknown) => void;
|
||||
}
|
||||
).markLocalEmbeddingProviderDegraded(createLocalWorkerExitError());
|
||||
|
||||
expect(manager.getCachedEmbeddingAvailability()).toBeNull();
|
||||
await expect(manager.probeEmbeddingAvailability()).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: expect.stringContaining("Local embeddings degraded"),
|
||||
});
|
||||
});
|
||||
|
||||
it("activates configured fallback when local embeddings degrade during search", async () => {
|
||||
const cfg = createCfg({
|
||||
storePath: path.join(workspaceDir, "index-search-degraded-fallback.sqlite"),
|
||||
fallback: "fallback-provider",
|
||||
hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 },
|
||||
});
|
||||
const manager = await getPersistentManager(cfg);
|
||||
|
||||
await manager.sync({ reason: "test" });
|
||||
const callsBeforeSearch = providerCalls.length;
|
||||
(
|
||||
manager as unknown as {
|
||||
provider: {
|
||||
id: string;
|
||||
model: string;
|
||||
embedQuery: () => Promise<number[]>;
|
||||
embedBatch: (texts: string[]) => Promise<number[][]>;
|
||||
close: () => Promise<void>;
|
||||
};
|
||||
}
|
||||
).provider = {
|
||||
id: "local",
|
||||
model: "mock-embed",
|
||||
embedQuery: async () => {
|
||||
throw createLocalWorkerExitError();
|
||||
},
|
||||
embedBatch: async (texts: string[]) => texts.map(() => [1, 0, 0, 0]),
|
||||
close: async () => {},
|
||||
};
|
||||
|
||||
const results = await manager.search("alpha");
|
||||
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
const resultKeys = results.map(
|
||||
(result) => `${result.source}:${result.path}:${result.startLine}:${result.endLine}`,
|
||||
);
|
||||
expect(new Set(resultKeys).size).toBe(resultKeys.length);
|
||||
expect(providerCalls.slice(callsBeforeSearch).map((call) => call.provider)).toContain(
|
||||
"fallback-provider",
|
||||
);
|
||||
expect(
|
||||
(
|
||||
manager as unknown as {
|
||||
provider: { id: string } | null;
|
||||
}
|
||||
).provider?.id,
|
||||
).toBe("fallback-provider");
|
||||
});
|
||||
|
||||
it("activates configured fallback after probe-time local degradation", async () => {
|
||||
const cfg = createCfg({
|
||||
storePath: path.join(workspaceDir, "index-probe-degraded-fallback.sqlite"),
|
||||
fallback: "fallback-provider",
|
||||
hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 },
|
||||
});
|
||||
const manager = await getPersistentManager(cfg);
|
||||
|
||||
await manager.sync({ reason: "test" });
|
||||
(
|
||||
manager as unknown as {
|
||||
provider: {
|
||||
id: string;
|
||||
model: string;
|
||||
embedQuery: () => Promise<number[]>;
|
||||
embedBatch: () => Promise<number[][]>;
|
||||
close: () => Promise<void>;
|
||||
};
|
||||
}
|
||||
).provider = {
|
||||
id: "local",
|
||||
model: "mock-embed",
|
||||
embedQuery: async () => {
|
||||
throw createLocalWorkerExitError();
|
||||
},
|
||||
embedBatch: async () => {
|
||||
throw createLocalWorkerExitError();
|
||||
},
|
||||
close: async () => {},
|
||||
};
|
||||
const callsBeforeSearch = providerCalls.length;
|
||||
|
||||
await expect(manager.probeEmbeddingAvailability()).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: expect.stringContaining("Local embedding worker exited"),
|
||||
});
|
||||
|
||||
const results = await manager.search("alpha");
|
||||
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
expect(providerCalls.slice(callsBeforeSearch).map((call) => call.provider)).toContain(
|
||||
"fallback-provider",
|
||||
);
|
||||
expect(
|
||||
(
|
||||
manager as unknown as {
|
||||
provider: { id: string } | null;
|
||||
}
|
||||
).provider?.id,
|
||||
).toBe("fallback-provider");
|
||||
});
|
||||
|
||||
it("streams embedding cache rows during safe reindex", async () => {
|
||||
vi.stubEnv("OPENCLAW_TEST_MEMORY_UNSAFE_REINDEX", "0");
|
||||
type EmbeddingCacheRow = {
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
|
||||
export const MEMORY_EMBEDDING_OPERATION_ERROR_CODE = "MEMORY_EMBEDDING_OPERATION_FAILED";
|
||||
|
||||
export type MemoryEmbeddingOperationKind = "query" | "batch" | "structured-batch";
|
||||
|
||||
export type MemoryEmbeddingOperationError = Error & {
|
||||
code: typeof MEMORY_EMBEDDING_OPERATION_ERROR_CODE;
|
||||
operation: MemoryEmbeddingOperationKind;
|
||||
providerId?: string;
|
||||
cause?: unknown;
|
||||
};
|
||||
|
||||
export function createMemoryEmbeddingOperationError(params: {
|
||||
operation: MemoryEmbeddingOperationKind;
|
||||
providerId?: string;
|
||||
cause: unknown;
|
||||
}): MemoryEmbeddingOperationError {
|
||||
const message = formatErrorMessage(params.cause);
|
||||
const error = new Error(message) as MemoryEmbeddingOperationError;
|
||||
error.code = MEMORY_EMBEDDING_OPERATION_ERROR_CODE;
|
||||
error.operation = params.operation;
|
||||
if (params.providerId) {
|
||||
error.providerId = params.providerId;
|
||||
}
|
||||
error.cause = params.cause;
|
||||
return error;
|
||||
}
|
||||
|
||||
export function isMemoryEmbeddingOperationError(
|
||||
err: unknown,
|
||||
): err is MemoryEmbeddingOperationError {
|
||||
return (
|
||||
err instanceof Error &&
|
||||
(err as { code?: unknown }).code === MEMORY_EMBEDDING_OPERATION_ERROR_CODE
|
||||
);
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
loadMemoryEmbeddingCache,
|
||||
upsertMemoryEmbeddingCache,
|
||||
} from "./manager-embedding-cache.js";
|
||||
import { createMemoryEmbeddingOperationError } from "./manager-embedding-errors.js";
|
||||
import {
|
||||
buildMemoryEmbeddingBatches,
|
||||
buildTextEmbeddingInputs,
|
||||
@@ -144,6 +145,7 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
|
||||
protected abstract batchFailureLastError?: string;
|
||||
protected abstract batchFailureLastProvider?: string;
|
||||
protected abstract batchFailureLock: Promise<void>;
|
||||
protected abstract markLocalEmbeddingProviderDegraded(err: unknown): void;
|
||||
|
||||
protected pruneEmbeddingCacheIfNeeded(): void {
|
||||
if (!this.cache.enabled) {
|
||||
@@ -195,9 +197,13 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
|
||||
const inputs = buildTextEmbeddingInputs(batch);
|
||||
const hasStructuredInputs = inputs.some((input) => hasNonTextEmbeddingParts(input));
|
||||
if (hasStructuredInputs && !provider.embedBatchInputs) {
|
||||
throw new Error(
|
||||
`Embedding provider "${provider.id}" does not support multimodal memory inputs.`,
|
||||
);
|
||||
throw createMemoryEmbeddingOperationError({
|
||||
operation: "structured-batch",
|
||||
providerId: provider.id,
|
||||
cause: new Error(
|
||||
`Embedding provider "${provider.id}" does not support multimodal memory inputs.`,
|
||||
),
|
||||
});
|
||||
}
|
||||
const batchEmbeddings = hasStructuredInputs
|
||||
? await this.embedBatchInputsWithRetry(inputs)
|
||||
@@ -324,27 +330,36 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
|
||||
if (!provider) {
|
||||
throw new Error("Cannot embed batch in FTS-only mode (no embedding provider)");
|
||||
}
|
||||
return await runMemoryEmbeddingRetryLoop({
|
||||
run: async () => {
|
||||
const timeoutMs = this.resolveEmbeddingTimeout("batch");
|
||||
log.debug("memory embeddings: batch start", {
|
||||
provider: provider.id,
|
||||
items: texts.length,
|
||||
timeoutMs,
|
||||
});
|
||||
return await runEmbeddingOperationWithTimeout({
|
||||
timeoutMs,
|
||||
message: `memory embeddings batch timed out after ${Math.round(timeoutMs / 1000)}s`,
|
||||
run: async (signal) => await provider.embedBatch(texts, { signal }),
|
||||
});
|
||||
},
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
waitForRetry: async (delayMs) => {
|
||||
await this.waitForEmbeddingRetry(delayMs, "retrying");
|
||||
},
|
||||
maxAttempts: EMBEDDING_RETRY_MAX_ATTEMPTS,
|
||||
baseDelayMs: EMBEDDING_RETRY_BASE_DELAY_MS,
|
||||
});
|
||||
try {
|
||||
return await runMemoryEmbeddingRetryLoop({
|
||||
run: async () => {
|
||||
const timeoutMs = this.resolveEmbeddingTimeout("batch");
|
||||
log.debug("memory embeddings: batch start", {
|
||||
provider: provider.id,
|
||||
items: texts.length,
|
||||
timeoutMs,
|
||||
});
|
||||
return await runEmbeddingOperationWithTimeout({
|
||||
timeoutMs,
|
||||
message: `memory embeddings batch timed out after ${Math.round(timeoutMs / 1000)}s`,
|
||||
run: async (signal) => await provider.embedBatch(texts, { signal }),
|
||||
});
|
||||
},
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
waitForRetry: async (delayMs) => {
|
||||
await this.waitForEmbeddingRetry(delayMs, "retrying");
|
||||
},
|
||||
maxAttempts: EMBEDDING_RETRY_MAX_ATTEMPTS,
|
||||
baseDelayMs: EMBEDDING_RETRY_BASE_DELAY_MS,
|
||||
});
|
||||
} catch (err) {
|
||||
this.markLocalEmbeddingProviderDegraded(err);
|
||||
throw createMemoryEmbeddingOperationError({
|
||||
operation: "batch",
|
||||
providerId: provider.id,
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
protected async embedBatchInputsWithRetry(inputs: EmbeddingInput[]): Promise<number[][]> {
|
||||
@@ -356,27 +371,36 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
|
||||
if (!embedBatchInputs) {
|
||||
return await this.embedBatchWithRetry(inputs.map((input) => input.text));
|
||||
}
|
||||
return await runMemoryEmbeddingRetryLoop({
|
||||
run: async () => {
|
||||
const timeoutMs = this.resolveEmbeddingTimeout("batch");
|
||||
log.debug("memory embeddings: structured batch start", {
|
||||
provider: provider.id,
|
||||
items: inputs.length,
|
||||
timeoutMs,
|
||||
});
|
||||
return await runEmbeddingOperationWithTimeout({
|
||||
timeoutMs,
|
||||
message: `memory embeddings batch timed out after ${Math.round(timeoutMs / 1000)}s`,
|
||||
run: async (signal) => await embedBatchInputs(inputs, { signal }),
|
||||
});
|
||||
},
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
waitForRetry: async (delayMs) => {
|
||||
await this.waitForEmbeddingRetry(delayMs, "retrying structured batch");
|
||||
},
|
||||
maxAttempts: EMBEDDING_RETRY_MAX_ATTEMPTS,
|
||||
baseDelayMs: EMBEDDING_RETRY_BASE_DELAY_MS,
|
||||
});
|
||||
try {
|
||||
return await runMemoryEmbeddingRetryLoop({
|
||||
run: async () => {
|
||||
const timeoutMs = this.resolveEmbeddingTimeout("batch");
|
||||
log.debug("memory embeddings: structured batch start", {
|
||||
provider: provider.id,
|
||||
items: inputs.length,
|
||||
timeoutMs,
|
||||
});
|
||||
return await runEmbeddingOperationWithTimeout({
|
||||
timeoutMs,
|
||||
message: `memory embeddings batch timed out after ${Math.round(timeoutMs / 1000)}s`,
|
||||
run: async (signal) => await embedBatchInputs(inputs, { signal }),
|
||||
});
|
||||
},
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
waitForRetry: async (delayMs) => {
|
||||
await this.waitForEmbeddingRetry(delayMs, "retrying structured batch");
|
||||
},
|
||||
maxAttempts: EMBEDDING_RETRY_MAX_ATTEMPTS,
|
||||
baseDelayMs: EMBEDDING_RETRY_BASE_DELAY_MS,
|
||||
});
|
||||
} catch (err) {
|
||||
this.markLocalEmbeddingProviderDegraded(err);
|
||||
throw createMemoryEmbeddingOperationError({
|
||||
operation: "structured-batch",
|
||||
providerId: provider.id,
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async waitForEmbeddingRetry(delayMs: number, action: string): Promise<void> {
|
||||
@@ -405,11 +429,20 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
|
||||
}
|
||||
const timeoutMs = this.resolveEmbeddingTimeout("query");
|
||||
log.debug("memory embeddings: query start", { provider: provider.id, timeoutMs });
|
||||
return await runEmbeddingOperationWithTimeout({
|
||||
timeoutMs,
|
||||
message: `memory embeddings query timed out after ${Math.round(timeoutMs / 1000)}s`,
|
||||
run: async (signal) => await provider.embedQuery(text, { signal }),
|
||||
});
|
||||
try {
|
||||
return await runEmbeddingOperationWithTimeout({
|
||||
timeoutMs,
|
||||
message: `memory embeddings query timed out after ${Math.round(timeoutMs / 1000)}s`,
|
||||
run: async (signal) => await provider.embedQuery(text, { signal }),
|
||||
});
|
||||
} catch (err) {
|
||||
this.markLocalEmbeddingProviderDegraded(err);
|
||||
throw createMemoryEmbeddingOperationError({
|
||||
operation: "query",
|
||||
providerId: provider.id,
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
protected async withTimeout<T>(
|
||||
|
||||
@@ -4,6 +4,10 @@ import {
|
||||
resolveMemoryIndexConcurrency,
|
||||
runEmbeddingOperationWithTimeout,
|
||||
} from "./manager-embedding-ops.js";
|
||||
import {
|
||||
isLocalEmbeddingWorkerFailure,
|
||||
LOCAL_EMBEDDING_WORKER_ERROR_CODES,
|
||||
} from "./manager-local-worker-errors.js";
|
||||
|
||||
describe("memory embedding timeout resolution", () => {
|
||||
it("uses hosted defaults for inline embedding calls", () => {
|
||||
@@ -38,6 +42,34 @@ describe("memory embedding timeout resolution", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("local embedding worker failure detection", () => {
|
||||
it("matches structured local worker failure codes", () => {
|
||||
expect(
|
||||
isLocalEmbeddingWorkerFailure(
|
||||
Object.assign(new Error("Local embedding worker exited unexpectedly (exit code 134)"), {
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.exited,
|
||||
reason: "exit",
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isLocalEmbeddingWorkerFailure(
|
||||
Object.assign(new Error("Local embedding worker process failed"), {
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.processError,
|
||||
reason: "process-error",
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isLocalEmbeddingWorkerFailure(
|
||||
Object.assign(new Error("Local embedding request aborted"), {
|
||||
code: "ABORT_ERR",
|
||||
}),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("memory embedding timeout abort", () => {
|
||||
it("aborts the provider operation when the timeout fires", async () => {
|
||||
let signalSeen: AbortSignal | undefined;
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
export const LOCAL_EMBEDDING_WORKER_ERROR_CODES = {
|
||||
exited: "LOCAL_EMBEDDING_WORKER_EXITED",
|
||||
processError: "LOCAL_EMBEDDING_WORKER_PROCESS_ERROR",
|
||||
ipcError: "LOCAL_EMBEDDING_WORKER_IPC_ERROR",
|
||||
} as const;
|
||||
|
||||
export type LocalEmbeddingWorkerFailureCode =
|
||||
(typeof LOCAL_EMBEDDING_WORKER_ERROR_CODES)[keyof typeof LOCAL_EMBEDDING_WORKER_ERROR_CODES];
|
||||
|
||||
export type LocalEmbeddingWorkerFailureError = Error & {
|
||||
code: LocalEmbeddingWorkerFailureCode;
|
||||
};
|
||||
|
||||
const LOCAL_EMBEDDING_WORKER_FAILURE_CODES = new Set<string>(
|
||||
Object.values(LOCAL_EMBEDDING_WORKER_ERROR_CODES),
|
||||
);
|
||||
|
||||
export function isLocalEmbeddingWorkerFailure(
|
||||
err: unknown,
|
||||
): err is LocalEmbeddingWorkerFailureError {
|
||||
return (
|
||||
err instanceof Error &&
|
||||
LOCAL_EMBEDDING_WORKER_FAILURE_CODES.has(String((err as { code?: unknown }).code))
|
||||
);
|
||||
}
|
||||
@@ -15,8 +15,96 @@ type MemoryResolvedProviderState = {
|
||||
fallbackReason?: string;
|
||||
providerUnavailableReason?: string;
|
||||
providerRuntime?: EmbeddingProviderRuntime;
|
||||
lifecycle: MemoryProviderLifecycleState;
|
||||
};
|
||||
|
||||
export type MemoryProviderLifecycleState =
|
||||
| {
|
||||
mode: "pending";
|
||||
requestedProvider: string;
|
||||
}
|
||||
| {
|
||||
mode: "active";
|
||||
providerId: string;
|
||||
}
|
||||
| {
|
||||
mode: "degraded";
|
||||
providerId: string;
|
||||
reason: string;
|
||||
code?: string;
|
||||
}
|
||||
| {
|
||||
mode: "fallback-active";
|
||||
providerId: string;
|
||||
fallbackFrom: string;
|
||||
reason: string;
|
||||
}
|
||||
| {
|
||||
mode: "fts-only";
|
||||
reason: string;
|
||||
attemptedProviderId?: string;
|
||||
};
|
||||
|
||||
export function createPendingMemoryProviderLifecycle(
|
||||
requestedProvider: string,
|
||||
): MemoryProviderLifecycleState {
|
||||
return { mode: "pending", requestedProvider };
|
||||
}
|
||||
|
||||
export function createDegradedMemoryProviderLifecycle(params: {
|
||||
providerId: string;
|
||||
reason: string;
|
||||
code?: string;
|
||||
}): MemoryProviderLifecycleState {
|
||||
return {
|
||||
mode: "degraded",
|
||||
providerId: params.providerId,
|
||||
reason: params.reason,
|
||||
...(params.code ? { code: params.code } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveProviderLifecycle(
|
||||
result: Pick<
|
||||
EmbeddingProviderResult,
|
||||
| "provider"
|
||||
| "fallbackFrom"
|
||||
| "fallbackReason"
|
||||
| "providerUnavailableReason"
|
||||
| "requestedProvider"
|
||||
>,
|
||||
): MemoryProviderLifecycleState {
|
||||
if (result.provider && result.fallbackFrom) {
|
||||
return {
|
||||
mode: "fallback-active",
|
||||
providerId: result.provider.id,
|
||||
fallbackFrom: result.fallbackFrom,
|
||||
reason: result.fallbackReason ?? "fallback activated",
|
||||
};
|
||||
}
|
||||
if (result.provider) {
|
||||
return { mode: "active", providerId: result.provider.id };
|
||||
}
|
||||
return {
|
||||
mode: "fts-only",
|
||||
reason: result.providerUnavailableReason ?? "No embedding provider available",
|
||||
attemptedProviderId: result.requestedProvider,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveFallbackCurrentProviderId(params: {
|
||||
provider: EmbeddingProvider | null;
|
||||
lifecycle: MemoryProviderLifecycleState;
|
||||
}): string | null {
|
||||
if (params.provider) {
|
||||
return params.provider.id;
|
||||
}
|
||||
if (params.lifecycle.mode === "degraded") {
|
||||
return params.lifecycle.providerId;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function resolveMemoryPrimaryProviderRequest(params: {
|
||||
settings: ResolvedMemorySearchConfig;
|
||||
}): {
|
||||
@@ -46,7 +134,12 @@ export function resolveMemoryPrimaryProviderRequest(params: {
|
||||
export function resolveMemoryProviderState(
|
||||
result: Pick<
|
||||
EmbeddingProviderResult,
|
||||
"provider" | "fallbackFrom" | "fallbackReason" | "providerUnavailableReason" | "runtime"
|
||||
| "provider"
|
||||
| "fallbackFrom"
|
||||
| "fallbackReason"
|
||||
| "providerUnavailableReason"
|
||||
| "runtime"
|
||||
| "requestedProvider"
|
||||
>,
|
||||
): MemoryResolvedProviderState {
|
||||
return {
|
||||
@@ -55,6 +148,7 @@ export function resolveMemoryProviderState(
|
||||
fallbackReason: result.fallbackReason,
|
||||
providerUnavailableReason: result.providerUnavailableReason,
|
||||
providerRuntime: result.runtime,
|
||||
lifecycle: resolveProviderLifecycle(result),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -68,8 +162,21 @@ export function applyMemoryFallbackProviderState(params: {
|
||||
...params.current,
|
||||
fallbackFrom: params.fallbackFrom,
|
||||
fallbackReason: params.reason,
|
||||
providerUnavailableReason: undefined,
|
||||
provider: params.result.provider,
|
||||
providerRuntime: params.result.runtime,
|
||||
lifecycle: params.result.provider
|
||||
? {
|
||||
mode: "fallback-active",
|
||||
providerId: params.result.provider.id,
|
||||
fallbackFrom: params.fallbackFrom,
|
||||
reason: params.reason,
|
||||
}
|
||||
: {
|
||||
mode: "fts-only",
|
||||
reason: params.reason,
|
||||
attemptedProviderId: params.fallbackFrom,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -52,6 +52,8 @@ class SessionDeltaHarness extends MemoryManagerSyncOps {
|
||||
};
|
||||
protected readonly vector = { enabled: false, available: false };
|
||||
protected readonly cache = { enabled: false };
|
||||
protected providerUnavailableReason?: string;
|
||||
protected providerLifecycle = { mode: "active" as const, providerId: "test" };
|
||||
protected db = null as unknown as DatabaseSync;
|
||||
|
||||
readonly syncCalls: SyncParams[] = [];
|
||||
|
||||
@@ -54,6 +54,8 @@ class SessionStartupCatchupHarness extends MemoryManagerSyncOps {
|
||||
};
|
||||
protected readonly vector = { enabled: false, available: false };
|
||||
protected readonly cache = { enabled: false };
|
||||
protected providerUnavailableReason?: string;
|
||||
protected providerLifecycle = { mode: "active" as const, providerId: "test" };
|
||||
protected db: DatabaseSync;
|
||||
|
||||
readonly syncCalls: SyncParams[] = [];
|
||||
|
||||
@@ -42,9 +42,12 @@ import {
|
||||
} from "./embeddings.js";
|
||||
import { runMemoryAtomicReindex } from "./manager-atomic-reindex.js";
|
||||
import { closeMemoryDatabase, openMemoryDatabaseAtPath } from "./manager-db.js";
|
||||
import { isMemoryEmbeddingOperationError } from "./manager-embedding-errors.js";
|
||||
import {
|
||||
applyMemoryFallbackProviderState,
|
||||
resolveMemoryFallbackProviderRequest,
|
||||
resolveFallbackCurrentProviderId,
|
||||
type MemoryProviderLifecycleState,
|
||||
} from "./manager-provider-state.js";
|
||||
import {
|
||||
resolveConfiguredScopeHash,
|
||||
@@ -170,6 +173,8 @@ export abstract class MemoryManagerSyncOps {
|
||||
protected abstract readonly settings: ResolvedMemorySearchConfig;
|
||||
protected provider: EmbeddingProvider | null = null;
|
||||
protected fallbackFrom?: EmbeddingProviderId;
|
||||
protected abstract providerUnavailableReason?: string;
|
||||
protected abstract providerLifecycle: MemoryProviderLifecycleState;
|
||||
protected providerRuntime?: EmbeddingProviderRuntime;
|
||||
protected abstract batch: {
|
||||
enabled: boolean;
|
||||
@@ -1131,7 +1136,7 @@ export abstract class MemoryManagerSyncOps {
|
||||
syncSessionFiles: async (targetedParams) => {
|
||||
await this.syncSessionFiles(targetedParams);
|
||||
},
|
||||
shouldFallbackOnError: (message) => this.shouldFallbackOnError(message),
|
||||
shouldFallbackOnError: (err) => this.shouldFallbackOnError(err),
|
||||
activateFallbackProvider: async (reason) => await this.activateFallbackProvider(reason),
|
||||
runSafeReindex: async (reindexParams) => {
|
||||
await this.runSafeReindex(reindexParams);
|
||||
@@ -1205,7 +1210,7 @@ export abstract class MemoryManagerSyncOps {
|
||||
} catch (err) {
|
||||
const reason = formatErrorMessage(err);
|
||||
const activated =
|
||||
this.shouldFallbackOnError(reason) && (await this.activateFallbackProvider(reason));
|
||||
this.shouldFallbackOnError(err) && (await this.activateFallbackProvider(reason));
|
||||
if (activated) {
|
||||
await this.runSafeReindex({
|
||||
reason: params?.reason ?? "fallback",
|
||||
@@ -1214,12 +1219,21 @@ export abstract class MemoryManagerSyncOps {
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (!this.provider && this.fts.enabled && this.shouldFallbackOnError(err)) {
|
||||
log.warn(`memory embeddings unavailable; rebuilding lexical memory index only: ${reason}`);
|
||||
await this.runSafeReindex({
|
||||
reason: params?.reason ?? "embedding-degraded",
|
||||
force: true,
|
||||
progress: progress ?? undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
private shouldFallbackOnError(message: string): boolean {
|
||||
return /embedding|embeddings|batch/i.test(message);
|
||||
protected shouldFallbackOnError(err: unknown): boolean {
|
||||
return isMemoryEmbeddingOperationError(err);
|
||||
}
|
||||
|
||||
protected resolveBatchConfig(): {
|
||||
@@ -1240,19 +1254,22 @@ export abstract class MemoryManagerSyncOps {
|
||||
};
|
||||
}
|
||||
|
||||
private async activateFallbackProvider(reason: string): Promise<boolean> {
|
||||
protected async activateFallbackProvider(reason: string): Promise<boolean> {
|
||||
const currentProviderId = resolveFallbackCurrentProviderId({
|
||||
provider: this.provider,
|
||||
lifecycle: this.providerLifecycle,
|
||||
});
|
||||
const fallbackRequest = resolveMemoryFallbackProviderRequest({
|
||||
cfg: this.cfg,
|
||||
settings: this.settings,
|
||||
currentProviderId: this.provider?.id ?? null,
|
||||
currentProviderId,
|
||||
});
|
||||
if (!fallbackRequest || !this.provider) {
|
||||
if (!fallbackRequest || !currentProviderId) {
|
||||
return false;
|
||||
}
|
||||
if (this.fallbackFrom) {
|
||||
return false;
|
||||
}
|
||||
const fallbackFrom = this.provider.id;
|
||||
|
||||
const fallbackResult = await createEmbeddingProvider({
|
||||
config: this.cfg,
|
||||
@@ -1267,8 +1284,9 @@ export abstract class MemoryManagerSyncOps {
|
||||
fallbackReason: this.fallbackReason,
|
||||
providerUnavailableReason: undefined,
|
||||
providerRuntime: this.providerRuntime,
|
||||
lifecycle: this.providerLifecycle,
|
||||
},
|
||||
fallbackFrom,
|
||||
fallbackFrom: currentProviderId,
|
||||
reason,
|
||||
result: fallbackResult,
|
||||
});
|
||||
@@ -1276,6 +1294,8 @@ export abstract class MemoryManagerSyncOps {
|
||||
this.fallbackReason = fallbackState.fallbackReason;
|
||||
this.provider = fallbackState.provider;
|
||||
this.providerRuntime = fallbackState.providerRuntime;
|
||||
this.providerUnavailableReason = fallbackState.providerUnavailableReason;
|
||||
this.providerLifecycle = fallbackState.lifecycle;
|
||||
this.providerKey = this.computeProviderKey();
|
||||
this.batch = this.resolveBatchConfig();
|
||||
log.warn(`memory embeddings: switched to fallback provider (${fallbackRequest.provider})`, {
|
||||
@@ -1284,7 +1304,7 @@ export abstract class MemoryManagerSyncOps {
|
||||
return true;
|
||||
}
|
||||
|
||||
private async runSafeReindex(params: {
|
||||
protected async runSafeReindex(params: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
progress?: MemorySyncProgressState;
|
||||
|
||||
@@ -80,6 +80,8 @@ class SessionSyncYieldHarness extends MemoryManagerSyncOps {
|
||||
};
|
||||
protected readonly vector = { enabled: false, available: false };
|
||||
protected readonly cache = { enabled: false };
|
||||
protected providerUnavailableReason?: string;
|
||||
protected providerLifecycle = { mode: "active" as const, providerId: "test" };
|
||||
protected db = createDbMock();
|
||||
|
||||
readonly indexedPaths: string[] = [];
|
||||
|
||||
@@ -34,7 +34,7 @@ export async function runMemoryTargetedSessionSync(params: {
|
||||
targetSessionFiles?: string[];
|
||||
progress?: TargetedSyncProgress;
|
||||
}) => Promise<void>;
|
||||
shouldFallbackOnError: (message: string) => boolean;
|
||||
shouldFallbackOnError: (err: unknown) => boolean;
|
||||
activateFallbackProvider: (reason: string) => Promise<boolean>;
|
||||
runSafeReindex: (params: {
|
||||
reason?: string;
|
||||
@@ -70,7 +70,7 @@ export async function runMemoryTargetedSessionSync(params: {
|
||||
} catch (err) {
|
||||
const reason = formatErrorMessage(err);
|
||||
const activated =
|
||||
params.shouldFallbackOnError(reason) && (await params.activateFallbackProvider(reason));
|
||||
params.shouldFallbackOnError(err) && (await params.activateFallbackProvider(reason));
|
||||
if (!activated) {
|
||||
throw err;
|
||||
}
|
||||
|
||||
@@ -80,6 +80,7 @@ describe("memory manager mistral provider wiring", () => {
|
||||
|
||||
const state = resolveMemoryProviderState({
|
||||
provider: mistralProvider,
|
||||
requestedProvider: "mistral",
|
||||
runtime: mistralRuntime,
|
||||
fallbackFrom: undefined,
|
||||
fallbackReason: undefined,
|
||||
@@ -102,6 +103,7 @@ describe("memory manager mistral provider wiring", () => {
|
||||
const mistralProvider = createProvider("mistral");
|
||||
const current = resolveMemoryProviderState({
|
||||
provider: createProvider("openai"),
|
||||
requestedProvider: "openai",
|
||||
runtime: openAiRuntime,
|
||||
fallbackFrom: undefined,
|
||||
fallbackReason: undefined,
|
||||
@@ -124,6 +126,30 @@ describe("memory manager mistral provider wiring", () => {
|
||||
expect(fallbackState.providerRuntime).toBe(mistralRuntime);
|
||||
});
|
||||
|
||||
it("clears provider unavailable reason after fallback activation", () => {
|
||||
const fallbackState = applyMemoryFallbackProviderState({
|
||||
current: resolveMemoryProviderState({
|
||||
provider: null,
|
||||
requestedProvider: "local",
|
||||
fallbackFrom: undefined,
|
||||
fallbackReason: undefined,
|
||||
providerUnavailableReason: "Local embeddings degraded: worker crashed",
|
||||
runtime: undefined,
|
||||
}),
|
||||
fallbackFrom: "local",
|
||||
reason: "worker crashed",
|
||||
result: {
|
||||
provider: createProvider("openai"),
|
||||
runtime: {
|
||||
id: "openai",
|
||||
cacheKeyData: { provider: "openai", model: "text-embedding-3-small" },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(fallbackState.providerUnavailableReason).toBeUndefined();
|
||||
});
|
||||
|
||||
it("uses default ollama model when activating ollama fallback", () => {
|
||||
const request = resolveMemoryFallbackProviderRequest({
|
||||
cfg: {} as OpenClawConfig,
|
||||
|
||||
@@ -38,9 +38,13 @@ import {
|
||||
} from "./manager-cache.js";
|
||||
import { closeMemoryDatabase } from "./manager-db.js";
|
||||
import { MemoryManagerEmbeddingOps } from "./manager-embedding-ops.js";
|
||||
import { isLocalEmbeddingWorkerFailure } from "./manager-local-worker-errors.js";
|
||||
import {
|
||||
createDegradedMemoryProviderLifecycle,
|
||||
createPendingMemoryProviderLifecycle,
|
||||
resolveMemoryPrimaryProviderRequest,
|
||||
resolveMemoryProviderState,
|
||||
type MemoryProviderLifecycleState,
|
||||
} from "./manager-provider-state.js";
|
||||
import { resolveMemorySearchPreflight } from "./manager-search-preflight.js";
|
||||
import { searchKeyword, searchVector } from "./manager-search.js";
|
||||
@@ -124,7 +128,8 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
private providerInitialized = false;
|
||||
protected override fallbackFrom?: EmbeddingProviderId;
|
||||
protected override fallbackReason?: string;
|
||||
private providerUnavailableReason?: string;
|
||||
protected providerUnavailableReason?: string;
|
||||
protected override providerLifecycle: MemoryProviderLifecycleState;
|
||||
protected override providerRuntime?: EmbeddingProviderRuntime;
|
||||
protected batch: {
|
||||
enabled: boolean;
|
||||
@@ -239,6 +244,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
this.settings = params.settings;
|
||||
this.provider = null;
|
||||
this.requestedProvider = params.settings.provider;
|
||||
this.providerLifecycle = createPendingMemoryProviderLifecycle(this.requestedProvider);
|
||||
if (params.providerResult) {
|
||||
this.applyProviderResult(params.providerResult);
|
||||
}
|
||||
@@ -283,6 +289,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
this.fallbackFrom = providerState.fallbackFrom;
|
||||
this.fallbackReason = providerState.fallbackReason;
|
||||
this.providerUnavailableReason = providerState.providerUnavailableReason;
|
||||
this.providerLifecycle = providerState.lifecycle;
|
||||
this.providerRuntime = providerState.providerRuntime;
|
||||
this.providerInitialized = true;
|
||||
}
|
||||
@@ -312,6 +319,35 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
}
|
||||
}
|
||||
|
||||
protected markLocalEmbeddingProviderDegraded(err: unknown): void {
|
||||
if (this.provider?.id !== "local") {
|
||||
return;
|
||||
}
|
||||
if (!isLocalEmbeddingWorkerFailure(err)) {
|
||||
return;
|
||||
}
|
||||
const message = formatErrorMessage(err);
|
||||
const degradedProvider = this.provider;
|
||||
this.provider = null;
|
||||
this.providerRuntime = undefined;
|
||||
this.providerUnavailableReason = `Local embeddings degraded: ${message}`;
|
||||
this.providerLifecycle = createDegradedMemoryProviderLifecycle({
|
||||
providerId: degradedProvider.id,
|
||||
reason: message,
|
||||
code: err.code,
|
||||
});
|
||||
EMBEDDING_PROBE_CACHE.delete(this.cacheKey);
|
||||
this.providerKey = this.computeProviderKey();
|
||||
this.batch = this.resolveBatchConfig();
|
||||
this.vector.semanticAvailable = false;
|
||||
void Promise.resolve(degradedProvider.close?.()).catch((err: unknown) => {
|
||||
log.debug(`memory embeddings: failed to close degraded local provider: ${String(err)}`);
|
||||
});
|
||||
log.warn("memory embeddings: local provider degraded after worker failure", {
|
||||
error: message,
|
||||
});
|
||||
}
|
||||
|
||||
async warmSession(sessionKey?: string): Promise<void> {
|
||||
if (!this.settings.sync.onSessionStart) {
|
||||
return;
|
||||
@@ -394,6 +430,20 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
Math.max(1, Math.floor(maxResults * hybrid.candidateMultiplier)),
|
||||
);
|
||||
|
||||
if (!this.provider && this.providerLifecycle.mode === "degraded") {
|
||||
const activatedFallback = await this.activateFallbackProvider(
|
||||
this.providerLifecycle.reason,
|
||||
).catch((fallbackErr: unknown) => {
|
||||
log.warn(
|
||||
`memory search: failed to activate fallback provider: ${formatErrorMessage(fallbackErr)}`,
|
||||
);
|
||||
return false;
|
||||
});
|
||||
if (activatedFallback) {
|
||||
await this.runSafeReindex({ reason: "fallback", force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// FTS-only mode: no embedding provider available
|
||||
if (!this.provider) {
|
||||
if (!this.fts.enabled || !this.fts.available) {
|
||||
@@ -461,7 +511,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
}
|
||||
|
||||
// If FTS isn't available, hybrid mode cannot use keyword search; degrade to vector-only.
|
||||
const keywordResults =
|
||||
const loadKeywordResults = async () =>
|
||||
hybrid.enabled && this.fts.enabled && this.fts.available
|
||||
? await this.searchKeyword(
|
||||
cleaned,
|
||||
@@ -473,8 +523,32 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
return [];
|
||||
})
|
||||
: [];
|
||||
let keywordResults = await loadKeywordResults();
|
||||
|
||||
const queryVec = await this.embedQueryWithTimeout(cleaned);
|
||||
let queryVec: number[];
|
||||
try {
|
||||
queryVec = await this.embedQueryWithTimeout(cleaned);
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
const activatedFallback = this.shouldFallbackOnError(err)
|
||||
? await this.activateFallbackProvider(message).catch((fallbackErr: unknown) => {
|
||||
log.warn(
|
||||
`memory search: failed to activate fallback provider: ${formatErrorMessage(fallbackErr)}`,
|
||||
);
|
||||
return false;
|
||||
})
|
||||
: false;
|
||||
if (activatedFallback) {
|
||||
await this.runSafeReindex({ reason: "fallback", force: true });
|
||||
keywordResults = await loadKeywordResults();
|
||||
queryVec = await this.embedQueryWithTimeout(cleaned);
|
||||
} else if (!this.provider && this.fts.enabled && this.fts.available) {
|
||||
log.warn(`memory search: embeddings unavailable; using keyword-only results: ${message}`);
|
||||
return this.selectScoredResults(keywordResults, maxResults, minScore, 0);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
const hasVector = queryVec.some((v) => v !== 0);
|
||||
const vectorResults = hasVector
|
||||
? await this.searchVector(queryVec, candidates, sourceFilterList).catch((err) => {
|
||||
@@ -846,6 +920,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
|
||||
},
|
||||
custom: {
|
||||
searchMode: providerInfo.searchMode,
|
||||
providerState: this.providerLifecycle,
|
||||
providerUnavailableReason: this.providerUnavailableReason,
|
||||
readonlyRecovery: {
|
||||
attempts: this.readonlyRecoveryAttempts,
|
||||
|
||||
@@ -1401,6 +1401,8 @@ export class QmdMemoryManager implements MemorySearchManager {
|
||||
qmd: {
|
||||
collections: this.qmd.collections.length,
|
||||
lastUpdateAt: this.lastUpdateAt,
|
||||
embedFailures: this.embedFailureCount,
|
||||
embedBackoffUntil: this.embedBackoffUntil,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
export const LOCAL_EMBEDDING_WORKER_ERROR_CODES = {
|
||||
exited: "LOCAL_EMBEDDING_WORKER_EXITED",
|
||||
processError: "LOCAL_EMBEDDING_WORKER_PROCESS_ERROR",
|
||||
ipcError: "LOCAL_EMBEDDING_WORKER_IPC_ERROR",
|
||||
} as const;
|
||||
|
||||
export type LocalEmbeddingWorkerFailureCode =
|
||||
(typeof LOCAL_EMBEDDING_WORKER_ERROR_CODES)[keyof typeof LOCAL_EMBEDDING_WORKER_ERROR_CODES];
|
||||
|
||||
export type LocalEmbeddingWorkerFailureReason = "exit" | "signal" | "process-error" | "ipc";
|
||||
|
||||
export type LocalEmbeddingWorkerFailureError = Error & {
|
||||
code: LocalEmbeddingWorkerFailureCode;
|
||||
reason: LocalEmbeddingWorkerFailureReason;
|
||||
exitCode?: number | null;
|
||||
signal?: NodeJS.Signals | null;
|
||||
};
|
||||
|
||||
const LOCAL_EMBEDDING_WORKER_FAILURE_CODES = new Set<string>(
|
||||
Object.values(LOCAL_EMBEDDING_WORKER_ERROR_CODES),
|
||||
);
|
||||
|
||||
export function createLocalEmbeddingWorkerFailureError(params: {
|
||||
message: string;
|
||||
code: LocalEmbeddingWorkerFailureCode;
|
||||
reason: LocalEmbeddingWorkerFailureReason;
|
||||
exitCode?: number | null;
|
||||
signal?: NodeJS.Signals | null;
|
||||
cause?: unknown;
|
||||
}): LocalEmbeddingWorkerFailureError {
|
||||
return Object.assign(new Error(params.message), {
|
||||
code: params.code,
|
||||
reason: params.reason,
|
||||
...(params.exitCode !== undefined ? { exitCode: params.exitCode } : {}),
|
||||
...(params.signal !== undefined ? { signal: params.signal } : {}),
|
||||
...(params.cause !== undefined ? { cause: params.cause } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
export function isLocalEmbeddingWorkerFailure(
|
||||
err: unknown,
|
||||
): err is LocalEmbeddingWorkerFailureError {
|
||||
return (
|
||||
err instanceof Error &&
|
||||
LOCAL_EMBEDDING_WORKER_FAILURE_CODES.has(String((err as { code?: unknown }).code))
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import { createLocalEmbeddingProviderInProcess } from "./embeddings.js";
|
||||
import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.types.js";
|
||||
|
||||
type LocalEmbeddingWorkerRequest =
|
||||
| {
|
||||
id: number;
|
||||
type: "initialize";
|
||||
options: EmbeddingProviderOptions;
|
||||
}
|
||||
| {
|
||||
id: number;
|
||||
type: "embedQuery";
|
||||
options: EmbeddingProviderOptions;
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
id: number;
|
||||
type: "embedBatch";
|
||||
options: EmbeddingProviderOptions;
|
||||
texts: string[];
|
||||
}
|
||||
| {
|
||||
id: number;
|
||||
type: "close";
|
||||
};
|
||||
|
||||
type LocalEmbeddingWorkerSerializedError = {
|
||||
message: string;
|
||||
code?: string;
|
||||
};
|
||||
|
||||
let provider: EmbeddingProvider | null = null;
|
||||
let providerOptionsKey: string | null = null;
|
||||
let requestQueue: Promise<void> = Promise.resolve();
|
||||
|
||||
function send(message: unknown): void {
|
||||
if (typeof process.send === "function") {
|
||||
process.send(message);
|
||||
}
|
||||
}
|
||||
|
||||
async function getProvider(options: EmbeddingProviderOptions): Promise<EmbeddingProvider> {
|
||||
const key = JSON.stringify(options);
|
||||
if (provider && providerOptionsKey === key) {
|
||||
return provider;
|
||||
}
|
||||
await provider?.close?.();
|
||||
provider = await createLocalEmbeddingProviderInProcess(options);
|
||||
providerOptionsKey = key;
|
||||
return provider;
|
||||
}
|
||||
|
||||
async function closeProvider(): Promise<void> {
|
||||
const current = provider;
|
||||
provider = null;
|
||||
providerOptionsKey = null;
|
||||
await current?.close?.();
|
||||
}
|
||||
|
||||
function serializeError(err: unknown): LocalEmbeddingWorkerSerializedError {
|
||||
if (!(err instanceof Error)) {
|
||||
return { message: String(err) };
|
||||
}
|
||||
const code = (err as Error & { code?: unknown }).code;
|
||||
return {
|
||||
message: err.message,
|
||||
...(typeof code === "string" ? { code } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function handleRequest(request: LocalEmbeddingWorkerRequest): Promise<void> {
|
||||
if (request.type === "close") {
|
||||
await closeProvider();
|
||||
send({ id: request.id, ok: true });
|
||||
return;
|
||||
}
|
||||
|
||||
const currentProvider = await getProvider(request.options);
|
||||
if (request.type === "initialize") {
|
||||
send({ id: request.id, ok: true });
|
||||
return;
|
||||
}
|
||||
if (request.type === "embedQuery") {
|
||||
const value = await currentProvider.embedQuery(request.text);
|
||||
send({ id: request.id, ok: true, value });
|
||||
return;
|
||||
}
|
||||
|
||||
const value = await currentProvider.embedBatch(request.texts);
|
||||
send({ id: request.id, ok: true, value });
|
||||
}
|
||||
|
||||
process.on("message", (message) => {
|
||||
const request = message as LocalEmbeddingWorkerRequest;
|
||||
requestQueue = requestQueue.then(async () => {
|
||||
try {
|
||||
await handleRequest(request);
|
||||
} catch (err) {
|
||||
send({ id: request.id, ok: false, error: serializeError(err) });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
process.once("disconnect", () => {
|
||||
void closeProvider().finally(() => {
|
||||
process.exit(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,364 @@
|
||||
import { fork, type ChildProcess } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { DEFAULT_LOCAL_MODEL } from "./embedding-defaults.js";
|
||||
import {
|
||||
createLocalEmbeddingWorkerFailureError,
|
||||
LOCAL_EMBEDDING_WORKER_ERROR_CODES,
|
||||
} from "./embedding-worker-errors.js";
|
||||
import type { LocalEmbeddingProviderRuntimeOptions } from "./embeddings.js";
|
||||
import type {
|
||||
EmbeddingProvider,
|
||||
EmbeddingProviderCallOptions,
|
||||
EmbeddingProviderOptions,
|
||||
} from "./embeddings.types.js";
|
||||
import { normalizeOptionalString } from "./string-utils.js";
|
||||
|
||||
type LocalEmbeddingWorkerRequestPayload =
|
||||
| {
|
||||
type: "initialize";
|
||||
options: EmbeddingProviderOptions;
|
||||
}
|
||||
| {
|
||||
type: "embedQuery";
|
||||
options: EmbeddingProviderOptions;
|
||||
text: string;
|
||||
}
|
||||
| {
|
||||
type: "embedBatch";
|
||||
options: EmbeddingProviderOptions;
|
||||
texts: string[];
|
||||
}
|
||||
| {
|
||||
type: "close";
|
||||
};
|
||||
|
||||
type LocalEmbeddingWorkerRequest = LocalEmbeddingWorkerRequestPayload & { id: number };
|
||||
|
||||
type LocalEmbeddingWorkerResponse =
|
||||
| {
|
||||
id: number;
|
||||
ok: true;
|
||||
value?: number[] | number[][];
|
||||
}
|
||||
| {
|
||||
id: number;
|
||||
ok: false;
|
||||
error:
|
||||
| string
|
||||
| {
|
||||
message?: string;
|
||||
code?: string;
|
||||
};
|
||||
};
|
||||
|
||||
type PendingRequest = {
|
||||
resolve: (value: number[] | number[][] | undefined) => void;
|
||||
reject: (err: unknown) => void;
|
||||
abort?: () => void;
|
||||
};
|
||||
|
||||
function resolveDefaultWorkerScriptPath(): string {
|
||||
const currentPath = fileURLToPath(import.meta.url);
|
||||
const extension = path.extname(currentPath);
|
||||
const currentName = path.basename(currentPath);
|
||||
const sibling =
|
||||
extension === ".ts"
|
||||
? "embeddings-worker-child.ts"
|
||||
: currentName.startsWith("embeddings-worker.")
|
||||
? "embeddings-worker-child.js"
|
||||
: "memory-core-local-embedding-worker.js";
|
||||
return path.join(path.dirname(currentPath), sibling);
|
||||
}
|
||||
|
||||
function serializeLocalEmbeddingOptions(
|
||||
options: EmbeddingProviderOptions,
|
||||
): EmbeddingProviderOptions {
|
||||
return {
|
||||
config: {},
|
||||
provider: "local",
|
||||
model: options.model,
|
||||
fallback: "none",
|
||||
local: options.local,
|
||||
};
|
||||
}
|
||||
|
||||
function createWorkerExitError(code: number | null, signal: NodeJS.Signals | null): Error {
|
||||
const detail = signal ? `signal ${signal}` : `exit code ${code ?? "unknown"}`;
|
||||
return createLocalEmbeddingWorkerFailureError({
|
||||
message: `Local embedding worker exited unexpectedly (${detail})`,
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.exited,
|
||||
reason: signal ? "signal" : "exit",
|
||||
exitCode: code,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
|
||||
function createWorkerResponseError(error: LocalEmbeddingWorkerResponse & { ok: false }): Error {
|
||||
if (typeof error.error === "object" && error.error) {
|
||||
const message = error.error.message || "Local embedding worker failed";
|
||||
const workerError = new Error(message) as Error & { code?: string };
|
||||
if (error.error.code) {
|
||||
workerError.code = error.error.code;
|
||||
}
|
||||
return workerError;
|
||||
}
|
||||
return new Error(error.error || "Local embedding worker failed");
|
||||
}
|
||||
|
||||
const WORKER_UNSAFE_EXEC_ARGV_FLAGS = new Set(["--inspect", "--inspect-brk"]);
|
||||
|
||||
const WORKER_UNSAFE_EXEC_ARGV_FLAGS_WITH_VALUE = new Set([
|
||||
"--eval",
|
||||
"-e",
|
||||
"--print",
|
||||
"-p",
|
||||
"--input-type",
|
||||
"--inspect-port",
|
||||
]);
|
||||
|
||||
const WORKER_UNSAFE_EXEC_ARGV_OPTION_PREFIXES = [
|
||||
"--eval=",
|
||||
"--print=",
|
||||
"--input-type=",
|
||||
"--inspect=",
|
||||
"--inspect-brk=",
|
||||
"--inspect-port=",
|
||||
];
|
||||
|
||||
const WORKER_CLOSE_GRACE_MS = 250;
|
||||
|
||||
function resolveWorkerExecArgv(): string[] {
|
||||
const args: string[] = [];
|
||||
let skipNext = false;
|
||||
for (const arg of process.execArgv) {
|
||||
if (skipNext) {
|
||||
skipNext = false;
|
||||
continue;
|
||||
}
|
||||
if (WORKER_UNSAFE_EXEC_ARGV_FLAGS.has(arg)) {
|
||||
continue;
|
||||
}
|
||||
if (WORKER_UNSAFE_EXEC_ARGV_FLAGS_WITH_VALUE.has(arg)) {
|
||||
skipNext = true;
|
||||
continue;
|
||||
}
|
||||
if (WORKER_UNSAFE_EXEC_ARGV_OPTION_PREFIXES.some((prefix) => arg.startsWith(prefix))) {
|
||||
continue;
|
||||
}
|
||||
args.push(arg);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
class LocalEmbeddingWorkerClient {
|
||||
private child: ChildProcess | null = null;
|
||||
private nextRequestId = 1;
|
||||
private pending = new Map<number, PendingRequest>();
|
||||
|
||||
constructor(private readonly scriptPath: string) {}
|
||||
|
||||
async initialize(options: EmbeddingProviderOptions): Promise<void> {
|
||||
await this.send({ type: "initialize", options });
|
||||
}
|
||||
|
||||
async embedQuery(
|
||||
options: EmbeddingProviderOptions,
|
||||
text: string,
|
||||
callOptions?: EmbeddingProviderCallOptions,
|
||||
): Promise<number[]> {
|
||||
const result = await this.send({ type: "embedQuery", options, text }, callOptions);
|
||||
return Array.isArray(result) ? (result as number[]) : [];
|
||||
}
|
||||
|
||||
async embedBatch(
|
||||
options: EmbeddingProviderOptions,
|
||||
texts: string[],
|
||||
callOptions?: EmbeddingProviderCallOptions,
|
||||
): Promise<number[][]> {
|
||||
const result = await this.send({ type: "embedBatch", options, texts }, callOptions);
|
||||
return Array.isArray(result) ? (result as number[][]) : [];
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
const child = this.child;
|
||||
if (!child) {
|
||||
return;
|
||||
}
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
const closeRequest = this.send({ type: "close" }).then(() => "closed" as const);
|
||||
const closeTimeout = new Promise<"timeout">((resolve) => {
|
||||
timeout = setTimeout(() => resolve("timeout"), WORKER_CLOSE_GRACE_MS);
|
||||
timeout.unref?.();
|
||||
});
|
||||
try {
|
||||
const result = await Promise.race([closeRequest, closeTimeout]);
|
||||
if (result === "timeout") {
|
||||
closeRequest.catch(() => {});
|
||||
}
|
||||
} finally {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
this.shutdownChild();
|
||||
}
|
||||
}
|
||||
|
||||
private ensureChild(): ChildProcess {
|
||||
if (this.child?.connected) {
|
||||
return this.child;
|
||||
}
|
||||
|
||||
const child = fork(this.scriptPath, [], {
|
||||
execArgv: resolveWorkerExecArgv(),
|
||||
serialization: "json",
|
||||
stdio: ["ignore", "ignore", "ignore", "ipc"],
|
||||
});
|
||||
child.on("message", (message) => this.handleMessage(message));
|
||||
child.on("exit", (code, signal) => {
|
||||
if (this.child === child) {
|
||||
this.child = null;
|
||||
}
|
||||
this.rejectPending(createWorkerExitError(code, signal));
|
||||
});
|
||||
child.on("error", (err) => {
|
||||
if (this.child === child) {
|
||||
this.child = null;
|
||||
}
|
||||
this.rejectPending(
|
||||
createLocalEmbeddingWorkerFailureError({
|
||||
message: `Local embedding worker process failed: ${err.message}`,
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.processError,
|
||||
reason: "process-error",
|
||||
cause: err,
|
||||
}),
|
||||
);
|
||||
});
|
||||
this.child = child;
|
||||
return child;
|
||||
}
|
||||
|
||||
private async send(
|
||||
request: LocalEmbeddingWorkerRequestPayload,
|
||||
options?: EmbeddingProviderCallOptions,
|
||||
): Promise<number[] | number[][] | undefined> {
|
||||
options?.signal?.throwIfAborted();
|
||||
const child = this.ensureChild();
|
||||
const id = this.nextRequestId++;
|
||||
const payload = { ...request, id } as LocalEmbeddingWorkerRequest;
|
||||
return await new Promise((resolve, reject) => {
|
||||
const pending: PendingRequest = { resolve, reject };
|
||||
if (options?.signal) {
|
||||
const abort = () => {
|
||||
this.pending.delete(id);
|
||||
this.shutdownChild();
|
||||
reject(options.signal?.reason ?? new Error("Local embedding request aborted"));
|
||||
};
|
||||
options.signal.addEventListener("abort", abort, { once: true });
|
||||
pending.abort = () => options.signal?.removeEventListener("abort", abort);
|
||||
}
|
||||
this.pending.set(id, pending);
|
||||
child.send(payload, (err) => {
|
||||
if (err) {
|
||||
this.pending.delete(id);
|
||||
pending.abort?.();
|
||||
reject(
|
||||
createLocalEmbeddingWorkerFailureError({
|
||||
message: `Local embedding worker IPC failed: ${err.message}`,
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.ipcError,
|
||||
reason: "ipc",
|
||||
cause: err,
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private handleMessage(message: unknown): void {
|
||||
const response = message as Partial<LocalEmbeddingWorkerResponse>;
|
||||
if (typeof response.id !== "number") {
|
||||
return;
|
||||
}
|
||||
const pending = this.pending.get(response.id);
|
||||
if (!pending) {
|
||||
return;
|
||||
}
|
||||
this.pending.delete(response.id);
|
||||
pending.abort?.();
|
||||
if (response.ok) {
|
||||
pending.resolve(response.value);
|
||||
return;
|
||||
}
|
||||
pending.reject(
|
||||
createWorkerResponseError(response as LocalEmbeddingWorkerResponse & { ok: false }),
|
||||
);
|
||||
}
|
||||
|
||||
private shutdownChild(): void {
|
||||
const child = this.child;
|
||||
this.child = null;
|
||||
if (!child) {
|
||||
return;
|
||||
}
|
||||
if (child.connected) {
|
||||
child.disconnect();
|
||||
}
|
||||
if (!child.killed) {
|
||||
child.kill();
|
||||
}
|
||||
}
|
||||
|
||||
private rejectPending(err: unknown): void {
|
||||
const pending = [...this.pending.values()];
|
||||
this.pending.clear();
|
||||
for (const entry of pending) {
|
||||
entry.abort?.();
|
||||
entry.reject(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function createLocalEmbeddingWorkerProvider(
|
||||
options: EmbeddingProviderOptions,
|
||||
runtimeOptions?: LocalEmbeddingProviderRuntimeOptions,
|
||||
): Promise<EmbeddingProvider> {
|
||||
const modelPath = normalizeOptionalString(options.local?.modelPath) || DEFAULT_LOCAL_MODEL;
|
||||
const workerOptions = serializeLocalEmbeddingOptions(options);
|
||||
const client = new LocalEmbeddingWorkerClient(
|
||||
runtimeOptions?.workerScriptPath ?? resolveDefaultWorkerScriptPath(),
|
||||
);
|
||||
try {
|
||||
await client.initialize(workerOptions);
|
||||
} catch (err) {
|
||||
await client.close().catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
let closed = false;
|
||||
|
||||
const throwIfClosed = () => {
|
||||
if (closed) {
|
||||
throw new Error("Local embedding provider has been closed");
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
id: "local",
|
||||
model: modelPath,
|
||||
embedQuery: async (text, callOptions) => {
|
||||
throwIfClosed();
|
||||
return await client.embedQuery(workerOptions, text, callOptions);
|
||||
},
|
||||
embedBatch: async (texts, callOptions) => {
|
||||
throwIfClosed();
|
||||
return await client.embedBatch(workerOptions, texts, callOptions);
|
||||
},
|
||||
close: async () => {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
closed = true;
|
||||
await client.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,5 +1,14 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createLocalEmbeddingProvider, DEFAULT_LOCAL_MODEL } from "./embeddings.js";
|
||||
import { LOCAL_EMBEDDING_WORKER_ERROR_CODES } from "./embedding-worker-errors.js";
|
||||
import { createLocalEmbeddingWorkerProvider } from "./embeddings-worker.js";
|
||||
import {
|
||||
createLocalEmbeddingProvider,
|
||||
createLocalEmbeddingProviderInProcess,
|
||||
DEFAULT_LOCAL_MODEL,
|
||||
} from "./embeddings.js";
|
||||
|
||||
const nodeLlamaMock = vi.hoisted(() => ({
|
||||
importNodeLlamaCpp: vi.fn(),
|
||||
@@ -39,10 +48,11 @@ function mockLocalEmbeddingRuntime(vector = new Float32Array([2.35, 3.45, 0.63,
|
||||
.fn()
|
||||
.mockResolvedValue({ getEmbeddingFor, dispose: disposeContext });
|
||||
const loadModel = vi.fn().mockResolvedValue({ createEmbeddingContext, dispose: disposeModel });
|
||||
const getLlama = vi.fn(async () => ({ loadModel, dispose: disposeLlama }));
|
||||
const resolveModelFile = vi.fn(async (modelPath: string) => `/resolved/${modelPath}`);
|
||||
|
||||
nodeLlamaMock.importNodeLlamaCpp.mockResolvedValue({
|
||||
getLlama: async () => ({ loadModel, dispose: disposeLlama }),
|
||||
getLlama,
|
||||
resolveModelFile,
|
||||
LlamaLogLevel: { error: 0 },
|
||||
} as never);
|
||||
@@ -52,6 +62,7 @@ function mockLocalEmbeddingRuntime(vector = new Float32Array([2.35, 3.45, 0.63,
|
||||
disposeContext,
|
||||
disposeLlama,
|
||||
disposeModel,
|
||||
getLlama,
|
||||
getEmbeddingFor,
|
||||
loadModel,
|
||||
resolveModelFile,
|
||||
@@ -62,7 +73,7 @@ describe("local embedding provider", () => {
|
||||
it("normalizes local embeddings and resolves the default local model", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
@@ -92,7 +103,7 @@ describe("local embedding provider", () => {
|
||||
it("passes default contextSize (4096) to createEmbeddingContext when not configured", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
@@ -109,7 +120,7 @@ describe("local embedding provider", () => {
|
||||
it("passes configured contextSize to createEmbeddingContext", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
@@ -127,7 +138,7 @@ describe("local embedding provider", () => {
|
||||
it('passes "auto" contextSize to createEmbeddingContext when explicitly set', async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
@@ -142,10 +153,43 @@ describe("local embedding provider", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("runs local batch embeddings sequentially", async () => {
|
||||
const calls: string[] = [];
|
||||
const firstGate = createDeferred<{ vector: Float32Array }>();
|
||||
const secondGate = createDeferred<{ vector: Float32Array }>();
|
||||
const getEmbeddingFor = vi.fn((text: string) => {
|
||||
calls.push(text);
|
||||
return text === "first" ? firstGate.promise : secondGate.promise;
|
||||
});
|
||||
nodeLlamaMock.importNodeLlamaCpp.mockResolvedValue({
|
||||
getLlama: vi.fn(async () => ({
|
||||
loadModel: vi.fn(async () => ({
|
||||
createEmbeddingContext: vi.fn(async () => ({ getEmbeddingFor })),
|
||||
})),
|
||||
})),
|
||||
resolveModelFile: vi.fn(async () => "/resolved/model.gguf"),
|
||||
LlamaLogLevel: { error: 0 },
|
||||
} as never);
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
const batchPromise = provider.embedBatch(["first", "second"]);
|
||||
await expect.poll(() => calls.join(",")).toBe("first");
|
||||
firstGate.resolve({ vector: new Float32Array([1, 0]) });
|
||||
await expect.poll(() => calls.join(",")).toBe("first,second");
|
||||
secondGate.resolve({ vector: new Float32Array([0, 1]) });
|
||||
|
||||
await expect(batchPromise).resolves.toHaveLength(2);
|
||||
});
|
||||
|
||||
it("trims explicit local model paths and cache directories", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime(new Float32Array([1, 0]));
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
@@ -172,7 +216,7 @@ describe("local embedding provider", () => {
|
||||
it("disposes cached local llama resources when closed", async () => {
|
||||
const runtime = mockLocalEmbeddingRuntime();
|
||||
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
@@ -199,7 +243,7 @@ describe("local embedding provider", () => {
|
||||
resolveModelFile: vi.fn(async (modelPath: string) => `/resolved/${modelPath}`),
|
||||
LlamaLogLevel: { error: 0 },
|
||||
} as never);
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
@@ -235,7 +279,7 @@ describe("local embedding provider", () => {
|
||||
}),
|
||||
LlamaLogLevel: { error: 0 },
|
||||
} as never);
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
@@ -274,7 +318,7 @@ describe("local embedding provider", () => {
|
||||
resolveModelFile: vi.fn(async () => "/resolved/model.gguf"),
|
||||
LlamaLogLevel: { error: 0 },
|
||||
} as never);
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
const provider = await createLocalEmbeddingProviderInProcess({
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
@@ -291,4 +335,226 @@ describe("local embedding provider", () => {
|
||||
createContextGate.reject(new Error("context create aborted"));
|
||||
await expect(embedPromise).rejects.toThrow("context create aborted");
|
||||
});
|
||||
|
||||
it("uses a worker process for the public local provider", async () => {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-local-embedding-worker-"));
|
||||
const workerScript = path.join(tempDir, "worker.cjs");
|
||||
await fs.writeFile(
|
||||
workerScript,
|
||||
`
|
||||
process.on("message", (message) => {
|
||||
if (message.type === "initialize") {
|
||||
process.send({ id: message.id, ok: true });
|
||||
return;
|
||||
}
|
||||
if (message.type === "embedQuery") {
|
||||
process.send({ id: message.id, ok: true, value: [1, 0] });
|
||||
return;
|
||||
}
|
||||
if (message.type === "embedBatch") {
|
||||
process.send({ id: message.id, ok: true, value: message.texts.map(() => [0, 1]) });
|
||||
return;
|
||||
}
|
||||
process.send({ id: message.id, ok: true });
|
||||
});
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const provider = await createLocalEmbeddingWorkerProvider(
|
||||
{
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
},
|
||||
{ workerScriptPath: workerScript },
|
||||
);
|
||||
|
||||
await expect(provider.embedQuery("hello")).resolves.toEqual([1, 0]);
|
||||
await expect(provider.embedBatch(["a", "b"])).resolves.toEqual([
|
||||
[0, 1],
|
||||
[0, 1],
|
||||
]);
|
||||
await expect(provider.close?.()).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("terminates the worker when close runs behind a pending request", async () => {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-local-embedding-worker-"));
|
||||
const workerScript = path.join(tempDir, "worker.cjs");
|
||||
const embedStartedPath = path.join(tempDir, "embed-started");
|
||||
await fs.writeFile(
|
||||
workerScript,
|
||||
`
|
||||
const fs = require("node:fs");
|
||||
const embedStartedPath = ${JSON.stringify(embedStartedPath)};
|
||||
let busy = false;
|
||||
|
||||
process.on("message", (message) => {
|
||||
if (busy) {
|
||||
return;
|
||||
}
|
||||
if (message.type === "initialize") {
|
||||
process.send({ id: message.id, ok: true });
|
||||
return;
|
||||
}
|
||||
if (message.type === "embedQuery") {
|
||||
busy = true;
|
||||
fs.writeFileSync(embedStartedPath, "1");
|
||||
}
|
||||
});
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const provider = await createLocalEmbeddingWorkerProvider(
|
||||
{
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
},
|
||||
{ workerScriptPath: workerScript },
|
||||
);
|
||||
|
||||
const embedPromise = provider.embedQuery("stuck");
|
||||
const embedError = embedPromise.then(
|
||||
() => undefined,
|
||||
(err) => err,
|
||||
);
|
||||
await expect
|
||||
.poll(async () => {
|
||||
try {
|
||||
await fs.access(embedStartedPath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})
|
||||
.toBe(true);
|
||||
|
||||
const closePromise = provider.close?.() ?? Promise.resolve();
|
||||
const closeResult = await Promise.race([
|
||||
closePromise.then(() => "closed" as const),
|
||||
new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 1_000)),
|
||||
]);
|
||||
|
||||
expect(closeResult).toBe("closed");
|
||||
await expect(embedError).resolves.toMatchObject({
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.exited,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not pass inline-source or inspector exec args to the file-backed worker", async () => {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-local-embedding-worker-"));
|
||||
const workerScript = path.join(tempDir, "worker.cjs");
|
||||
await fs.writeFile(
|
||||
workerScript,
|
||||
`
|
||||
process.on("message", (message) => {
|
||||
if (message.type === "initialize" || message.type === "close") {
|
||||
process.send({ id: message.id, ok: true });
|
||||
return;
|
||||
}
|
||||
process.send({ id: message.id, ok: true, value: [process.execArgv.length] });
|
||||
});
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const originalExecArgv = [...process.execArgv];
|
||||
let provider: Awaited<ReturnType<typeof createLocalEmbeddingWorkerProvider>> | undefined;
|
||||
try {
|
||||
process.execArgv.splice(
|
||||
0,
|
||||
process.execArgv.length,
|
||||
"--eval",
|
||||
"setInterval(() => {}, 1000)",
|
||||
"--print",
|
||||
"1 + 1",
|
||||
"--input-type=module",
|
||||
"--inspect-brk=127.0.0.1:0",
|
||||
"--inspect-port",
|
||||
"0",
|
||||
);
|
||||
provider = await createLocalEmbeddingWorkerProvider(
|
||||
{
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
},
|
||||
{ workerScriptPath: workerScript },
|
||||
);
|
||||
await expect(provider.embedQuery("hello")).resolves.toEqual([0]);
|
||||
} finally {
|
||||
process.execArgv.splice(0, process.execArgv.length, ...originalExecArgv);
|
||||
await provider?.close?.();
|
||||
}
|
||||
});
|
||||
|
||||
it("reports worker initialization failures during provider creation", async () => {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-local-embedding-worker-"));
|
||||
const workerScript = path.join(tempDir, "worker.cjs");
|
||||
await fs.writeFile(
|
||||
workerScript,
|
||||
`
|
||||
process.on("message", (message) => {
|
||||
process.send({
|
||||
id: message.id,
|
||||
ok: false,
|
||||
error: { message: "Cannot find package 'node-llama-cpp'", code: "ERR_MODULE_NOT_FOUND" },
|
||||
});
|
||||
});
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
try {
|
||||
await createLocalEmbeddingWorkerProvider(
|
||||
{
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
},
|
||||
{ workerScriptPath: workerScript },
|
||||
);
|
||||
throw new Error("expected local embedding provider creation to fail");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(Error);
|
||||
expect((err as Error).message).toBe("Cannot find package 'node-llama-cpp'");
|
||||
expect((err as Error & { code?: string }).code).toBe("ERR_MODULE_NOT_FOUND");
|
||||
}
|
||||
});
|
||||
|
||||
it("reports worker exits with structured failure codes", async () => {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-local-embedding-worker-"));
|
||||
const workerScript = path.join(tempDir, "worker.cjs");
|
||||
await fs.writeFile(
|
||||
workerScript,
|
||||
`
|
||||
process.on("message", (message) => {
|
||||
if (message.type === "initialize") {
|
||||
process.send({ id: message.id, ok: true });
|
||||
return;
|
||||
}
|
||||
process.exit(134);
|
||||
});
|
||||
`,
|
||||
"utf8",
|
||||
);
|
||||
const provider = await createLocalEmbeddingWorkerProvider(
|
||||
{
|
||||
config: {} as never,
|
||||
provider: "local",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
},
|
||||
{ workerScriptPath: workerScript },
|
||||
);
|
||||
|
||||
await expect(provider.embedQuery("hello")).rejects.toMatchObject({
|
||||
code: LOCAL_EMBEDDING_WORKER_ERROR_CODES.exited,
|
||||
reason: "exit",
|
||||
exitCode: 134,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { DEFAULT_LOCAL_MODEL } from "./embedding-defaults.js";
|
||||
import { sanitizeAndNormalizeEmbedding } from "./embedding-vectors.js";
|
||||
import { createLocalEmbeddingWorkerProvider } from "./embeddings-worker.js";
|
||||
import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.types.js";
|
||||
import {
|
||||
importNodeLlamaCpp,
|
||||
@@ -24,6 +25,10 @@ export type {
|
||||
|
||||
export { DEFAULT_LOCAL_MODEL } from "./embedding-defaults.js";
|
||||
|
||||
export type LocalEmbeddingProviderRuntimeOptions = {
|
||||
workerScriptPath?: string;
|
||||
};
|
||||
|
||||
async function disposeResources(
|
||||
resources: Array<DisposableResource | null | undefined>,
|
||||
): Promise<void> {
|
||||
@@ -42,6 +47,12 @@ async function disposeResources(
|
||||
|
||||
export async function createLocalEmbeddingProvider(
|
||||
options: EmbeddingProviderOptions,
|
||||
): Promise<EmbeddingProvider> {
|
||||
return await createLocalEmbeddingWorkerProvider(options);
|
||||
}
|
||||
|
||||
export async function createLocalEmbeddingProviderInProcess(
|
||||
options: EmbeddingProviderOptions,
|
||||
): Promise<EmbeddingProvider> {
|
||||
const modelPath = normalizeOptionalString(options.local?.modelPath) || DEFAULT_LOCAL_MODEL;
|
||||
const modelCacheDir = normalizeOptionalString(options.local?.modelCacheDir);
|
||||
@@ -85,7 +96,9 @@ export async function createLocalEmbeddingProvider(
|
||||
initAbortController = abortController;
|
||||
try {
|
||||
if (!llama) {
|
||||
const nextLlama = await getLlama({ logLevel: LlamaLogLevel.error });
|
||||
const nextLlama = await getLlama({
|
||||
logLevel: LlamaLogLevel.error,
|
||||
});
|
||||
llama = await disposeAndThrowIfClosed(nextLlama);
|
||||
}
|
||||
if (!embeddingModel) {
|
||||
@@ -138,14 +151,13 @@ export async function createLocalEmbeddingProvider(
|
||||
const ctx = await ensureContext();
|
||||
throwIfClosed();
|
||||
options?.signal?.throwIfAborted();
|
||||
const embeddings = await Promise.all(
|
||||
texts.map(async (text) => {
|
||||
throwIfClosed();
|
||||
options?.signal?.throwIfAborted();
|
||||
const embedding = await ctx.getEmbeddingFor(text);
|
||||
return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector));
|
||||
}),
|
||||
);
|
||||
const embeddings: number[][] = [];
|
||||
for (const text of texts) {
|
||||
throwIfClosed();
|
||||
options?.signal?.throwIfAborted();
|
||||
const embedding = await ctx.getEmbeddingFor(text);
|
||||
embeddings.push(sanitizeAndNormalizeEmbedding(Array.from(embedding.vector)));
|
||||
}
|
||||
return embeddings;
|
||||
},
|
||||
close: async () => {
|
||||
|
||||
@@ -56,6 +56,23 @@ describe("config schema regressions", () => {
|
||||
expect(res.ok).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects local memorySearch GPU policy", () => {
|
||||
const res = validateConfigObject({
|
||||
agents: {
|
||||
defaults: {
|
||||
memorySearch: {
|
||||
provider: "local",
|
||||
local: {
|
||||
gpu: "cpu",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(res.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts memorySearch.qmd.extraCollections", () => {
|
||||
const res = validateConfigObject({
|
||||
agents: {
|
||||
|
||||
@@ -62,6 +62,7 @@ export type MemoryEmbeddingProviderCreateOptions = {
|
||||
local?: {
|
||||
modelPath?: string;
|
||||
modelCacheDir?: string;
|
||||
contextSize?: number | "auto";
|
||||
};
|
||||
outputDimensionality?: number;
|
||||
taskType?:
|
||||
|
||||
@@ -323,6 +323,8 @@ function buildUnifiedDistEntries(): Record<string, string> {
|
||||
"plugin-sdk/qa-runtime": "src/plugin-sdk/qa-runtime.ts",
|
||||
}
|
||||
: {}),
|
||||
"memory-core-local-embedding-worker":
|
||||
"packages/memory-host-sdk/src/host/embeddings-worker-child.ts",
|
||||
...listBundledPluginEntrySources(rootBundledPluginBuildEntries),
|
||||
...bundledHookEntries,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user