perf: reduce hot-path scans

This commit is contained in:
Vincent Koc
2026-06-23 09:54:33 +08:00
parent 1252378018
commit 53f64525be
12 changed files with 355 additions and 52 deletions
@@ -19,6 +19,7 @@ import {
import {
applyShortTermPromotions,
auditShortTermPromotionArtifacts,
filterLiveShortTermRecallEntries,
isShortTermMemoryPath,
loadShortTermPromotionDreamingStats,
recordGroundedShortTermCandidates,
@@ -171,6 +172,42 @@ describe("short-term promotion", () => {
});
});
it("deduplicates source-file checks within a recall batch", async () => {
await withTempWorkspace(async (workspaceDir) => {
const notePath = await writeDailyMemoryNote(workspaceDir, "2026-04-03", [
"Deduplicated source check note.",
]);
const relativePath = path.relative(workspaceDir, notePath).replaceAll("\\", "/");
const entry = {
key: "duplicate-source",
path: relativePath,
startLine: 1,
endLine: 1,
source: "memory" as const,
snippet: "Deduplicated source check note.",
recallCount: 1,
dailyCount: 1,
groundedCount: 0,
totalScore: 0.9,
maxScore: 0.9,
firstRecalledAt: "2026-04-03T00:00:00.000Z",
lastRecalledAt: "2026-04-03T00:00:00.000Z",
queryHashes: ["query"],
recallDays: ["2026-04-03"],
conceptTags: [],
};
const statSpy = vi.spyOn(fs, "stat");
const live = await filterLiveShortTermRecallEntries({
workspaceDir,
entries: [entry, { ...entry, key: "duplicate-source-2" }],
});
expect(live).toHaveLength(2);
expect(statSpy).toHaveBeenCalledTimes(1);
});
});
it("falls back when the injected recall timestamp is outside Date range", async () => {
vi.spyOn(Date, "now").mockReturnValue(Date.UTC(2026, 4, 30, 12, 0, 0));
await withTempWorkspace(async (workspaceDir) => {
@@ -1302,39 +1302,47 @@ export async function loadShortTermPromotionDreamingStats(params: {
};
}
async function shortTermRecallSourceExists(params: {
workspaceDir: string;
entry: Pick<ShortTermRecallEntry, "path">;
}): Promise<boolean> {
const workspaceDir = params.workspaceDir.trim();
if (!workspaceDir) {
return false;
}
for (const sourcePath of resolveShortTermSourcePathCandidates(workspaceDir, params.entry.path)) {
try {
const stat = await fs.stat(sourcePath);
if (stat.isFile()) {
return true;
}
} catch (err) {
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
continue;
}
throw err;
async function shortTermRecallSourceIsFile(sourcePath: string): Promise<boolean> {
try {
const stat = await fs.stat(sourcePath);
return stat.isFile();
} catch (err) {
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
return false;
}
throw err;
}
return false;
}
export async function filterLiveShortTermRecallEntries(params: {
workspaceDir: string;
entries: ShortTermRecallEntry[];
}): Promise<ShortTermRecallEntry[]> {
const workspaceDir = params.workspaceDir.trim();
if (!workspaceDir) {
return [];
}
const sourceFileChecks = new Map<string, Promise<boolean>>();
const checkSourceFile = (sourcePath: string): Promise<boolean> => {
const existing = sourceFileChecks.get(sourcePath);
if (existing) {
return existing;
}
const check = shortTermRecallSourceIsFile(sourcePath);
sourceFileChecks.set(sourcePath, check);
return check;
};
const results = await Promise.all(
params.entries.map(async (entry) => ({
entry,
exists: await shortTermRecallSourceExists({ workspaceDir: params.workspaceDir, entry }),
})),
params.entries.map(async (entry) => {
let exists = false;
for (const sourcePath of resolveShortTermSourcePathCandidates(workspaceDir, entry.path)) {
if (await checkSourceFile(sourcePath)) {
exists = true;
break;
}
}
return { entry, exists };
}),
);
return results.filter((result) => result.exists).map((result) => result.entry);
}
@@ -107,6 +107,19 @@ describe("renderMarkdownIRChunksWithinLimit", () => {
expect(chunks.map((chunk) => chunk.source.text)).toEqual(["A", "😀"]);
});
it("keeps split order while processing the worklist as a stack", () => {
const text = "abcdefghijklmnopqrstuvwx";
const chunks = renderMarkdownIRChunksWithinLimit({
ir: markdownToIR(text),
limit: 5,
renderChunk: (chunk) => chunk.text,
measureRendered: (rendered) => rendered.length,
});
expect(chunks.map((chunk) => chunk.source.text).join("")).toBe(text);
expect(chunks.every((chunk) => chunk.rendered.length <= 5)).toBe(true);
});
it("treats Infinity as no size cap and returns a single chunk", () => {
const text = "one two three four five six seven eight nine ten";
const ir = markdownToIR(text);
@@ -56,11 +56,14 @@ export function renderMarkdownIRChunksWithinLimit<TRendered>(
}
const normalizedLimit = resolveIntegerOption(options.limit, 1, { min: 1 });
const pending = chunkMarkdownIR(options.ir, normalizedLimit);
// Treat the pending worklist as a stack so each dequeue/enqueue stays O(1).
// The initial reverse keeps the final order stable while avoiding shift/unshift
// moving every remaining chunk for long messages.
const pending = chunkMarkdownIR(options.ir, normalizedLimit).toReversed();
const finalized: MarkdownIR[] = [];
while (pending.length > 0) {
const chunk = pending.shift();
const chunk = pending.pop();
if (!chunk) {
continue;
}
@@ -77,7 +80,12 @@ export function renderMarkdownIRChunksWithinLimit<TRendered>(
finalized.push(chunk);
continue;
}
pending.unshift(...split);
for (let index = split.length - 1; index >= 0; index -= 1) {
const next = split[index];
if (next) {
pending.push(next);
}
}
}
return coalesceWhitespaceOnlyMarkdownIRChunks(finalized, normalizedLimit, options).map(
+7 -4
View File
@@ -2854,6 +2854,7 @@ async function processOpenAICompletionsStream(
const toolCallBlocksByIndex = new Map<number, ToolCallBlock>();
const toolCallBlocksById = new Map<string, ToolCallBlock>();
const toolCallBlockBytes = new WeakMap<ToolCallBlock, number>();
const toolCallBlockIndices = new WeakMap<ToolCallBlock, number>();
let sawStopFinishReason = false;
const blockIndex = () => output.content.length - 1;
const measureUtf8Bytes = (text: string) => Buffer.byteLength(text, "utf8");
@@ -2986,14 +2987,15 @@ async function processOpenAICompletionsStream(
};
currentBlock = block;
output.content.push(block);
toolCallBlockIndices.set(block, output.content.length - 1);
pushStreamEvent({
type: "toolcall_start",
contentIndex: output.content.indexOf(block),
contentIndex: toolCallBlockIndices.get(block) ?? -1,
partial: output,
});
pushStreamEvent({
type: "toolcall_delta",
contentIndex: output.content.indexOf(block),
contentIndex: toolCallBlockIndices.get(block) ?? -1,
delta: toolCall.partialArgs,
partial: output,
});
@@ -3186,9 +3188,10 @@ async function processOpenAICompletionsStream(
...(initialSig ? { thoughtSignature: initialSig } : {}),
};
output.content.push(block);
toolCallBlockIndices.set(block, output.content.length - 1);
pushStreamEvent({
type: "toolcall_start",
contentIndex: output.content.indexOf(block),
contentIndex: toolCallBlockIndices.get(block) ?? -1,
partial: output,
});
}
@@ -3218,7 +3221,7 @@ async function processOpenAICompletionsStream(
block.arguments = parseStreamingJson(block.partialArgs);
pushStreamEvent({
type: "toolcall_delta",
contentIndex: output.content.indexOf(block),
contentIndex: toolCallBlockIndices.get(block) ?? -1,
delta: toolCall.function.arguments,
partial: output,
});
+52 -1
View File
@@ -223,7 +223,7 @@ async function resolveTranscriptLeafIdFromTrailingControls(
return { appendMode: "active" };
}
async function readTranscriptLeafInfo(transcriptPath: string): Promise<TranscriptLeafInfo> {
async function readTranscriptLeafInfoForward(transcriptPath: string): Promise<TranscriptLeafInfo> {
let leafId: string | undefined;
let hasParentLinkedEntries = false;
let nonSessionEntryCount = 0;
@@ -266,6 +266,57 @@ async function readTranscriptLeafInfo(transcriptPath: string): Promise<Transcrip
};
}
async function readTranscriptLeafInfo(transcriptPath: string): Promise<TranscriptLeafInfo> {
let latestEntryId: string | undefined;
for await (const line of streamSessionTranscriptLinesReverse(transcriptPath)) {
const lineInfo = readTranscriptLineInfo(line);
if (!lineInfo.entryId) {
continue;
}
if (lineInfo.invalidLeafControl) {
break;
}
if (lineInfo.leafControl) {
if (latestEntryId) {
const valid = await validateTranscriptLeafControlReferences({
transcriptPath,
leafControlId: lineInfo.entryId,
leafControl: lineInfo.leafControl,
});
if (!valid) {
break;
}
return {
leafId: latestEntryId,
appendMode: lineInfo.leafControl.appendMode === "side" ? "side" : "active",
hasParentLinkedEntries: true,
nonSessionEntryCount: 0,
};
}
const resolvedLeaf = await resolveTranscriptLeafIdFromTrailingControls(transcriptPath);
return {
...(resolvedLeaf.leafId ? { leafId: resolvedLeaf.leafId } : {}),
appendMode: resolvedLeaf.appendMode,
hasParentLinkedEntries: true,
nonSessionEntryCount: 0,
};
}
latestEntryId ??= lineInfo.entryId;
if (lineInfo.isCanonicalEntry && lineInfo.hasParentLinkedEntry) {
return {
leafId: latestEntryId,
appendMode: lineInfo.appendMode === "side" ? "side" : "active",
hasParentLinkedEntries: true,
nonSessionEntryCount: 0,
};
}
// A latest entry without parent linkage may be a legacy linear transcript.
// Fall back to the full scan only when migration detection needs it.
break;
}
return await readTranscriptLeafInfoForward(transcriptPath);
}
async function migrateLinearTranscriptToParentLinked(transcriptPath: string): Promise<{
leafId?: string;
}> {
+103
View File
@@ -1365,6 +1365,28 @@ describe("appendAssistantMessageToSessionTranscript", () => {
}
});
it("uses a reverse tail scan for modern parent-linked appends", async () => {
const sessionFile = resolveSessionTranscriptPathInDir(
"tail-scan-session",
fixture.sessionsDir(),
);
await appendSessionTranscriptMessage({
transcriptPath: sessionFile,
message: { role: "user", content: "root" },
});
const createReadStreamSpy = vi.spyOn(fs, "createReadStream");
try {
await appendSessionTranscriptMessage({
transcriptPath: sessionFile,
message: { role: "assistant", content: "reply" },
});
expect(createReadStreamSpy).not.toHaveBeenCalled();
} finally {
createReadStreamSpy.mockRestore();
}
});
it("separates message and event appends from an unterminated transcript entry", async () => {
const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir());
fs.mkdirSync(path.dirname(sessionFile), { recursive: true });
@@ -1731,6 +1753,87 @@ describe("appendAssistantMessageToSessionTranscript", () => {
).toEqual([activeEntry.id, nextUser.messageId]);
});
it("preserves a side append cursor when metadata follows its leaf control", async () => {
const sessionFile = resolveSessionTranscriptPathInDir(
"side-append-mode-with-trailing-metadata-transcript-session",
fixture.sessionsDir(),
);
const activeEntry = {
type: "message",
id: "active-entry",
parentId: null,
timestamp: "2026-05-30T12:00:00.000Z",
message: { role: "user", content: "active question" },
};
const sideEntry = {
type: "message",
id: "side-entry",
parentId: activeEntry.id,
timestamp: "2026-05-30T12:00:01.000Z",
message: { role: "assistant", content: "first side delivery" },
};
const sideLeaf = {
type: "leaf",
id: "side-leaf",
parentId: sideEntry.id,
timestamp: "2026-05-30T12:00:02.000Z",
targetId: activeEntry.id,
appendParentId: sideEntry.id,
appendMode: "side",
};
const metadata = {
type: "metadata",
id: "post-leaf-metadata",
parentId: sideLeaf.id,
};
fs.writeFileSync(
sessionFile,
[
{
type: "session",
version: 3,
id: "side-append-mode-with-trailing-metadata-transcript-session",
timestamp: "2026-05-30T12:00:00.000Z",
cwd: fixture.sessionsDir(),
},
activeEntry,
sideEntry,
sideLeaf,
metadata,
]
.map((entry) => JSON.stringify(entry))
.join("\n") + "\n",
);
const appended = await appendSessionTranscriptMessage({
transcriptPath: sessionFile,
message: {
role: "assistant",
provider: "openclaw",
model: "delivery-mirror",
content: "second side delivery",
},
});
const appendedEntry = fs
.readFileSync(sessionFile, "utf8")
.trim()
.split("\n")
.map(
(line) =>
JSON.parse(line) as {
id?: string;
parentId?: string | null;
appendMode?: string;
},
)
.find((entry) => entry.id === appended.messageId);
expect(appendedEntry).toMatchObject({
parentId: metadata.id,
appendMode: "side",
});
});
it("ignores dangling leaf references when choosing the direct append parent", async () => {
const sessionFile = resolveSessionTranscriptPathInDir(
"invalid-leaf-append-parent-transcript-session",
@@ -859,6 +859,24 @@ describe("createManagedOutgoingImageBlocks", () => {
expect(requireBlock(blocks).type).toBe("image");
});
it("allows managed inbound image paths before validating explicit roots", async () => {
const inboundPath = path.join(stateDir, "media", "inbound", "inbound.png");
await fs.mkdir(path.dirname(inboundPath), { recursive: true });
await fs.writeFile(inboundPath, Buffer.from(TINY_PNG_BASE64, "base64"));
await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => {
const blocks = await createManagedOutgoingImageBlocks({
sessionKey: "agent:main:main",
mediaUrls: [inboundPath],
stateDir,
localRoots: [path.parse(stateDir).root],
});
expect(blocks).toHaveLength(1);
expect(requireBlock(blocks).type).toBe("image");
});
});
it("rejects relative local image paths that resolve outside allowed roots", async () => {
const allowedWorkspaceDir = path.join(stateDir, "workspace");
const outsidePath = path.join(stateDir, "outside.png");
+32 -7
View File
@@ -9,7 +9,7 @@ import { getRuntimeConfig } from "../config/config.js";
import { resolveStateDir } from "../config/paths.js";
import { readLocalFileSafely } from "../infra/fs-safe.js";
import { tryReadJson, writeJson } from "../infra/json-files.js";
import { assertLocalMediaAllowed } from "../media/local-media-access.js";
import { assertLocalMediaAllowed, resolveLocalMediaRoots } from "../media/local-media-access.js";
import { resolveLocalMediaPath } from "../media/local-media-path.js";
import {
createImageProcessor,
@@ -299,8 +299,14 @@ function parseImageDataUrl(
};
}
async function getVariantStats(filePath: string) {
const { buffer: metadataBuffer, stat } = await readLocalFileSafely({ filePath });
async function getVariantStats(params: { filePath: string; buffer?: Buffer; sizeBytes?: number }) {
const loaded = params.buffer
? { buffer: params.buffer, sizeBytes: params.sizeBytes ?? params.buffer.byteLength }
: await (async () => {
const { buffer, stat } = await readLocalFileSafely({ filePath: params.filePath });
return { buffer, sizeBytes: stat.size };
})();
const metadataBuffer = loaded.buffer;
const metadata = (await getImageMetadata(metadataBuffer).catch(() => null)) ?? {
width: null,
height: null,
@@ -308,7 +314,7 @@ async function getVariantStats(filePath: string) {
return {
width: metadata.width ?? null,
height: metadata.height ?? null,
sizeBytes: Number.isFinite(stat.size) ? stat.size : null,
sizeBytes: Number.isFinite(loaded.sizeBytes) ? loaded.sizeBytes : null,
};
}
@@ -837,6 +843,7 @@ export async function createManagedOutgoingImageBlocks(params: {
const stateDir = params.stateDir ?? resolveStateDir();
const limits = resolveManagedImageAttachmentLimits(params.limits);
const blocks: ManagedImageBlock[] = [];
let resolvedLocalRoots: readonly string[] | undefined;
for (const [index, mediaUrl] of mediaUrls.entries()) {
const fallbackAlt = `Generated image ${index + 1}`;
const parsedDataUrl = parseImageDataUrl(mediaUrl, fallbackAlt, limits);
@@ -864,7 +871,17 @@ export async function createManagedOutgoingImageBlocks(params: {
: await (async () => {
const localMediaPath = resolveLocalMediaPath(mediaUrl);
if (localMediaPath) {
await assertLocalMediaAllowed(localMediaPath, params.localRoots);
const localRoots = params.localRoots;
const localMediaOptions =
localRoots === "any"
? undefined
: {
resolveRoots: async () => {
resolvedLocalRoots ??= await resolveLocalMediaRoots(localRoots);
return resolvedLocalRoots;
},
};
await assertLocalMediaAllowed(localMediaPath, localRoots, localMediaOptions);
}
return await saveMediaSource(
mediaUrl,
@@ -892,7 +909,11 @@ export async function createManagedOutgoingImageBlocks(params: {
: (await readLocalFileSafely({ filePath: savedOriginal.path })).buffer;
validateManagedImageBuffer(originalBuffer, alt, limits);
let originalStats = await getVariantStats(savedOriginal.path);
let originalStats = await getVariantStats({
filePath: savedOriginal.path,
buffer: originalBuffer,
sizeBytes: savedOriginal.size,
});
if (originalStats.sizeBytes != null && originalStats.sizeBytes > limits.maxBytes) {
throw createManagedImageAttachmentError(
`Managed image attachment ${JSON.stringify(alt)} exceeds the ${formatLimitMiB(limits.maxBytes)} byte limit`,
@@ -930,7 +951,11 @@ export async function createManagedOutgoingImageBlocks(params: {
savedOriginalContentType = replacement.contentType ?? resized.contentType;
savedOriginalPath = savedOriginal.path;
originalBuffer = resized.buffer;
originalStats = await getVariantStats(savedOriginal.path);
originalStats = await getVariantStats({
filePath: savedOriginal.path,
buffer: originalBuffer,
sizeBytes: savedOriginal.size,
});
effectiveMetadata = orientManagedImageMetadata(
originalBuffer,
originalStats.width != null && originalStats.height != null
+9 -4
View File
@@ -191,7 +191,12 @@ export const streamOpenAICompletions: StreamFunction<
// text-lane transition) and again by the end-of-stream loop; guard so its
// *_end event is emitted exactly once.
const finishedBlocks = new Set<StreamingBlock>();
const getContentIndex = (block: StreamingBlock) => blocks.indexOf(block);
const contentIndices = new WeakMap<StreamingBlock, number>();
const appendBlock = (block: StreamingBlock) => {
contentIndices.set(block, blocks.length);
blocks.push(block);
};
const getContentIndex = (block: StreamingBlock) => contentIndices.get(block) ?? -1;
const finishBlock = (block: StreamingBlock) => {
const contentIndex = getContentIndex(block);
if (contentIndex === -1 || finishedBlocks.has(block)) {
@@ -229,7 +234,7 @@ export const streamOpenAICompletions: StreamFunction<
const ensureTextBlock = () => {
if (!textBlock) {
textBlock = { type: "text", text: "" };
blocks.push(textBlock);
appendBlock(textBlock);
stream.push({
type: "text_start",
contentIndex: getContentIndex(textBlock),
@@ -245,7 +250,7 @@ export const streamOpenAICompletions: StreamFunction<
thinking: "",
thinkingSignature,
};
blocks.push(thinkingBlock);
appendBlock(thinkingBlock);
stream.push({
type: "thinking_start",
contentIndex: getContentIndex(thinkingBlock),
@@ -307,7 +312,7 @@ export const streamOpenAICompletions: StreamFunction<
if (toolCall.id) {
toolCallBlocksById.set(toolCall.id, block);
}
blocks.push(block);
appendBlock(block);
stream.push({
type: "toolcall_start",
contentIndex: getContentIndex(block),
+35 -8
View File
@@ -34,11 +34,39 @@ export function getDefaultLocalRoots(): readonly string[] {
return getDefaultMediaLocalRoots();
}
/** Resolves an allowlist once for callers that validate several media paths. */
export async function resolveLocalMediaRoots(
localRoots?: readonly string[],
): Promise<readonly string[]> {
const roots = localRoots ?? getDefaultLocalRoots();
return await Promise.all(
roots.map(async (root) => {
let resolvedRoot: string;
try {
resolvedRoot = await fs.realpath(root);
} catch {
resolvedRoot = path.resolve(root);
}
if (resolvedRoot === path.parse(resolvedRoot).root) {
throw new LocalMediaAccessError(
"invalid-root",
`Invalid localRoots entry (refuses filesystem root): ${root}. Pass a narrower directory.`,
);
}
return resolvedRoot;
}),
);
}
/** Verifies that a local media path is managed inbound media or lives under allowed roots. */
export async function assertLocalMediaAllowed(
mediaPath: string,
localRoots: readonly string[] | "any" | undefined,
options?: { inboundRoots?: readonly string[] },
options?: {
inboundRoots?: readonly string[];
resolvedRoots?: readonly string[];
resolveRoots?: () => Promise<readonly string[]>;
},
): Promise<void> {
if (localRoots === "any") {
return;
@@ -86,13 +114,12 @@ export async function assertLocalMediaAllowed(
}
}
for (const root of roots) {
let resolvedRoot: string;
try {
resolvedRoot = await fs.realpath(root);
} catch {
resolvedRoot = path.resolve(root);
}
const resolvedRoots =
options?.resolvedRoots ??
(await options?.resolveRoots?.()) ??
(await resolveLocalMediaRoots(roots));
for (const [index, resolvedRoot] of resolvedRoots.entries()) {
const root = roots[index] ?? resolvedRoot;
if (resolvedRoot === path.parse(resolvedRoot).root) {
throw new LocalMediaAccessError(
"invalid-root",
+6 -1
View File
@@ -11,7 +11,12 @@ export * from "../media/fetch.js";
export * from "../media/ffmpeg-limits.js";
export * from "@openclaw/media-core/inbound-path-policy";
export * from "../media/load-options.js";
export * from "../media/local-media-access.js";
export {
assertLocalMediaAllowed,
getDefaultLocalRoots,
LocalMediaAccessError,
type LocalMediaAccessErrorCode,
} from "../media/local-media-access.js";
export * from "../media/local-roots.js";
export {
IMAGE_REDUCE_QUALITY_STEPS,