mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-26 20:35:39 -06:00
perf: reduce hot-path scans
This commit is contained in:
@@ -19,6 +19,7 @@ import {
|
||||
import {
|
||||
applyShortTermPromotions,
|
||||
auditShortTermPromotionArtifacts,
|
||||
filterLiveShortTermRecallEntries,
|
||||
isShortTermMemoryPath,
|
||||
loadShortTermPromotionDreamingStats,
|
||||
recordGroundedShortTermCandidates,
|
||||
@@ -171,6 +172,42 @@ describe("short-term promotion", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("deduplicates source-file checks within a recall batch", async () => {
|
||||
await withTempWorkspace(async (workspaceDir) => {
|
||||
const notePath = await writeDailyMemoryNote(workspaceDir, "2026-04-03", [
|
||||
"Deduplicated source check note.",
|
||||
]);
|
||||
const relativePath = path.relative(workspaceDir, notePath).replaceAll("\\", "/");
|
||||
const entry = {
|
||||
key: "duplicate-source",
|
||||
path: relativePath,
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
source: "memory" as const,
|
||||
snippet: "Deduplicated source check note.",
|
||||
recallCount: 1,
|
||||
dailyCount: 1,
|
||||
groundedCount: 0,
|
||||
totalScore: 0.9,
|
||||
maxScore: 0.9,
|
||||
firstRecalledAt: "2026-04-03T00:00:00.000Z",
|
||||
lastRecalledAt: "2026-04-03T00:00:00.000Z",
|
||||
queryHashes: ["query"],
|
||||
recallDays: ["2026-04-03"],
|
||||
conceptTags: [],
|
||||
};
|
||||
const statSpy = vi.spyOn(fs, "stat");
|
||||
|
||||
const live = await filterLiveShortTermRecallEntries({
|
||||
workspaceDir,
|
||||
entries: [entry, { ...entry, key: "duplicate-source-2" }],
|
||||
});
|
||||
|
||||
expect(live).toHaveLength(2);
|
||||
expect(statSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back when the injected recall timestamp is outside Date range", async () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(Date.UTC(2026, 4, 30, 12, 0, 0));
|
||||
await withTempWorkspace(async (workspaceDir) => {
|
||||
|
||||
@@ -1302,39 +1302,47 @@ export async function loadShortTermPromotionDreamingStats(params: {
|
||||
};
|
||||
}
|
||||
|
||||
async function shortTermRecallSourceExists(params: {
|
||||
workspaceDir: string;
|
||||
entry: Pick<ShortTermRecallEntry, "path">;
|
||||
}): Promise<boolean> {
|
||||
const workspaceDir = params.workspaceDir.trim();
|
||||
if (!workspaceDir) {
|
||||
return false;
|
||||
}
|
||||
for (const sourcePath of resolveShortTermSourcePathCandidates(workspaceDir, params.entry.path)) {
|
||||
try {
|
||||
const stat = await fs.stat(sourcePath);
|
||||
if (stat.isFile()) {
|
||||
return true;
|
||||
}
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
async function shortTermRecallSourceIsFile(sourcePath: string): Promise<boolean> {
|
||||
try {
|
||||
const stat = await fs.stat(sourcePath);
|
||||
return stat.isFile();
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return false;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function filterLiveShortTermRecallEntries(params: {
|
||||
workspaceDir: string;
|
||||
entries: ShortTermRecallEntry[];
|
||||
}): Promise<ShortTermRecallEntry[]> {
|
||||
const workspaceDir = params.workspaceDir.trim();
|
||||
if (!workspaceDir) {
|
||||
return [];
|
||||
}
|
||||
const sourceFileChecks = new Map<string, Promise<boolean>>();
|
||||
const checkSourceFile = (sourcePath: string): Promise<boolean> => {
|
||||
const existing = sourceFileChecks.get(sourcePath);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const check = shortTermRecallSourceIsFile(sourcePath);
|
||||
sourceFileChecks.set(sourcePath, check);
|
||||
return check;
|
||||
};
|
||||
const results = await Promise.all(
|
||||
params.entries.map(async (entry) => ({
|
||||
entry,
|
||||
exists: await shortTermRecallSourceExists({ workspaceDir: params.workspaceDir, entry }),
|
||||
})),
|
||||
params.entries.map(async (entry) => {
|
||||
let exists = false;
|
||||
for (const sourcePath of resolveShortTermSourcePathCandidates(workspaceDir, entry.path)) {
|
||||
if (await checkSourceFile(sourcePath)) {
|
||||
exists = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return { entry, exists };
|
||||
}),
|
||||
);
|
||||
return results.filter((result) => result.exists).map((result) => result.entry);
|
||||
}
|
||||
|
||||
@@ -107,6 +107,19 @@ describe("renderMarkdownIRChunksWithinLimit", () => {
|
||||
expect(chunks.map((chunk) => chunk.source.text)).toEqual(["A", "😀"]);
|
||||
});
|
||||
|
||||
it("keeps split order while processing the worklist as a stack", () => {
|
||||
const text = "abcdefghijklmnopqrstuvwx";
|
||||
const chunks = renderMarkdownIRChunksWithinLimit({
|
||||
ir: markdownToIR(text),
|
||||
limit: 5,
|
||||
renderChunk: (chunk) => chunk.text,
|
||||
measureRendered: (rendered) => rendered.length,
|
||||
});
|
||||
|
||||
expect(chunks.map((chunk) => chunk.source.text).join("")).toBe(text);
|
||||
expect(chunks.every((chunk) => chunk.rendered.length <= 5)).toBe(true);
|
||||
});
|
||||
|
||||
it("treats Infinity as no size cap and returns a single chunk", () => {
|
||||
const text = "one two three four five six seven eight nine ten";
|
||||
const ir = markdownToIR(text);
|
||||
|
||||
@@ -56,11 +56,14 @@ export function renderMarkdownIRChunksWithinLimit<TRendered>(
|
||||
}
|
||||
|
||||
const normalizedLimit = resolveIntegerOption(options.limit, 1, { min: 1 });
|
||||
const pending = chunkMarkdownIR(options.ir, normalizedLimit);
|
||||
// Treat the pending worklist as a stack so each dequeue/enqueue stays O(1).
|
||||
// The initial reverse keeps the final order stable while avoiding shift/unshift
|
||||
// moving every remaining chunk for long messages.
|
||||
const pending = chunkMarkdownIR(options.ir, normalizedLimit).toReversed();
|
||||
const finalized: MarkdownIR[] = [];
|
||||
|
||||
while (pending.length > 0) {
|
||||
const chunk = pending.shift();
|
||||
const chunk = pending.pop();
|
||||
if (!chunk) {
|
||||
continue;
|
||||
}
|
||||
@@ -77,7 +80,12 @@ export function renderMarkdownIRChunksWithinLimit<TRendered>(
|
||||
finalized.push(chunk);
|
||||
continue;
|
||||
}
|
||||
pending.unshift(...split);
|
||||
for (let index = split.length - 1; index >= 0; index -= 1) {
|
||||
const next = split[index];
|
||||
if (next) {
|
||||
pending.push(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return coalesceWhitespaceOnlyMarkdownIRChunks(finalized, normalizedLimit, options).map(
|
||||
|
||||
@@ -2854,6 +2854,7 @@ async function processOpenAICompletionsStream(
|
||||
const toolCallBlocksByIndex = new Map<number, ToolCallBlock>();
|
||||
const toolCallBlocksById = new Map<string, ToolCallBlock>();
|
||||
const toolCallBlockBytes = new WeakMap<ToolCallBlock, number>();
|
||||
const toolCallBlockIndices = new WeakMap<ToolCallBlock, number>();
|
||||
let sawStopFinishReason = false;
|
||||
const blockIndex = () => output.content.length - 1;
|
||||
const measureUtf8Bytes = (text: string) => Buffer.byteLength(text, "utf8");
|
||||
@@ -2986,14 +2987,15 @@ async function processOpenAICompletionsStream(
|
||||
};
|
||||
currentBlock = block;
|
||||
output.content.push(block);
|
||||
toolCallBlockIndices.set(block, output.content.length - 1);
|
||||
pushStreamEvent({
|
||||
type: "toolcall_start",
|
||||
contentIndex: output.content.indexOf(block),
|
||||
contentIndex: toolCallBlockIndices.get(block) ?? -1,
|
||||
partial: output,
|
||||
});
|
||||
pushStreamEvent({
|
||||
type: "toolcall_delta",
|
||||
contentIndex: output.content.indexOf(block),
|
||||
contentIndex: toolCallBlockIndices.get(block) ?? -1,
|
||||
delta: toolCall.partialArgs,
|
||||
partial: output,
|
||||
});
|
||||
@@ -3186,9 +3188,10 @@ async function processOpenAICompletionsStream(
|
||||
...(initialSig ? { thoughtSignature: initialSig } : {}),
|
||||
};
|
||||
output.content.push(block);
|
||||
toolCallBlockIndices.set(block, output.content.length - 1);
|
||||
pushStreamEvent({
|
||||
type: "toolcall_start",
|
||||
contentIndex: output.content.indexOf(block),
|
||||
contentIndex: toolCallBlockIndices.get(block) ?? -1,
|
||||
partial: output,
|
||||
});
|
||||
}
|
||||
@@ -3218,7 +3221,7 @@ async function processOpenAICompletionsStream(
|
||||
block.arguments = parseStreamingJson(block.partialArgs);
|
||||
pushStreamEvent({
|
||||
type: "toolcall_delta",
|
||||
contentIndex: output.content.indexOf(block),
|
||||
contentIndex: toolCallBlockIndices.get(block) ?? -1,
|
||||
delta: toolCall.function.arguments,
|
||||
partial: output,
|
||||
});
|
||||
|
||||
@@ -223,7 +223,7 @@ async function resolveTranscriptLeafIdFromTrailingControls(
|
||||
return { appendMode: "active" };
|
||||
}
|
||||
|
||||
async function readTranscriptLeafInfo(transcriptPath: string): Promise<TranscriptLeafInfo> {
|
||||
async function readTranscriptLeafInfoForward(transcriptPath: string): Promise<TranscriptLeafInfo> {
|
||||
let leafId: string | undefined;
|
||||
let hasParentLinkedEntries = false;
|
||||
let nonSessionEntryCount = 0;
|
||||
@@ -266,6 +266,57 @@ async function readTranscriptLeafInfo(transcriptPath: string): Promise<Transcrip
|
||||
};
|
||||
}
|
||||
|
||||
async function readTranscriptLeafInfo(transcriptPath: string): Promise<TranscriptLeafInfo> {
|
||||
let latestEntryId: string | undefined;
|
||||
for await (const line of streamSessionTranscriptLinesReverse(transcriptPath)) {
|
||||
const lineInfo = readTranscriptLineInfo(line);
|
||||
if (!lineInfo.entryId) {
|
||||
continue;
|
||||
}
|
||||
if (lineInfo.invalidLeafControl) {
|
||||
break;
|
||||
}
|
||||
if (lineInfo.leafControl) {
|
||||
if (latestEntryId) {
|
||||
const valid = await validateTranscriptLeafControlReferences({
|
||||
transcriptPath,
|
||||
leafControlId: lineInfo.entryId,
|
||||
leafControl: lineInfo.leafControl,
|
||||
});
|
||||
if (!valid) {
|
||||
break;
|
||||
}
|
||||
return {
|
||||
leafId: latestEntryId,
|
||||
appendMode: lineInfo.leafControl.appendMode === "side" ? "side" : "active",
|
||||
hasParentLinkedEntries: true,
|
||||
nonSessionEntryCount: 0,
|
||||
};
|
||||
}
|
||||
const resolvedLeaf = await resolveTranscriptLeafIdFromTrailingControls(transcriptPath);
|
||||
return {
|
||||
...(resolvedLeaf.leafId ? { leafId: resolvedLeaf.leafId } : {}),
|
||||
appendMode: resolvedLeaf.appendMode,
|
||||
hasParentLinkedEntries: true,
|
||||
nonSessionEntryCount: 0,
|
||||
};
|
||||
}
|
||||
latestEntryId ??= lineInfo.entryId;
|
||||
if (lineInfo.isCanonicalEntry && lineInfo.hasParentLinkedEntry) {
|
||||
return {
|
||||
leafId: latestEntryId,
|
||||
appendMode: lineInfo.appendMode === "side" ? "side" : "active",
|
||||
hasParentLinkedEntries: true,
|
||||
nonSessionEntryCount: 0,
|
||||
};
|
||||
}
|
||||
// A latest entry without parent linkage may be a legacy linear transcript.
|
||||
// Fall back to the full scan only when migration detection needs it.
|
||||
break;
|
||||
}
|
||||
return await readTranscriptLeafInfoForward(transcriptPath);
|
||||
}
|
||||
|
||||
async function migrateLinearTranscriptToParentLinked(transcriptPath: string): Promise<{
|
||||
leafId?: string;
|
||||
}> {
|
||||
|
||||
@@ -1365,6 +1365,28 @@ describe("appendAssistantMessageToSessionTranscript", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("uses a reverse tail scan for modern parent-linked appends", async () => {
|
||||
const sessionFile = resolveSessionTranscriptPathInDir(
|
||||
"tail-scan-session",
|
||||
fixture.sessionsDir(),
|
||||
);
|
||||
await appendSessionTranscriptMessage({
|
||||
transcriptPath: sessionFile,
|
||||
message: { role: "user", content: "root" },
|
||||
});
|
||||
|
||||
const createReadStreamSpy = vi.spyOn(fs, "createReadStream");
|
||||
try {
|
||||
await appendSessionTranscriptMessage({
|
||||
transcriptPath: sessionFile,
|
||||
message: { role: "assistant", content: "reply" },
|
||||
});
|
||||
expect(createReadStreamSpy).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
createReadStreamSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("separates message and event appends from an unterminated transcript entry", async () => {
|
||||
const sessionFile = resolveSessionTranscriptPathInDir(sessionId, fixture.sessionsDir());
|
||||
fs.mkdirSync(path.dirname(sessionFile), { recursive: true });
|
||||
@@ -1731,6 +1753,87 @@ describe("appendAssistantMessageToSessionTranscript", () => {
|
||||
).toEqual([activeEntry.id, nextUser.messageId]);
|
||||
});
|
||||
|
||||
it("preserves a side append cursor when metadata follows its leaf control", async () => {
|
||||
const sessionFile = resolveSessionTranscriptPathInDir(
|
||||
"side-append-mode-with-trailing-metadata-transcript-session",
|
||||
fixture.sessionsDir(),
|
||||
);
|
||||
const activeEntry = {
|
||||
type: "message",
|
||||
id: "active-entry",
|
||||
parentId: null,
|
||||
timestamp: "2026-05-30T12:00:00.000Z",
|
||||
message: { role: "user", content: "active question" },
|
||||
};
|
||||
const sideEntry = {
|
||||
type: "message",
|
||||
id: "side-entry",
|
||||
parentId: activeEntry.id,
|
||||
timestamp: "2026-05-30T12:00:01.000Z",
|
||||
message: { role: "assistant", content: "first side delivery" },
|
||||
};
|
||||
const sideLeaf = {
|
||||
type: "leaf",
|
||||
id: "side-leaf",
|
||||
parentId: sideEntry.id,
|
||||
timestamp: "2026-05-30T12:00:02.000Z",
|
||||
targetId: activeEntry.id,
|
||||
appendParentId: sideEntry.id,
|
||||
appendMode: "side",
|
||||
};
|
||||
const metadata = {
|
||||
type: "metadata",
|
||||
id: "post-leaf-metadata",
|
||||
parentId: sideLeaf.id,
|
||||
};
|
||||
fs.writeFileSync(
|
||||
sessionFile,
|
||||
[
|
||||
{
|
||||
type: "session",
|
||||
version: 3,
|
||||
id: "side-append-mode-with-trailing-metadata-transcript-session",
|
||||
timestamp: "2026-05-30T12:00:00.000Z",
|
||||
cwd: fixture.sessionsDir(),
|
||||
},
|
||||
activeEntry,
|
||||
sideEntry,
|
||||
sideLeaf,
|
||||
metadata,
|
||||
]
|
||||
.map((entry) => JSON.stringify(entry))
|
||||
.join("\n") + "\n",
|
||||
);
|
||||
|
||||
const appended = await appendSessionTranscriptMessage({
|
||||
transcriptPath: sessionFile,
|
||||
message: {
|
||||
role: "assistant",
|
||||
provider: "openclaw",
|
||||
model: "delivery-mirror",
|
||||
content: "second side delivery",
|
||||
},
|
||||
});
|
||||
|
||||
const appendedEntry = fs
|
||||
.readFileSync(sessionFile, "utf8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map(
|
||||
(line) =>
|
||||
JSON.parse(line) as {
|
||||
id?: string;
|
||||
parentId?: string | null;
|
||||
appendMode?: string;
|
||||
},
|
||||
)
|
||||
.find((entry) => entry.id === appended.messageId);
|
||||
expect(appendedEntry).toMatchObject({
|
||||
parentId: metadata.id,
|
||||
appendMode: "side",
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores dangling leaf references when choosing the direct append parent", async () => {
|
||||
const sessionFile = resolveSessionTranscriptPathInDir(
|
||||
"invalid-leaf-append-parent-transcript-session",
|
||||
|
||||
@@ -859,6 +859,24 @@ describe("createManagedOutgoingImageBlocks", () => {
|
||||
expect(requireBlock(blocks).type).toBe("image");
|
||||
});
|
||||
|
||||
it("allows managed inbound image paths before validating explicit roots", async () => {
|
||||
const inboundPath = path.join(stateDir, "media", "inbound", "inbound.png");
|
||||
await fs.mkdir(path.dirname(inboundPath), { recursive: true });
|
||||
await fs.writeFile(inboundPath, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
|
||||
await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => {
|
||||
const blocks = await createManagedOutgoingImageBlocks({
|
||||
sessionKey: "agent:main:main",
|
||||
mediaUrls: [inboundPath],
|
||||
stateDir,
|
||||
localRoots: [path.parse(stateDir).root],
|
||||
});
|
||||
|
||||
expect(blocks).toHaveLength(1);
|
||||
expect(requireBlock(blocks).type).toBe("image");
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects relative local image paths that resolve outside allowed roots", async () => {
|
||||
const allowedWorkspaceDir = path.join(stateDir, "workspace");
|
||||
const outsidePath = path.join(stateDir, "outside.png");
|
||||
|
||||
@@ -9,7 +9,7 @@ import { getRuntimeConfig } from "../config/config.js";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import { readLocalFileSafely } from "../infra/fs-safe.js";
|
||||
import { tryReadJson, writeJson } from "../infra/json-files.js";
|
||||
import { assertLocalMediaAllowed } from "../media/local-media-access.js";
|
||||
import { assertLocalMediaAllowed, resolveLocalMediaRoots } from "../media/local-media-access.js";
|
||||
import { resolveLocalMediaPath } from "../media/local-media-path.js";
|
||||
import {
|
||||
createImageProcessor,
|
||||
@@ -299,8 +299,14 @@ function parseImageDataUrl(
|
||||
};
|
||||
}
|
||||
|
||||
async function getVariantStats(filePath: string) {
|
||||
const { buffer: metadataBuffer, stat } = await readLocalFileSafely({ filePath });
|
||||
async function getVariantStats(params: { filePath: string; buffer?: Buffer; sizeBytes?: number }) {
|
||||
const loaded = params.buffer
|
||||
? { buffer: params.buffer, sizeBytes: params.sizeBytes ?? params.buffer.byteLength }
|
||||
: await (async () => {
|
||||
const { buffer, stat } = await readLocalFileSafely({ filePath: params.filePath });
|
||||
return { buffer, sizeBytes: stat.size };
|
||||
})();
|
||||
const metadataBuffer = loaded.buffer;
|
||||
const metadata = (await getImageMetadata(metadataBuffer).catch(() => null)) ?? {
|
||||
width: null,
|
||||
height: null,
|
||||
@@ -308,7 +314,7 @@ async function getVariantStats(filePath: string) {
|
||||
return {
|
||||
width: metadata.width ?? null,
|
||||
height: metadata.height ?? null,
|
||||
sizeBytes: Number.isFinite(stat.size) ? stat.size : null,
|
||||
sizeBytes: Number.isFinite(loaded.sizeBytes) ? loaded.sizeBytes : null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -837,6 +843,7 @@ export async function createManagedOutgoingImageBlocks(params: {
|
||||
const stateDir = params.stateDir ?? resolveStateDir();
|
||||
const limits = resolveManagedImageAttachmentLimits(params.limits);
|
||||
const blocks: ManagedImageBlock[] = [];
|
||||
let resolvedLocalRoots: readonly string[] | undefined;
|
||||
for (const [index, mediaUrl] of mediaUrls.entries()) {
|
||||
const fallbackAlt = `Generated image ${index + 1}`;
|
||||
const parsedDataUrl = parseImageDataUrl(mediaUrl, fallbackAlt, limits);
|
||||
@@ -864,7 +871,17 @@ export async function createManagedOutgoingImageBlocks(params: {
|
||||
: await (async () => {
|
||||
const localMediaPath = resolveLocalMediaPath(mediaUrl);
|
||||
if (localMediaPath) {
|
||||
await assertLocalMediaAllowed(localMediaPath, params.localRoots);
|
||||
const localRoots = params.localRoots;
|
||||
const localMediaOptions =
|
||||
localRoots === "any"
|
||||
? undefined
|
||||
: {
|
||||
resolveRoots: async () => {
|
||||
resolvedLocalRoots ??= await resolveLocalMediaRoots(localRoots);
|
||||
return resolvedLocalRoots;
|
||||
},
|
||||
};
|
||||
await assertLocalMediaAllowed(localMediaPath, localRoots, localMediaOptions);
|
||||
}
|
||||
return await saveMediaSource(
|
||||
mediaUrl,
|
||||
@@ -892,7 +909,11 @@ export async function createManagedOutgoingImageBlocks(params: {
|
||||
: (await readLocalFileSafely({ filePath: savedOriginal.path })).buffer;
|
||||
validateManagedImageBuffer(originalBuffer, alt, limits);
|
||||
|
||||
let originalStats = await getVariantStats(savedOriginal.path);
|
||||
let originalStats = await getVariantStats({
|
||||
filePath: savedOriginal.path,
|
||||
buffer: originalBuffer,
|
||||
sizeBytes: savedOriginal.size,
|
||||
});
|
||||
if (originalStats.sizeBytes != null && originalStats.sizeBytes > limits.maxBytes) {
|
||||
throw createManagedImageAttachmentError(
|
||||
`Managed image attachment ${JSON.stringify(alt)} exceeds the ${formatLimitMiB(limits.maxBytes)} byte limit`,
|
||||
@@ -930,7 +951,11 @@ export async function createManagedOutgoingImageBlocks(params: {
|
||||
savedOriginalContentType = replacement.contentType ?? resized.contentType;
|
||||
savedOriginalPath = savedOriginal.path;
|
||||
originalBuffer = resized.buffer;
|
||||
originalStats = await getVariantStats(savedOriginal.path);
|
||||
originalStats = await getVariantStats({
|
||||
filePath: savedOriginal.path,
|
||||
buffer: originalBuffer,
|
||||
sizeBytes: savedOriginal.size,
|
||||
});
|
||||
effectiveMetadata = orientManagedImageMetadata(
|
||||
originalBuffer,
|
||||
originalStats.width != null && originalStats.height != null
|
||||
|
||||
@@ -191,7 +191,12 @@ export const streamOpenAICompletions: StreamFunction<
|
||||
// text-lane transition) and again by the end-of-stream loop; guard so its
|
||||
// *_end event is emitted exactly once.
|
||||
const finishedBlocks = new Set<StreamingBlock>();
|
||||
const getContentIndex = (block: StreamingBlock) => blocks.indexOf(block);
|
||||
const contentIndices = new WeakMap<StreamingBlock, number>();
|
||||
const appendBlock = (block: StreamingBlock) => {
|
||||
contentIndices.set(block, blocks.length);
|
||||
blocks.push(block);
|
||||
};
|
||||
const getContentIndex = (block: StreamingBlock) => contentIndices.get(block) ?? -1;
|
||||
const finishBlock = (block: StreamingBlock) => {
|
||||
const contentIndex = getContentIndex(block);
|
||||
if (contentIndex === -1 || finishedBlocks.has(block)) {
|
||||
@@ -229,7 +234,7 @@ export const streamOpenAICompletions: StreamFunction<
|
||||
const ensureTextBlock = () => {
|
||||
if (!textBlock) {
|
||||
textBlock = { type: "text", text: "" };
|
||||
blocks.push(textBlock);
|
||||
appendBlock(textBlock);
|
||||
stream.push({
|
||||
type: "text_start",
|
||||
contentIndex: getContentIndex(textBlock),
|
||||
@@ -245,7 +250,7 @@ export const streamOpenAICompletions: StreamFunction<
|
||||
thinking: "",
|
||||
thinkingSignature,
|
||||
};
|
||||
blocks.push(thinkingBlock);
|
||||
appendBlock(thinkingBlock);
|
||||
stream.push({
|
||||
type: "thinking_start",
|
||||
contentIndex: getContentIndex(thinkingBlock),
|
||||
@@ -307,7 +312,7 @@ export const streamOpenAICompletions: StreamFunction<
|
||||
if (toolCall.id) {
|
||||
toolCallBlocksById.set(toolCall.id, block);
|
||||
}
|
||||
blocks.push(block);
|
||||
appendBlock(block);
|
||||
stream.push({
|
||||
type: "toolcall_start",
|
||||
contentIndex: getContentIndex(block),
|
||||
|
||||
@@ -34,11 +34,39 @@ export function getDefaultLocalRoots(): readonly string[] {
|
||||
return getDefaultMediaLocalRoots();
|
||||
}
|
||||
|
||||
/** Resolves an allowlist once for callers that validate several media paths. */
|
||||
export async function resolveLocalMediaRoots(
|
||||
localRoots?: readonly string[],
|
||||
): Promise<readonly string[]> {
|
||||
const roots = localRoots ?? getDefaultLocalRoots();
|
||||
return await Promise.all(
|
||||
roots.map(async (root) => {
|
||||
let resolvedRoot: string;
|
||||
try {
|
||||
resolvedRoot = await fs.realpath(root);
|
||||
} catch {
|
||||
resolvedRoot = path.resolve(root);
|
||||
}
|
||||
if (resolvedRoot === path.parse(resolvedRoot).root) {
|
||||
throw new LocalMediaAccessError(
|
||||
"invalid-root",
|
||||
`Invalid localRoots entry (refuses filesystem root): ${root}. Pass a narrower directory.`,
|
||||
);
|
||||
}
|
||||
return resolvedRoot;
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/** Verifies that a local media path is managed inbound media or lives under allowed roots. */
|
||||
export async function assertLocalMediaAllowed(
|
||||
mediaPath: string,
|
||||
localRoots: readonly string[] | "any" | undefined,
|
||||
options?: { inboundRoots?: readonly string[] },
|
||||
options?: {
|
||||
inboundRoots?: readonly string[];
|
||||
resolvedRoots?: readonly string[];
|
||||
resolveRoots?: () => Promise<readonly string[]>;
|
||||
},
|
||||
): Promise<void> {
|
||||
if (localRoots === "any") {
|
||||
return;
|
||||
@@ -86,13 +114,12 @@ export async function assertLocalMediaAllowed(
|
||||
}
|
||||
}
|
||||
|
||||
for (const root of roots) {
|
||||
let resolvedRoot: string;
|
||||
try {
|
||||
resolvedRoot = await fs.realpath(root);
|
||||
} catch {
|
||||
resolvedRoot = path.resolve(root);
|
||||
}
|
||||
const resolvedRoots =
|
||||
options?.resolvedRoots ??
|
||||
(await options?.resolveRoots?.()) ??
|
||||
(await resolveLocalMediaRoots(roots));
|
||||
for (const [index, resolvedRoot] of resolvedRoots.entries()) {
|
||||
const root = roots[index] ?? resolvedRoot;
|
||||
if (resolvedRoot === path.parse(resolvedRoot).root) {
|
||||
throw new LocalMediaAccessError(
|
||||
"invalid-root",
|
||||
|
||||
@@ -11,7 +11,12 @@ export * from "../media/fetch.js";
|
||||
export * from "../media/ffmpeg-limits.js";
|
||||
export * from "@openclaw/media-core/inbound-path-policy";
|
||||
export * from "../media/load-options.js";
|
||||
export * from "../media/local-media-access.js";
|
||||
export {
|
||||
assertLocalMediaAllowed,
|
||||
getDefaultLocalRoots,
|
||||
LocalMediaAccessError,
|
||||
type LocalMediaAccessErrorCode,
|
||||
} from "../media/local-media-access.js";
|
||||
export * from "../media/local-roots.js";
|
||||
export {
|
||||
IMAGE_REDUCE_QUALITY_STEPS,
|
||||
|
||||
Reference in New Issue
Block a user