mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-16 23:52:40 -06:00
0a8e3604ba
* refactor(sessions): migrate runtime storage to sqlite * test(sessions): fix sqlite CI regressions * test(sessions): align remaining sqlite fixtures * fix(codex): require sqlite trajectory recorder * test(sessions): align orphan recovery sqlite fixture * test(sessions): align sqlite rebase fixtures * fix(sessions): finish current-main integration of the sqlite flip Resolve the whole-store SDK removal across its owner boundary: drop the loadSessionStore re-export and the registry whole-store wrappers, wire hasTrackedActiveSessionRun into gateway chat, complete the preserveLockedHarnessIds cleanup contract, flip the codex thread-history import to storePath targets, and port remaining main-side tests from file-store helpers to session accessor reads. * chore: drop committed pebbles log, revert plugin-inspector bump, refresh generated docs Remove the 1.8k-line .pebbles/events.jsonl work log from the branch, restore the plugin-inspector advisory lane to main's pinned 0.3.10 so the supply-chain bump gets its own review, and regenerate docs_map, the plugin SDK API baseline, and the export-surface ratchet for the merged tree. * feat(sessions): keep archived transcripts by default with zstd cold storage Codex-style retention: deleting or resetting a session archives its transcript as a zstd-compressed JSONL artifact (plain when the runtime lacks node:zlib zstd) and keeps it until the disk budget evicts oldest first. resetArchiveRetention now governs both deleted and reset archives and defaults to keep; maxDiskBytes defaults to 2gb so retention stays bounded, with archives evicted before live sessions. The cron reaper follows the same knob instead of deleting archives on its own timer. * fix(state): converge agent DB migration lineages and bound database growth Merge coherence: run both structure-gated legacy memory-schema repairs (flip-lineage drop, main-lineage identity rebuild) before the flip migration so pre-flip v1/v2 and pre-merge flip v1/v4 databases all converge, and hoist foreign_keys=OFF outside the schema transaction where the pragma was silently ignored and the v1 sessions rebuild cascade-deleted session_entries. Growth guards: fresh agent DBs enable auto_vacuum=INCREMENTAL, WAL maintenance releases freed pages in bounded passes (never a blocking full VACUUM), and doctor reports state/agent DB bloat from freelist stats. * fix(codex): resolve the store path for thread-history import via the SDK The supervision catalog passed the legacy sessionFile locator to the storePath-targeted transcript mirror; resolve the agent store path with the session-store SDK helper instead of a runtime-object seam so test fakes and headless callers need no extra surface. Drop the obsolete missing-session-id preprocessing case: sessions rows are NOT NULL on session_id and upsert repairs id-less patches at write time. * fix(sessions): fail safe on malformed disk-budget config and doctor stat errors A malformed explicit maxDiskBytes disables the budget instead of falling back to the destructive 2gb default the user never chose, and the doctor bloat check skips databases whose paths stat-fail instead of aborting doctor. * fix(sessions): complete sqlite conflict translations * test(sqlite): align hardening checks with maintenance * test(sessions): inspect compressed transcript archives * fix(tests): await session seeds and drop unused helpers flagged by CI lint The five unawaited writeSessionStoreSeed calls raced their SQLite seeds against the assertions, failing compact shards; the bloat probe drops a useless initializer and the merged tests drop now-unused helpers. * test(sessions): type legacy proof events directly * test(sessions): align hardening contracts * perf(sessions): read usage transcript sizes from SQL aggregates Usage/cost scans walked every session and materialized every transcript event just to re-stringify it for a byte estimate — the #86718 stall class reborn on the DB. readTranscriptStatsSync sums stored JSON bytes in SQLite without loading a single row. * fix(sessions): re-root foreign-root transcript paths onto the current sessions dir Restored backups, moved OPENCLAW_STATE_DIR, and rehearsal copies carry absolute sessionFile paths from the old root; the containment fallback kept those foreign paths, so migration read (and would archive) files in the original root and reported local copies missing. Re-root the canonical agents/<id>/sessions suffix onto the current dir when the file exists there; genuine cross-root layouts still fall through unchanged. * test(agents): seed harness admission through sqlite * fix(sqlite): close agent db on pragma setup failure * fix(doctor): compact and retrofit incremental auto-vacuum after session import The migration is the sanctioned offline window: post-import compact reclaims import churn and applies auto_vacuum=INCREMENTAL to databases created before the fresh-DB pragma existed, so runtime maintenance can release pages in bounded passes on every install. --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
772 lines
27 KiB
TypeScript
772 lines
27 KiB
TypeScript
// Codex plugin module implements transcript mirror behavior.
|
|
import { Buffer } from "node:buffer";
|
|
import { createHash } from "node:crypto";
|
|
import {
|
|
embeddedAgentLog,
|
|
formatErrorMessage,
|
|
runAgentHarnessBeforeMessageWriteHook,
|
|
type AgentMessage,
|
|
type EmbeddedRunAttemptParams,
|
|
type EmbeddedRunAttemptResult,
|
|
} from "openclaw/plugin-sdk/agent-harness-runtime";
|
|
import type { AssistantMessage, Usage } from "openclaw/plugin-sdk/llm";
|
|
import {
|
|
publishSessionTranscriptUpdateByIdentity,
|
|
withSessionTranscriptWriteLock,
|
|
type SessionTranscriptTargetParams,
|
|
type SessionTranscriptWriteLockParams,
|
|
} from "openclaw/plugin-sdk/session-transcript-runtime";
|
|
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
import type { CodexThread, JsonValue } from "./protocol.js";
|
|
|
|
type MirroredAgentMessage = Extract<AgentMessage, { role: "user" | "assistant" | "toolResult" }>;
|
|
type MirroredUserMessage = Extract<AgentMessage, { role: "user" }>;
|
|
|
|
export type CodexAppServerTranscriptMirrorResult = {
|
|
assistantMirrorIdentitiesOwned: string[];
|
|
userMessagesPresent: MirroredUserMessage[];
|
|
};
|
|
|
|
const MIRROR_IDENTITY_META_KEY = "mirrorIdentity" as const;
|
|
const MIRROR_ORIGIN_META_KEY = "mirrorOrigin" as const;
|
|
const CODEX_APP_SERVER_MIRROR_ORIGIN = "codex-app-server" as const;
|
|
const CODEX_HISTORY_IMPORT_MAX_MESSAGES = 200;
|
|
const CODEX_HISTORY_IMPORT_MAX_BYTES = 512 * 1024;
|
|
const CODEX_HISTORY_IMPORT_MAX_MESSAGE_BYTES = 64 * 1024;
|
|
const CODEX_HISTORY_TRUNCATION_SUFFIX = "\n\n[Message truncated during Codex history import.]";
|
|
const CODEX_HISTORY_ASSISTANT_API = "openai-chatgpt-responses" as const;
|
|
const CODEX_HISTORY_ASSISTANT_PROVIDER = "openai";
|
|
const CODEX_HISTORY_ASSISTANT_MODEL = "native-history";
|
|
const CODEX_HISTORY_ZERO_USAGE: Usage = {
|
|
input: 0,
|
|
output: 0,
|
|
cacheRead: 0,
|
|
cacheWrite: 0,
|
|
totalTokens: 0,
|
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
};
|
|
|
|
export type CodexThreadHistoryImportResult = {
|
|
importedMessages: number;
|
|
omittedMessages: number;
|
|
};
|
|
|
|
export type BoundedCodexThreadHistoryProjection = CodexThreadHistoryImportResult & {
|
|
responseItems: JsonValue[];
|
|
transcriptMessages: AgentMessage[];
|
|
};
|
|
|
|
type ProjectedCodexHistoryMessage = {
|
|
message: AgentMessage;
|
|
responseItem: JsonValue;
|
|
textBytes: number;
|
|
};
|
|
|
|
function isUtf8ContinuationByte(byte: number | undefined): boolean {
|
|
return byte !== undefined && (byte & 0xc0) === 0x80;
|
|
}
|
|
|
|
function truncateUtf8Prefix(value: string, maxBytes: number): string {
|
|
const bytes = Buffer.from(value);
|
|
if (bytes.byteLength <= maxBytes) {
|
|
return value;
|
|
}
|
|
let end = Math.max(0, maxBytes);
|
|
while (end > 0 && isUtf8ContinuationByte(bytes[end])) {
|
|
end -= 1;
|
|
}
|
|
return bytes.subarray(0, end).toString("utf8");
|
|
}
|
|
|
|
function normalizeImportedHistoryText(value: unknown): string | undefined {
|
|
if (typeof value !== "string") {
|
|
return undefined;
|
|
}
|
|
const text = value.trim();
|
|
if (!text) {
|
|
return undefined;
|
|
}
|
|
if (Buffer.byteLength(text, "utf8") <= CODEX_HISTORY_IMPORT_MAX_MESSAGE_BYTES) {
|
|
return text;
|
|
}
|
|
const suffixBytes = Buffer.byteLength(CODEX_HISTORY_TRUNCATION_SUFFIX, "utf8");
|
|
const contentLimitBytes = Math.max(0, CODEX_HISTORY_IMPORT_MAX_MESSAGE_BYTES - suffixBytes);
|
|
return `${truncateUtf8Prefix(text, contentLimitBytes)}${CODEX_HISTORY_TRUNCATION_SUFFIX}`;
|
|
}
|
|
|
|
function projectCodexUserItemText(item: Record<string, unknown>): string | undefined {
|
|
if (!Array.isArray(item.content)) {
|
|
return undefined;
|
|
}
|
|
const parts: string[] = [];
|
|
for (const value of item.content) {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
continue;
|
|
}
|
|
const input = value as Record<string, unknown>;
|
|
if (input.type === "text") {
|
|
const text = normalizeImportedHistoryText(input.text);
|
|
if (text) {
|
|
parts.push(text);
|
|
}
|
|
continue;
|
|
}
|
|
if (input.type === "image" || input.type === "localImage") {
|
|
parts.push("[Image attachment]");
|
|
continue;
|
|
}
|
|
if (input.type === "skill" || input.type === "mention") {
|
|
const name = normalizeOptionalString(input.name);
|
|
if (name) {
|
|
parts.push(`${input.type === "skill" ? "$" : "@"}${name}`);
|
|
}
|
|
}
|
|
}
|
|
return normalizeImportedHistoryText(parts.join("\n"));
|
|
}
|
|
|
|
function selectTurnsThroughBoundary(
|
|
thread: CodexThread,
|
|
throughTurnId: string | null,
|
|
): NonNullable<CodexThread["turns"]> {
|
|
if (throughTurnId === null) {
|
|
return [];
|
|
}
|
|
const turns = thread.turns ?? [];
|
|
const boundaryIndex = turns.findIndex((turn) => turn.id === throughTurnId);
|
|
if (boundaryIndex < 0) {
|
|
throw new Error(`Codex history boundary turn not found: ${throughTurnId}`);
|
|
}
|
|
const boundary = turns[boundaryIndex];
|
|
if (
|
|
boundary?.status !== "completed" &&
|
|
boundary?.status !== "interrupted" &&
|
|
boundary?.status !== "failed"
|
|
) {
|
|
throw new Error(`Codex history boundary turn is not terminal: ${throughTurnId}`);
|
|
}
|
|
return turns.slice(0, boundaryIndex + 1);
|
|
}
|
|
|
|
function projectCodexThreadHistory(params: {
|
|
thread: CodexThread;
|
|
throughTurnId: string | null;
|
|
importedAt: number;
|
|
modelProvider?: string;
|
|
}): ProjectedCodexHistoryMessage[] {
|
|
const projected: ProjectedCodexHistoryMessage[] = [];
|
|
const threadTimestamp =
|
|
typeof params.thread.createdAt === "number" && Number.isFinite(params.thread.createdAt)
|
|
? params.thread.createdAt * 1000
|
|
: params.importedAt;
|
|
let itemOffset = 0;
|
|
for (const turn of selectTurnsThroughBoundary(params.thread, params.throughTurnId)) {
|
|
for (const value of turn.items) {
|
|
const item = value as unknown as Record<string, unknown>;
|
|
const itemId = normalizeOptionalString(item.id);
|
|
const identity = `${turn.id}:${itemId ?? itemOffset}`;
|
|
const timestampSeconds =
|
|
item.type === "agentMessage"
|
|
? (turn.completedAt ?? turn.startedAt)
|
|
: (turn.startedAt ?? turn.completedAt);
|
|
const timestamp =
|
|
typeof timestampSeconds === "number" && Number.isFinite(timestampSeconds)
|
|
? timestampSeconds * 1000 + itemOffset
|
|
: threadTimestamp + itemOffset;
|
|
const text =
|
|
item.type === "userMessage"
|
|
? projectCodexUserItemText(item)
|
|
: item.type === "agentMessage"
|
|
? normalizeImportedHistoryText(item.text)
|
|
: undefined;
|
|
const role =
|
|
item.type === "userMessage"
|
|
? ("user" as const)
|
|
: item.type === "agentMessage"
|
|
? ("assistant" as const)
|
|
: undefined;
|
|
itemOffset += 1;
|
|
if (!text || !role) {
|
|
continue;
|
|
}
|
|
const message =
|
|
role === "assistant"
|
|
? attachCodexMirrorIdentity(
|
|
{
|
|
role,
|
|
content: [{ type: "text", text }],
|
|
api: CODEX_HISTORY_ASSISTANT_API,
|
|
provider:
|
|
normalizeOptionalString(params.modelProvider) ??
|
|
normalizeOptionalString(params.thread.modelProvider) ??
|
|
CODEX_HISTORY_ASSISTANT_PROVIDER,
|
|
model: CODEX_HISTORY_ASSISTANT_MODEL,
|
|
usage: CODEX_HISTORY_ZERO_USAGE,
|
|
stopReason: "stop",
|
|
timestamp,
|
|
} satisfies AssistantMessage,
|
|
identity,
|
|
)
|
|
: attachCodexMirrorIdentity({ role, content: text, timestamp } as AgentMessage, identity);
|
|
const phase =
|
|
item.phase === "commentary" || item.phase === "final_answer" ? item.phase : undefined;
|
|
projected.push({
|
|
message,
|
|
responseItem: {
|
|
type: "message",
|
|
role,
|
|
content: [
|
|
{
|
|
type: role === "assistant" ? "output_text" : "input_text",
|
|
text,
|
|
},
|
|
],
|
|
...(role === "assistant" && phase ? { phase } : {}),
|
|
},
|
|
textBytes: Buffer.byteLength(text, "utf8"),
|
|
});
|
|
}
|
|
}
|
|
return projected;
|
|
}
|
|
|
|
function selectBoundedCodexHistoryTail(
|
|
projected: ProjectedCodexHistoryMessage[],
|
|
): ProjectedCodexHistoryMessage[] {
|
|
const selected: ProjectedCodexHistoryMessage[] = [];
|
|
let selectedBytes = 0;
|
|
for (let index = projected.length - 1; index >= 0; index -= 1) {
|
|
const candidate = projected[index];
|
|
if (!candidate) {
|
|
continue;
|
|
}
|
|
if (
|
|
selected.length >= CODEX_HISTORY_IMPORT_MAX_MESSAGES ||
|
|
selectedBytes + candidate.textBytes > CODEX_HISTORY_IMPORT_MAX_BYTES
|
|
) {
|
|
break;
|
|
}
|
|
selected.push(candidate);
|
|
selectedBytes += candidate.textBytes;
|
|
}
|
|
return selected.toReversed();
|
|
}
|
|
|
|
/** Projects one terminal Codex history prefix into transcript and Responses API items. */
|
|
export function projectBoundedCodexThreadHistory(params: {
|
|
thread: CodexThread;
|
|
throughTurnId: string | null;
|
|
importedAt: number;
|
|
modelProvider?: string | null;
|
|
}): BoundedCodexThreadHistoryProjection {
|
|
const projected = projectCodexThreadHistory({
|
|
thread: params.thread,
|
|
throughTurnId: params.throughTurnId,
|
|
importedAt: params.importedAt,
|
|
...(params.modelProvider ? { modelProvider: params.modelProvider } : {}),
|
|
});
|
|
const selected = selectBoundedCodexHistoryTail(projected);
|
|
return {
|
|
importedMessages: selected.length,
|
|
omittedMessages: projected.length - selected.length,
|
|
responseItems: selected.map(({ responseItem }) => responseItem),
|
|
transcriptMessages: selected.map(({ message }) => message),
|
|
};
|
|
}
|
|
|
|
/** Imports a bounded, user-visible Codex history tail into a new OpenClaw transcript. */
|
|
export async function importCodexThreadHistoryToTranscript(params: {
|
|
thread: CodexThread;
|
|
throughTurnId: string | null;
|
|
storePath: string;
|
|
sessionId: string;
|
|
sessionKey: string;
|
|
agentId?: string;
|
|
cwd?: string;
|
|
modelProvider?: string | null;
|
|
config?: SessionTranscriptWriteLockParams["config"];
|
|
}): Promise<CodexThreadHistoryImportResult> {
|
|
const projection = projectBoundedCodexThreadHistory({
|
|
thread: params.thread,
|
|
throughTurnId: params.throughTurnId,
|
|
importedAt: Date.now(),
|
|
...(params.modelProvider ? { modelProvider: params.modelProvider } : {}),
|
|
});
|
|
if (projection.transcriptMessages.length > 0) {
|
|
await mirrorCodexAppServerTranscript({
|
|
storePath: params.storePath,
|
|
sessionId: params.sessionId,
|
|
sessionKey: params.sessionKey,
|
|
...(params.agentId ? { agentId: params.agentId } : {}),
|
|
...(params.cwd ? { cwd: params.cwd } : {}),
|
|
...(params.config ? { config: params.config } : {}),
|
|
messages: projection.transcriptMessages,
|
|
idempotencyScope: `codex-app-server:${params.thread.id}:history`,
|
|
});
|
|
}
|
|
return {
|
|
importedMessages: projection.importedMessages,
|
|
omittedMessages: projection.omittedMessages,
|
|
};
|
|
}
|
|
|
|
function attachCodexMirrorOrigin(message: AgentMessage): AgentMessage {
|
|
const record = message as unknown as Record<string, unknown>;
|
|
const existing = record["__openclaw"];
|
|
const baseMeta =
|
|
existing && typeof existing === "object" && !Array.isArray(existing)
|
|
? (existing as Record<string, unknown>)
|
|
: {};
|
|
return {
|
|
...record,
|
|
__openclaw: { ...baseMeta, [MIRROR_ORIGIN_META_KEY]: CODEX_APP_SERVER_MIRROR_ORIGIN },
|
|
} as unknown as AgentMessage;
|
|
}
|
|
|
|
function buildSenderLabel(params: {
|
|
senderId?: string;
|
|
senderName?: string;
|
|
senderUsername?: string;
|
|
senderE164?: string;
|
|
}): string | undefined {
|
|
const label = params.senderName ?? params.senderUsername ?? params.senderE164 ?? params.senderId;
|
|
if (!label) {
|
|
return undefined;
|
|
}
|
|
if (!params.senderId || label.includes(params.senderId)) {
|
|
return label;
|
|
}
|
|
return `${label} (${params.senderId})`;
|
|
}
|
|
|
|
function buildCodexUserPromptMessageFromPrepared(
|
|
params: EmbeddedRunAttemptParams,
|
|
preparedUserMessage: MirroredUserMessage | undefined,
|
|
): AgentMessage {
|
|
const senderId = normalizeOptionalString(params.senderId);
|
|
const senderName = normalizeOptionalString(params.senderName);
|
|
const senderUsername = normalizeOptionalString(params.senderUsername);
|
|
const senderE164 = normalizeOptionalString(params.senderE164);
|
|
const senderLabel = buildSenderLabel({ senderId, senderName, senderUsername, senderE164 });
|
|
const sourceChannel = normalizeOptionalString(
|
|
params.inputProvenance?.sourceChannel ?? params.messageChannel ?? params.messageProvider,
|
|
);
|
|
if (preparedUserMessage) {
|
|
return {
|
|
role: "user",
|
|
timestamp: Date.now(),
|
|
...(params.inputProvenance ? { provenance: params.inputProvenance } : {}),
|
|
...(sourceChannel ? { sourceChannel } : {}),
|
|
...(senderId ? { senderId } : {}),
|
|
...(senderName ? { senderName } : {}),
|
|
...(senderUsername ? { senderUsername } : {}),
|
|
...(senderE164 ? { senderE164 } : {}),
|
|
...(senderLabel ? { senderLabel } : {}),
|
|
...(preparedUserMessage as unknown as Record<string, unknown>),
|
|
} as AgentMessage;
|
|
}
|
|
return {
|
|
role: "user",
|
|
content: params.prompt,
|
|
timestamp: Date.now(),
|
|
...(params.inputProvenance ? { provenance: params.inputProvenance } : {}),
|
|
...(sourceChannel ? { sourceChannel } : {}),
|
|
...(senderId ? { senderId } : {}),
|
|
...(senderName ? { senderName } : {}),
|
|
...(senderUsername ? { senderUsername } : {}),
|
|
...(senderE164 ? { senderE164 } : {}),
|
|
...(senderLabel ? { senderLabel } : {}),
|
|
} as AgentMessage;
|
|
}
|
|
|
|
export function buildCodexUserPromptMessage(params: EmbeddedRunAttemptParams): AgentMessage {
|
|
return buildCodexUserPromptMessageFromPrepared(
|
|
params,
|
|
params.userTurnTranscriptRecorder?.message,
|
|
);
|
|
}
|
|
|
|
export async function buildResolvedCodexUserPromptMessage(
|
|
params: EmbeddedRunAttemptParams,
|
|
): Promise<AgentMessage> {
|
|
const resolvedMessage = await params.userTurnTranscriptRecorder?.resolveMessage();
|
|
return buildCodexUserPromptMessageFromPrepared(
|
|
params,
|
|
resolvedMessage ?? params.userTurnTranscriptRecorder?.message,
|
|
);
|
|
}
|
|
|
|
export async function mirrorTranscriptBestEffort(params: {
|
|
params: EmbeddedRunAttemptParams;
|
|
agentId?: string;
|
|
notifyUserMessagePersisted: (message: Extract<AgentMessage, { role: "user" }>) => void;
|
|
result: EmbeddedRunAttemptResult;
|
|
sessionKey?: string;
|
|
cwd: string;
|
|
threadId: string;
|
|
turnId: string;
|
|
}): Promise<boolean> {
|
|
try {
|
|
const messages = await resolveFinalCodexMirrorMessages({
|
|
params: params.params,
|
|
messagesSnapshot: params.result.messagesSnapshot,
|
|
turnId: params.turnId,
|
|
});
|
|
const mirrorResult = await mirrorCodexAppServerTranscript({
|
|
agentId: params.agentId,
|
|
sessionKey: params.sessionKey,
|
|
sessionId: params.params.sessionId,
|
|
storePath: params.params.sessionTarget?.storePath,
|
|
cwd: params.cwd,
|
|
messages,
|
|
// Scope is thread-stable. Each entry in `messagesSnapshot` is tagged
|
|
// with a per-turn `attachCodexMirrorIdentity` value carrying its own
|
|
// turnId, so distinct turns produce distinct dedupe keys via the
|
|
// identity (not via the scope). Dropping `turnId` from the scope here is
|
|
// what lets a re-emitted prior-turn entry collide with its existing key.
|
|
idempotencyScope: `codex-app-server:${params.threadId}`,
|
|
config: params.params.config,
|
|
});
|
|
for (const message of mirrorResult.userMessagesPresent) {
|
|
try {
|
|
params.notifyUserMessagePersisted(message);
|
|
} catch (error) {
|
|
embeddedAgentLog.warn("failed to notify codex app-server user-message persistence", {
|
|
error: formatErrorMessage(error),
|
|
});
|
|
}
|
|
}
|
|
return mirrorResult.assistantMirrorIdentitiesOwned.includes(`${params.turnId}:assistant`);
|
|
} catch (error) {
|
|
embeddedAgentLog.warn("failed to mirror codex app-server transcript", { error });
|
|
return false;
|
|
}
|
|
}
|
|
|
|
export async function resolveFinalCodexMirrorMessages(params: {
|
|
params: EmbeddedRunAttemptParams;
|
|
messagesSnapshot: AgentMessage[];
|
|
turnId: string;
|
|
}): Promise<AgentMessage[]> {
|
|
if (
|
|
params.params.suppressNextUserMessagePersistence ||
|
|
!params.params.userTurnTranscriptRecorder
|
|
) {
|
|
return params.messagesSnapshot;
|
|
}
|
|
const resolvedPrompt = attachCodexMirrorIdentity(
|
|
await buildResolvedCodexUserPromptMessage(params.params),
|
|
`${params.turnId}:prompt`,
|
|
);
|
|
const firstUserIndex = params.messagesSnapshot.findIndex((message) => message.role === "user");
|
|
if (firstUserIndex === -1) {
|
|
return [resolvedPrompt, ...params.messagesSnapshot];
|
|
}
|
|
const messages = params.messagesSnapshot.slice();
|
|
messages[firstUserIndex] = resolvedPrompt;
|
|
return messages;
|
|
}
|
|
|
|
export function createCodexAppServerUserMessagePersistenceNotifier(
|
|
runParams: EmbeddedRunAttemptParams,
|
|
): (message: Extract<AgentMessage, { role: "user" }>) => void {
|
|
let notified = false;
|
|
return (message) => {
|
|
if (notified) {
|
|
return;
|
|
}
|
|
notified = true;
|
|
runParams.userTurnTranscriptRecorder?.markRuntimePersisted(message);
|
|
try {
|
|
runParams.onUserMessagePersisted?.(message);
|
|
} catch (error) {
|
|
embeddedAgentLog.warn("codex app-server user persistence notification failed", {
|
|
error: formatErrorMessage(error),
|
|
});
|
|
}
|
|
};
|
|
}
|
|
|
|
export async function mirrorPromptAtTurnStartBestEffort(params: {
|
|
params: EmbeddedRunAttemptParams;
|
|
agentId?: string;
|
|
notifyUserMessagePersisted: (message: Extract<AgentMessage, { role: "user" }>) => void;
|
|
sessionKey?: string;
|
|
cwd: string;
|
|
threadId: string;
|
|
turnId: string;
|
|
}): Promise<void> {
|
|
if (params.params.suppressNextUserMessagePersistence) {
|
|
return;
|
|
}
|
|
try {
|
|
const mirrorPromise = (async () => {
|
|
const userPromptMessage = attachCodexMirrorIdentity(
|
|
await buildResolvedCodexUserPromptMessage(params.params),
|
|
`${params.turnId}:prompt`,
|
|
);
|
|
const mirrorResult = await mirrorCodexAppServerTranscript({
|
|
agentId: params.agentId,
|
|
sessionKey: params.sessionKey,
|
|
sessionId: params.params.sessionId,
|
|
storePath: params.params.sessionTarget?.storePath,
|
|
cwd: params.cwd,
|
|
messages: [userPromptMessage],
|
|
idempotencyScope: `codex-app-server:${params.threadId}`,
|
|
config: params.params.config,
|
|
});
|
|
for (const message of mirrorResult.userMessagesPresent) {
|
|
params.notifyUserMessagePersisted(message);
|
|
}
|
|
})();
|
|
params.params.userTurnTranscriptRecorder?.markRuntimePersistencePending(mirrorPromise);
|
|
await mirrorPromise;
|
|
} catch (error) {
|
|
embeddedAgentLog.warn("failed to mirror codex app-server prompt at turn start", { error });
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Tag a message with a stable logical identity for mirror dedupe. Callers
|
|
* should use a value that is invariant for the same logical message across
|
|
* re-emits (e.g. `${turnId}:prompt`, `${turnId}:assistant`) but distinct
|
|
* for genuinely-distinct messages (different turns, different kinds). When
|
|
* present this identity replaces the role/content fingerprint in the
|
|
* idempotency key, so the dedupe survives caller-scope rotation without
|
|
* collapsing distinct same-content turns.
|
|
*/
|
|
export function attachCodexMirrorIdentity<T extends AgentMessage>(message: T, identity: string): T {
|
|
const record = message as unknown as Record<string, unknown>;
|
|
const existing = record["__openclaw"];
|
|
const baseMeta =
|
|
existing && typeof existing === "object" && !Array.isArray(existing)
|
|
? (existing as Record<string, unknown>)
|
|
: {};
|
|
return {
|
|
...record,
|
|
__openclaw: { ...baseMeta, [MIRROR_IDENTITY_META_KEY]: identity },
|
|
} as unknown as T;
|
|
}
|
|
|
|
function readMirrorIdentity(message: MirroredAgentMessage): string | undefined {
|
|
const record = message as unknown as { __openclaw?: unknown };
|
|
const meta = record["__openclaw"];
|
|
if (!meta || typeof meta !== "object" || Array.isArray(meta)) {
|
|
return undefined;
|
|
}
|
|
const id = (meta as Record<string, unknown>)[MIRROR_IDENTITY_META_KEY];
|
|
return typeof id === "string" && id.length > 0 ? id : undefined;
|
|
}
|
|
|
|
// Fallback content fingerprint for callers that did not tag the message
|
|
// with a stable mirror identity. Only role and content participate; volatile
|
|
// metadata (timestamps, usage, etc.) is intentionally excluded so the
|
|
// fingerprint survives snapshot reordering inside a fixed scope. Distinct
|
|
// same-content turns are still distinguished by the caller's idempotency
|
|
// scope when callers route through this fallback.
|
|
function fingerprintMirrorMessageContent(message: MirroredAgentMessage): string {
|
|
const payload = JSON.stringify({ role: message.role, content: message.content });
|
|
return createHash("sha256").update(payload).digest("hex").slice(0, 16);
|
|
}
|
|
|
|
function buildMirrorDedupeIdentity(message: MirroredAgentMessage): string {
|
|
const explicit = readMirrorIdentity(message);
|
|
if (explicit) {
|
|
return explicit;
|
|
}
|
|
return `${message.role}:${fingerprintMirrorMessageContent(message)}`;
|
|
}
|
|
|
|
export async function mirrorCodexAppServerTranscript(params: {
|
|
sessionId: string;
|
|
cwd?: string;
|
|
sessionKey?: string;
|
|
agentId?: string;
|
|
storePath?: string;
|
|
messages: AgentMessage[];
|
|
idempotencyScope?: string;
|
|
config?: SessionTranscriptWriteLockParams["config"];
|
|
}): Promise<CodexAppServerTranscriptMirrorResult> {
|
|
const messages = params.messages.filter(
|
|
(message): message is MirroredAgentMessage =>
|
|
message.role === "user" || message.role === "assistant" || message.role === "toolResult",
|
|
);
|
|
if (messages.length === 0) {
|
|
return { assistantMirrorIdentitiesOwned: [], userMessagesPresent: [] };
|
|
}
|
|
|
|
const transcriptTarget = resolveCodexMirrorTranscriptTarget(params);
|
|
const mirrorBatch = await withSessionTranscriptWriteLock(
|
|
{ ...transcriptTarget, config: params.config },
|
|
async (transcript) => {
|
|
const nextAppendedUpdates: Array<{
|
|
messageId: string;
|
|
message: AgentMessage;
|
|
messageSeq: number;
|
|
}> = [];
|
|
const nextAssistantMirrorIdentitiesOwned = new Set<string>();
|
|
const nextUserMessagesPresent: MirroredUserMessage[] = [];
|
|
const mirrorState = readTranscriptMirrorState(await transcript.readEvents());
|
|
let nextMessageSeq = mirrorState.messageCount;
|
|
for (const message of messages) {
|
|
const dedupeIdentity = buildMirrorDedupeIdentity(message);
|
|
const sourceUserIdempotencyKey =
|
|
message.role === "user"
|
|
? normalizeOptionalString(
|
|
(message as unknown as { idempotencyKey?: unknown }).idempotencyKey,
|
|
)
|
|
: undefined;
|
|
// The gateway owns user-turn identity. Preserve its key so clients can
|
|
// correlate optimistic rows; provider mirror identity is only a fallback.
|
|
const idempotencyKey =
|
|
sourceUserIdempotencyKey ??
|
|
(params.idempotencyScope ? `${params.idempotencyScope}:${dedupeIdentity}` : undefined);
|
|
const transcriptMessage = {
|
|
...(attachCodexMirrorOrigin(message) as unknown as Record<string, unknown>),
|
|
...(idempotencyKey ? { idempotencyKey } : {}),
|
|
} as AgentMessage;
|
|
if (idempotencyKey && mirrorState.idempotencyKeys.has(idempotencyKey)) {
|
|
const persistedUserMessage = mirrorState.userMessagesByIdempotencyKey.get(idempotencyKey);
|
|
if (persistedUserMessage) {
|
|
nextUserMessagesPresent.push(persistedUserMessage);
|
|
}
|
|
if (message.role === "assistant") {
|
|
nextAssistantMirrorIdentitiesOwned.add(dedupeIdentity);
|
|
}
|
|
continue;
|
|
}
|
|
const nextMessage = runAgentHarnessBeforeMessageWriteHook({
|
|
message: transcriptMessage,
|
|
agentId: params.agentId,
|
|
sessionKey: params.sessionKey,
|
|
});
|
|
if (!nextMessage) {
|
|
if (message.role === "assistant") {
|
|
// A transcript hook deliberately blocked this logical assistant row.
|
|
// Treat that as an authoritative persistence decision so delivery
|
|
// does not bypass the hook with a fallback mirror.
|
|
nextAssistantMirrorIdentitiesOwned.add(dedupeIdentity);
|
|
}
|
|
continue;
|
|
}
|
|
const messageToAppend = (
|
|
idempotencyKey
|
|
? {
|
|
...(attachCodexMirrorOrigin(nextMessage) as unknown as Record<string, unknown>),
|
|
idempotencyKey,
|
|
}
|
|
: attachCodexMirrorOrigin(nextMessage)
|
|
) as AgentMessage;
|
|
const appended = await transcript.appendMessage({
|
|
message: messageToAppend,
|
|
idempotencyLookup: idempotencyKey ? "caller-checked" : "scan",
|
|
cwd: params.cwd,
|
|
});
|
|
if (!appended) {
|
|
continue;
|
|
}
|
|
const { messageId, message: appendedMessage } = appended;
|
|
if (message.role === "assistant") {
|
|
nextAssistantMirrorIdentitiesOwned.add(dedupeIdentity);
|
|
}
|
|
if (appendedMessage.role === "user") {
|
|
nextUserMessagesPresent.push(appendedMessage);
|
|
if (idempotencyKey) {
|
|
mirrorState.userMessagesByIdempotencyKey.set(idempotencyKey, appendedMessage);
|
|
}
|
|
}
|
|
nextMessageSeq += 1;
|
|
nextAppendedUpdates.push({
|
|
messageId,
|
|
message: appendedMessage,
|
|
messageSeq: nextMessageSeq,
|
|
});
|
|
if (idempotencyKey) {
|
|
mirrorState.idempotencyKeys.add(idempotencyKey);
|
|
}
|
|
}
|
|
return {
|
|
appendedUpdates: nextAppendedUpdates,
|
|
assistantMirrorIdentitiesOwned: [...nextAssistantMirrorIdentitiesOwned],
|
|
userMessagesPresent: nextUserMessagesPresent,
|
|
};
|
|
},
|
|
);
|
|
const { appendedUpdates, assistantMirrorIdentitiesOwned, userMessagesPresent } = mirrorBatch;
|
|
|
|
for (const update of appendedUpdates) {
|
|
try {
|
|
await publishSessionTranscriptUpdateByIdentity({
|
|
...transcriptTarget,
|
|
update: {
|
|
...(params.agentId ? { agentId: params.agentId } : {}),
|
|
message: update.message,
|
|
messageId: update.messageId,
|
|
messageSeq: update.messageSeq,
|
|
sessionKey: transcriptTarget.sessionKey,
|
|
},
|
|
});
|
|
} catch (error) {
|
|
// The transcript append is already committed. A transient live-update
|
|
// failure must not make dispatch append a second assistant message.
|
|
embeddedAgentLog.warn("failed to publish codex app-server transcript update", {
|
|
error: formatErrorMessage(error),
|
|
});
|
|
}
|
|
}
|
|
|
|
return { assistantMirrorIdentitiesOwned, userMessagesPresent };
|
|
}
|
|
|
|
function resolveCodexMirrorTranscriptTarget(params: {
|
|
agentId?: string;
|
|
sessionId: string;
|
|
sessionKey?: string;
|
|
storePath?: string;
|
|
}): SessionTranscriptTargetParams {
|
|
const sessionKey = params.sessionKey?.trim();
|
|
const storePath = params.storePath?.trim();
|
|
if (!sessionKey || !storePath) {
|
|
throw new Error("Codex transcript mirror requires a runtime session identity");
|
|
}
|
|
return {
|
|
...(params.agentId ? { agentId: params.agentId } : {}),
|
|
sessionId: params.sessionId,
|
|
sessionKey,
|
|
storePath,
|
|
};
|
|
}
|
|
|
|
function readTranscriptMirrorState(events: unknown[]): {
|
|
idempotencyKeys: Set<string>;
|
|
messageCount: number;
|
|
userMessagesByIdempotencyKey: Map<string, MirroredUserMessage>;
|
|
} {
|
|
const idempotencyKeys = new Set<string>();
|
|
const userMessagesByIdempotencyKey = new Map<string, MirroredUserMessage>();
|
|
let messageCount = 0;
|
|
for (const event of events) {
|
|
if (!event || typeof event !== "object" || Array.isArray(event)) {
|
|
continue;
|
|
}
|
|
const parsed = event as {
|
|
message?: AgentMessage & { idempotencyKey?: unknown };
|
|
type?: unknown;
|
|
};
|
|
if (parsed.type === "message") {
|
|
messageCount += 1;
|
|
}
|
|
if (typeof parsed.message?.idempotencyKey === "string") {
|
|
idempotencyKeys.add(parsed.message.idempotencyKey);
|
|
if (parsed.message.role === "user") {
|
|
userMessagesByIdempotencyKey.set(parsed.message.idempotencyKey, parsed.message);
|
|
}
|
|
}
|
|
}
|
|
return {
|
|
idempotencyKeys,
|
|
messageCount,
|
|
userMessagesByIdempotencyKey,
|
|
};
|
|
}
|