mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-25 20:05:46 -06:00
refactor(transcripts): store meeting captures in SQLite (#112910)
* refactor(transcripts): move meeting transcripts to sqlite * perf(transcripts): batch legacy utterance staging * fix(transcripts): report recovery moves on migration failure * fix(transcripts): refresh archive membership after recovery * fix(transcripts): omit failed summary export paths * fix(transcripts): type restore metadata tuple * fix(transcripts): align migration contract gates * fix(transcripts): verify case-aliased export ownership * fix(transcripts): allowlist doctor verifier sqlite query * fix(transcripts): preflight partial artifact recovery * fix(transcripts): stabilize exports and legacy path checks * fix(transcripts): resolve case-renamed doctor ownership * fix(transcripts): harden canonical export recovery * fix(transcripts): preserve pending import boundaries * refactor(transcripts): split migration insert transaction * perf(transcripts): query selected summary existence
This commit is contained in:
committed by
GitHub
parent
d124fb235f
commit
a2be4efb63
+42
-21
@@ -1,5 +1,5 @@
|
||||
---
|
||||
summary: "CLI reference for `openclaw transcripts` (list, show, and locate stored transcripts)"
|
||||
summary: "CLI reference for `openclaw transcripts` (list, show, and export stored transcripts)"
|
||||
read_when:
|
||||
- You want to read stored transcript summaries from the terminal
|
||||
- You need the path to a transcripts markdown summary
|
||||
@@ -9,10 +9,12 @@ title: "Transcripts CLI"
|
||||
|
||||
# `openclaw transcripts`
|
||||
|
||||
Read-only inspector for transcripts written by the `transcripts` agent tool.
|
||||
Capture, import, and summarization run through that tool, not this CLI.
|
||||
Inspector and export command for transcripts written by the `transcripts` agent
|
||||
tool. Capture, import, and summarization run through that tool, not this CLI.
|
||||
|
||||
Artifacts live under the state directory:
|
||||
Canonical transcript state lives in the shared SQLite database at
|
||||
`$OPENCLAW_STATE_DIR/state/openclaw.sqlite`. `show` and `path` explicitly
|
||||
materialize user-facing artifacts under the state directory:
|
||||
|
||||
```text
|
||||
$OPENCLAW_STATE_DIR/transcripts/YYYY-MM-DD/<session>/
|
||||
@@ -22,9 +24,11 @@ $OPENCLAW_STATE_DIR/transcripts/YYYY-MM-DD/<session>/
|
||||
summary.md
|
||||
```
|
||||
|
||||
Default state directory is `~/.openclaw`; override with `OPENCLAW_STATE_DIR`.
|
||||
The date directory comes from the session start time; the session directory is
|
||||
a filesystem-safe slug derived from the session id.
|
||||
These files are exports, not a second runtime store. OpenClaw does not read them
|
||||
back during capture, summarization, or listing. Default state directory is
|
||||
`~/.openclaw`; override with `OPENCLAW_STATE_DIR`. The date directory comes
|
||||
from the session start time; the session directory is a filesystem-safe slug
|
||||
derived from the session id.
|
||||
|
||||
## Commands
|
||||
|
||||
@@ -42,15 +46,15 @@ openclaw transcripts show <session> --json
|
||||
openclaw transcripts path <session> --json
|
||||
```
|
||||
|
||||
| Command | Description |
|
||||
| ----------------------------- | ----------------------------------------------- |
|
||||
| `list` | List stored sessions. |
|
||||
| `show <session>` | Print the stored `summary.md`. |
|
||||
| `path <session>` | Print the `summary.md` path. |
|
||||
| `path <session> --dir` | Print the session directory. |
|
||||
| `path <session> --metadata` | Print `metadata.json`. |
|
||||
| `path <session> --transcript` | Print `transcript.jsonl`. |
|
||||
| `--json` | Print machine-readable output (any subcommand). |
|
||||
| Command | Description |
|
||||
| ----------------------------- | ---------------------------------------------------- |
|
||||
| `list` | List stored sessions. |
|
||||
| `show <session>` | Print and materialize `summary.md`. |
|
||||
| `path <session>` | Materialize and print the `summary.md` path. |
|
||||
| `path <session> --dir` | Materialize all artifacts and print their directory. |
|
||||
| `path <session> --metadata` | Materialize and print `metadata.json`. |
|
||||
| `path <session> --transcript` | Materialize and print `transcript.jsonl`. |
|
||||
| `--json` | Print machine-readable output (any subcommand). |
|
||||
|
||||
`<session>` accepts either a bare session id or a date-qualified selector
|
||||
(`YYYY-MM-DD/<session>`). Use the qualified form when the same session id
|
||||
@@ -75,7 +79,9 @@ The selector is the safest value to pass back to `show` or `path`.
|
||||
`show --json` returns the stored session metadata, selector, session
|
||||
directory, summary path, and summary Markdown text.
|
||||
|
||||
`path --json` returns the selected path and whether that file exists.
|
||||
`path --json` returns the selected path and whether that artifact could be
|
||||
materialized. Metadata and transcript exports always exist for a stored
|
||||
session; a summary path reports `exists: false` until the session has a summary.
|
||||
|
||||
## Many sessions per day
|
||||
|
||||
@@ -94,15 +100,30 @@ when it will not repeat on the same date.
|
||||
|
||||
## Missing summaries
|
||||
|
||||
Live sessions write `summary.md` when the session stops; imported transcripts
|
||||
write it immediately after import. A session can appear in `list` without a
|
||||
summary while capture is still active, if a provider failed during stop, or if
|
||||
metadata was written before any utterances arrived.
|
||||
Live sessions store and materialize `summary.md` when the session stops;
|
||||
imported transcripts do so immediately after import. A session can appear in
|
||||
`list` without a summary while capture is still active, if a provider failed
|
||||
during stop, or if metadata was stored before any utterances arrived.
|
||||
|
||||
Use `path <session> --transcript` to inspect the raw append-only transcript,
|
||||
or run the `transcripts` tool's `summarize` action to regenerate the Markdown
|
||||
summary.
|
||||
|
||||
## Upgrading the legacy file store
|
||||
|
||||
OpenClaw releases that predate the SQLite store wrote canonical runtime state
|
||||
directly beneath `$OPENCLAW_STATE_DIR/transcripts/`. Run:
|
||||
|
||||
```bash
|
||||
openclaw doctor --fix
|
||||
```
|
||||
|
||||
Doctor imports the complete legacy tree into SQLite, verifies row counts and
|
||||
ordering, records migration receipts, and moves the verified source tree to a
|
||||
timestamped `transcripts.migrated-*` archive. Runtime commands do not fall back
|
||||
to the legacy files. Keep the archive until you have verified the imported
|
||||
sessions and any exports you rely on.
|
||||
|
||||
## Configuration
|
||||
|
||||
Capture is opt-in (live sources can join and record meeting audio). Enable it
|
||||
|
||||
@@ -2110,6 +2110,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
|
||||
- H2: Output
|
||||
- H2: Many sessions per day
|
||||
- H2: Missing summaries
|
||||
- H2: Upgrading the legacy file store
|
||||
- H2: Configuration
|
||||
|
||||
## cli/tui.md
|
||||
|
||||
@@ -54,6 +54,7 @@ const rawSqliteAllowPathGroups = {
|
||||
"src/state/openclaw-state-db-schema-repair.ts",
|
||||
"src/state/openclaw-state-db-startup-checkpoint.ts",
|
||||
"src/state/openclaw-state-db.ts",
|
||||
"src/transcripts/sqlite-schema.ts",
|
||||
"src/state/sqlite-schema-shape.test-support.ts",
|
||||
],
|
||||
"cross-process SQLite coordination locks": ["src/infra/device-identity-coordinator.ts"],
|
||||
@@ -90,6 +91,9 @@ const rawSqliteAllowPathGroups = {
|
||||
"src/infra/state-migrations.storage.ts",
|
||||
"src/infra/state-migrations.cron-run-logs.ts",
|
||||
"src/infra/state-migrations.debug-proxy.ts",
|
||||
"src/infra/state-migrations.meeting-transcripts-detection.ts",
|
||||
"src/infra/state-migrations.meeting-transcripts-files.ts",
|
||||
"src/infra/state-migrations.meeting-transcripts-verify.ts",
|
||||
],
|
||||
"shared database stores with direct DatabaseSync access": ["src/proxy-capture/store.sqlite.ts"],
|
||||
"Kysely-backed stores that own a DatabaseSync boundary": [
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
// Assertions for upgrade-survivor E2E scenarios.
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
@@ -16,6 +17,7 @@ const SCENARIOS = new Set([
|
||||
"configured-plugin-installs",
|
||||
"stale-source-plugin-shadow",
|
||||
"tilde-log-path",
|
||||
"meeting-transcripts-sqlite",
|
||||
"versioned-runtime-deps",
|
||||
]);
|
||||
|
||||
@@ -138,6 +140,50 @@ function seedLegacySessionMetadata(stateDir) {
|
||||
}
|
||||
}
|
||||
|
||||
function seedLegacyMeetingTranscripts(stateDir) {
|
||||
const sessionDir = path.join(stateDir, "transcripts", "2026-07-01", "design-review");
|
||||
const session = {
|
||||
sessionId: "design-review",
|
||||
title: "Design review",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-07-01T10:00:00.000Z",
|
||||
stoppedAt: "2026-07-01T10:30:00.000Z",
|
||||
};
|
||||
writeJson(path.join(sessionDir, "metadata.json"), session);
|
||||
write(
|
||||
path.join(sessionDir, "transcript.jsonl"),
|
||||
[
|
||||
JSON.stringify({
|
||||
id: "legacy-u-1",
|
||||
sessionId: session.sessionId,
|
||||
speaker: { label: "Alex" },
|
||||
text: "First shipped transcript line",
|
||||
final: true,
|
||||
}),
|
||||
JSON.stringify({
|
||||
id: "legacy-u-2",
|
||||
sessionId: session.sessionId,
|
||||
speaker: { label: "Sam" },
|
||||
text: "Second shipped transcript line",
|
||||
final: true,
|
||||
}),
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
const summary = {
|
||||
sessionId: session.sessionId,
|
||||
title: session.title,
|
||||
generatedAt: "2026-07-01T10:31:00.000Z",
|
||||
overview: "First shipped transcript line. Second shipped transcript line.",
|
||||
transcript: ["Alex: First shipped transcript line", "Sam: Second shipped transcript line"],
|
||||
decisions: [],
|
||||
actionItems: [],
|
||||
risks: [],
|
||||
utteranceCount: 2,
|
||||
};
|
||||
writeJson(path.join(sessionDir, "summary.json"), summary);
|
||||
write(path.join(sessionDir, "summary.md"), "# Design review\n\nShipped transcript summary.\n");
|
||||
}
|
||||
|
||||
function getScenario() {
|
||||
const scenario = process.env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIO || "base";
|
||||
assert(SCENARIOS.has(scenario), `unknown upgrade survivor scenario: ${scenario}`);
|
||||
@@ -195,6 +241,9 @@ function seedState() {
|
||||
title: "Existing user session",
|
||||
});
|
||||
seedLegacySessionMetadata(stateDir);
|
||||
if (scenario === "meeting-transcripts-sqlite") {
|
||||
seedLegacyMeetingTranscripts(stateDir);
|
||||
}
|
||||
|
||||
const runtimeRoot = path.join(stateDir, "plugin-runtime-deps");
|
||||
for (const plugin of ["discord", "telegram", "whatsapp"]) {
|
||||
@@ -257,6 +306,11 @@ function seedState() {
|
||||
function assertConfigSurvived() {
|
||||
const config = getConfig();
|
||||
const coverage = getCoverage();
|
||||
if (getScenario() === "meeting-transcripts-sqlite") {
|
||||
// This focused migration fixture proves state import/export across one published
|
||||
// baseline; the broad base scenario owns unrelated agent/channel config parity.
|
||||
return;
|
||||
}
|
||||
|
||||
if (acceptsIntent(coverage, "update")) {
|
||||
assert(config.update?.channel === "stable", "update.channel was not preserved");
|
||||
@@ -435,6 +489,9 @@ function assertStateSurvived() {
|
||||
if (stage !== "baseline") {
|
||||
assertSessionMetadataMigrated(stateDir);
|
||||
}
|
||||
if (scenario === "meeting-transcripts-sqlite") {
|
||||
assertMeetingTranscriptsMigrated(stateDir, stage);
|
||||
}
|
||||
const legacyRuntimeRoot = path.join(stateDir, "plugin-runtime-deps");
|
||||
if (stage === "baseline") {
|
||||
if (fs.existsSync(legacyRuntimeRoot)) {
|
||||
@@ -478,6 +535,92 @@ function assertStateSurvived() {
|
||||
}
|
||||
}
|
||||
|
||||
function assertMeetingTranscriptsMigrated(stateDir, stage) {
|
||||
const legacySessionDir = path.join(stateDir, "transcripts", "2026-07-01", "design-review");
|
||||
if (stage === "baseline") {
|
||||
assert(
|
||||
fs.existsSync(path.join(legacySessionDir, "transcript.jsonl")),
|
||||
"v2026.7.1 meeting transcript fixture missing before update",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
assert(!fs.existsSync(legacySessionDir), "legacy meeting transcript source was not archived");
|
||||
const archiveRoot = fs
|
||||
.readdirSync(stateDir)
|
||||
.find((entry) => entry.startsWith("transcripts.migrated-"));
|
||||
assert(archiveRoot, "meeting transcript migration archive missing");
|
||||
assert(
|
||||
fs.existsSync(
|
||||
path.join(stateDir, archiveRoot, "2026-07-01", "design-review", "transcript.jsonl"),
|
||||
),
|
||||
"archived meeting transcript JSONL missing",
|
||||
);
|
||||
|
||||
const databasePath = path.join(stateDir, "state", "openclaw.sqlite");
|
||||
const db = new DatabaseSync(databasePath, { readOnly: true });
|
||||
try {
|
||||
const session = db
|
||||
.prepare(
|
||||
"SELECT session_id, started_at, next_utterance_seq FROM meeting_transcript_sessions WHERE session_id = ?",
|
||||
)
|
||||
.get("design-review");
|
||||
assert(session?.started_at === "2026-07-01T10:00:00.000Z", "meeting session row missing");
|
||||
assert(session?.next_utterance_seq === 2, "meeting transcript sequence head changed");
|
||||
const utterances = db
|
||||
.prepare(
|
||||
"SELECT sequence, utterance_id, text FROM meeting_transcript_utterances WHERE session_id = ? ORDER BY sequence ASC",
|
||||
)
|
||||
.all("design-review");
|
||||
assert(
|
||||
JSON.stringify(utterances) ===
|
||||
JSON.stringify([
|
||||
{
|
||||
sequence: 0,
|
||||
utterance_id: "legacy-u-1",
|
||||
text: "First shipped transcript line",
|
||||
},
|
||||
{
|
||||
sequence: 1,
|
||||
utterance_id: "legacy-u-2",
|
||||
text: "Second shipped transcript line",
|
||||
},
|
||||
]),
|
||||
"meeting transcript utterance ordering changed",
|
||||
);
|
||||
const receipt = db
|
||||
.prepare(
|
||||
"SELECT status, removed_source, source_record_count FROM migration_sources WHERE migration_kind = ?",
|
||||
)
|
||||
.get("meeting-transcripts-files-v1");
|
||||
assert(receipt?.status === "archived", "meeting transcript migration receipt incomplete");
|
||||
assert(receipt?.removed_source === 1, "meeting transcript source removal was not recorded");
|
||||
assert(receipt?.source_record_count === 2, "meeting transcript receipt count changed");
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
|
||||
const exportedDir = execFileSync(
|
||||
"openclaw",
|
||||
["transcripts", "path", "2026-07-01/design-review", "--dir"],
|
||||
{ encoding: "utf8", env: process.env },
|
||||
).trim();
|
||||
assert(exportedDir === legacySessionDir, "meeting transcript export path changed");
|
||||
const exportedLines = fs
|
||||
.readFileSync(path.join(exportedDir, "transcript.jsonl"), "utf8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
assert(exportedLines[0]?.id === "legacy-u-1", "first exported utterance changed");
|
||||
assert(exportedLines[1]?.id === "legacy-u-2", "second exported utterance changed");
|
||||
assert(
|
||||
fs
|
||||
.readFileSync(path.join(exportedDir, "summary.md"), "utf8")
|
||||
.includes("Shipped transcript summary"),
|
||||
"summary.md was not materialized from SQLite",
|
||||
);
|
||||
}
|
||||
|
||||
function assertSessionMetadataMigrated(stateDir) {
|
||||
const legacyStorePath = path.join(stateDir, "sessions", "sessions.json");
|
||||
const agentSessionsDir = path.join(stateDir, "agents", "main", "sessions");
|
||||
|
||||
@@ -86,6 +86,7 @@ const UPGRADE_SURVIVOR_SCENARIOS = [
|
||||
"configured-plugin-installs",
|
||||
"stale-source-plugin-shadow",
|
||||
"tilde-log-path",
|
||||
"meeting-transcripts-sqlite",
|
||||
"versioned-runtime-deps",
|
||||
];
|
||||
|
||||
@@ -97,6 +98,10 @@ const UPGRADE_SURVIVOR_SCENARIO_ALIASES = new Map([
|
||||
// Pre-protocol catalogs are content-addressed. Unknown legacy blocks fail
|
||||
// closed instead of requiring a dependency or reimplementing a JavaScript parser.
|
||||
const LEGACY_UPGRADE_SURVIVOR_SCENARIO_CATALOGS = new Map([
|
||||
[
|
||||
"213e004a28814fe0f7bb33018ae59a709c2e6d6e13b273df82a0e7935fbaf5af",
|
||||
"base acpx-openclaw-tools-bridge feishu-channel bootstrap-persona channel-post-core-restore codex-allowlist-survival plugin-deps-cleanup configured-plugin-installs stale-source-plugin-shadow tilde-log-path meeting-transcripts-sqlite versioned-runtime-deps",
|
||||
],
|
||||
[
|
||||
"755557a6ea609e5b9d9fe9d61beb7e75651641e26a3f0ef2fe1fc3a973b398c8",
|
||||
"base feishu-channel bootstrap-persona channel-post-core-restore plugin-deps-cleanup configured-plugin-installs stale-source-plugin-shadow tilde-log-path versioned-runtime-deps",
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js";
|
||||
import type { TranscriptStopRequest } from "../../transcripts/provider-types.js";
|
||||
import { TranscriptsStore } from "../../transcripts/store.js";
|
||||
import { createTranscriptsAutoStartService, createTranscriptsTool } from "./transcripts-tool.js";
|
||||
@@ -38,7 +39,15 @@ async function createHarness(stateDir: string, pluginConfig: Record<string, unkn
|
||||
};
|
||||
}
|
||||
|
||||
function storeFor(stateDir: string): TranscriptsStore {
|
||||
return new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
|
||||
});
|
||||
}
|
||||
|
||||
describe("transcripts tool", () => {
|
||||
afterEach(() => closeOpenClawStateDatabaseForTest());
|
||||
|
||||
beforeEach(() => {
|
||||
getTranscriptSourceProviderMock.mockReset();
|
||||
});
|
||||
@@ -141,12 +150,14 @@ describe("transcripts tool", () => {
|
||||
expect(startupSignal?.aborted).toBe(false);
|
||||
await emitAfterStart?.();
|
||||
|
||||
await expect(
|
||||
fs.readFile(
|
||||
path.join(stateDir, "transcripts", currentDateDir(), "ongoing-meeting", "transcript.jsonl"),
|
||||
"utf8",
|
||||
),
|
||||
).resolves.toContain("captured after the start action completed\\nsecond\\tcolumn");
|
||||
const ongoingStore = storeFor(stateDir);
|
||||
const ongoingSession = await ongoingStore.readSession("ongoing-meeting");
|
||||
expect(ongoingSession).toBeDefined();
|
||||
await expect(ongoingStore.readUtterancesForSession(ongoingSession!)).resolves.toEqual([
|
||||
expect.objectContaining({
|
||||
text: "captured after the start action completed\nsecond\tcolumn",
|
||||
}),
|
||||
]);
|
||||
await tool.execute(
|
||||
"call-2",
|
||||
{ action: "stop", sessionId: "ongoing-meeting" },
|
||||
@@ -200,18 +211,10 @@ describe("transcripts tool", () => {
|
||||
),
|
||||
).rejects.toThrow("transcripts start aborted; provider cleanup failed: voice cleanup failed");
|
||||
|
||||
await expect(
|
||||
fs.readFile(
|
||||
path.join(
|
||||
stateDir,
|
||||
"transcripts",
|
||||
currentDateDir(),
|
||||
"cancelled-meeting-retry",
|
||||
"transcript.jsonl",
|
||||
),
|
||||
"utf8",
|
||||
),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
const cancelledStore = storeFor(stateDir);
|
||||
const cancelledSession = await cancelledStore.readSession("cancelled-meeting-retry");
|
||||
expect(cancelledSession).toBeDefined();
|
||||
await expect(cancelledStore.readUtterancesForSession(cancelledSession!)).resolves.toEqual([]);
|
||||
expect(stop).toHaveBeenCalledOnce();
|
||||
|
||||
await expect(
|
||||
@@ -440,12 +443,12 @@ describe("transcripts tool", () => {
|
||||
"utf8",
|
||||
),
|
||||
).resolves.toContain('"Alex: We decided to ship Discord first."');
|
||||
await expect(
|
||||
fs.readFile(
|
||||
path.join(stateDir, "transcripts", currentDateDir(), "design-review", "transcript.jsonl"),
|
||||
"utf8",
|
||||
),
|
||||
).resolves.toContain("Alex");
|
||||
const stored = await storeFor(stateDir).readSession("design-review");
|
||||
expect(stored).toBeDefined();
|
||||
await expect(storeFor(stateDir).readUtterancesForSession(stored!)).resolves.toEqual([
|
||||
expect.objectContaining({ text: "We decided to ship Discord first." }),
|
||||
expect.objectContaining({ text: "Action item: add Slack import later." }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("bounds summary input while retaining the full transcript", async () => {
|
||||
@@ -477,17 +480,16 @@ describe("transcripts tool", () => {
|
||||
);
|
||||
expect(summary).not.toContain("transcript line 0\n");
|
||||
expect(summary).toContain("transcript line 2000");
|
||||
const storedTranscript = await fs.readFile(
|
||||
path.join(stateDir, "transcripts", currentDateDir(), "long-meeting", "transcript.jsonl"),
|
||||
"utf8",
|
||||
);
|
||||
expect(storedTranscript).toContain("transcript line 0");
|
||||
expect(storedTranscript).toContain("transcript line 2000");
|
||||
const stored = await storeFor(stateDir).readSession("long-meeting");
|
||||
expect(stored).toBeDefined();
|
||||
const storedTranscript = await storeFor(stateDir).readUtterancesForSession(stored!);
|
||||
expect(storedTranscript[0]?.text).toContain("transcript line 0");
|
||||
expect(storedTranscript.at(-1)?.text).toContain("transcript line 2000");
|
||||
});
|
||||
|
||||
it("requires date-qualified selectors for repeated stored session ids", async () => {
|
||||
const stateDir = await makeStateDir();
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"));
|
||||
const store = storeFor(stateDir);
|
||||
await store.writeSession({
|
||||
sessionId: "standup",
|
||||
title: "Tuesday standup",
|
||||
@@ -618,17 +620,14 @@ describe("transcripts tool", () => {
|
||||
"utf8",
|
||||
),
|
||||
).resolves.toContain("publish the notes");
|
||||
await expect(
|
||||
fs.readFile(
|
||||
path.join(stateDir, "transcripts", currentDateDir(), "standup", "metadata.json"),
|
||||
"utf8",
|
||||
),
|
||||
).resolves.toContain("providerStopError");
|
||||
await expect(storeFor(stateDir).readSession("standup")).resolves.toMatchObject({
|
||||
metadata: { providerStopError: "Discord voice manager is unavailable" },
|
||||
});
|
||||
});
|
||||
|
||||
it("does not stop a current active session when summarizing an older dated duplicate", async () => {
|
||||
const stateDir = await makeStateDir();
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"));
|
||||
const store = storeFor(stateDir);
|
||||
const olderSession = {
|
||||
sessionId: "standup",
|
||||
title: "Older standup",
|
||||
@@ -745,12 +744,9 @@ describe("transcripts tool", () => {
|
||||
},
|
||||
});
|
||||
expect(request.startupWaitMs).toBe(30_000);
|
||||
await expect(
|
||||
fs.readFile(
|
||||
path.join(stateDir, "transcripts", currentDateDir(), "standup", "metadata.json"),
|
||||
"utf8",
|
||||
),
|
||||
).resolves.toContain("Standup");
|
||||
await expect(storeFor(stateDir).readSession("standup")).resolves.toMatchObject({
|
||||
title: "Standup",
|
||||
});
|
||||
await service.stop();
|
||||
expect(stop).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
@@ -67,7 +67,9 @@ const TranscriptsSchema = Type.Object(
|
||||
);
|
||||
|
||||
function createStore(ctx: TranscriptsRuntimeContext): TranscriptsStore {
|
||||
return new TranscriptsStore(path.join(ctx.stateDir, "transcripts"));
|
||||
return new TranscriptsStore(path.join(ctx.stateDir, "transcripts"), {
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: ctx.stateDir },
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForPendingAutoStartsToSettle(
|
||||
@@ -92,28 +94,24 @@ async function waitForPendingAutoStartsToSettle(
|
||||
}
|
||||
}
|
||||
|
||||
// Summaries are persisted beside the session so stop/import/summarize actions
|
||||
// return both model-readable details and a durable artifact path.
|
||||
// Tool stop/import/summarize actions explicitly materialize artifacts, but a
|
||||
// divergent export must not turn a successful canonical summary write into failure.
|
||||
async function summarizeAndPersist(params: {
|
||||
config: ReturnType<typeof resolveTranscriptsConfig>;
|
||||
store: TranscriptsStore;
|
||||
session: TranscriptSessionDescriptor;
|
||||
sessionDir?: string;
|
||||
}) {
|
||||
const utterances =
|
||||
params.sessionDir !== undefined
|
||||
? await params.store.readUtterancesFromSessionDir(params.sessionDir, {
|
||||
maxUtterances: params.config.maxUtterances,
|
||||
})
|
||||
: await params.store.readUtterancesForSession(params.session, {
|
||||
maxUtterances: params.config.maxUtterances,
|
||||
});
|
||||
const utterances = await params.store.readUtterancesForSession(params.session, {
|
||||
maxUtterances: params.config.maxUtterances,
|
||||
});
|
||||
const summary = summarizeTranscripts({ session: params.session, utterances });
|
||||
const summaryPath =
|
||||
params.sessionDir !== undefined
|
||||
? await params.store.writeSummaryToDir(summary, params.sessionDir)
|
||||
: await params.store.writeSummary(summary, params.session);
|
||||
return { summary, summaryPath };
|
||||
const intendedSummaryPath = await params.store.writeSummary(summary, params.session);
|
||||
try {
|
||||
const artifacts = await params.store.materializeSessionArtifacts(params.session, "all");
|
||||
return { summary, summaryPath: artifacts.summaryPath };
|
||||
} catch (error) {
|
||||
return { summary, intendedSummaryPath, summaryExportError: String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
async function stopTranscripts(params: {
|
||||
@@ -127,9 +125,9 @@ async function stopTranscripts(params: {
|
||||
});
|
||||
const directActive = activeSessions.get(sessionSelector);
|
||||
const resolvedEntry: TranscriptsSessionEntry | undefined = directActive
|
||||
? { session: directActive.session, sessionDir: params.store.sessionDir(directActive.session) }
|
||||
? undefined
|
||||
: await params.store.readSessionEntry(sessionSelector);
|
||||
const resolvedSession = resolvedEntry?.session;
|
||||
const resolvedSession = directActive?.session ?? resolvedEntry?.session;
|
||||
const activeCandidate =
|
||||
resolvedSession !== undefined ? activeSessions.get(resolvedSession.sessionId) : undefined;
|
||||
const activeMatchesResolved =
|
||||
@@ -188,18 +186,23 @@ async function stopTranscripts(params: {
|
||||
} else {
|
||||
await params.store.updateStopped(sessionSelector, stoppedAt);
|
||||
}
|
||||
const { summaryPath, summary } = await summarizeAndPersist({
|
||||
config: resolveTranscriptsConfig(params.ctx.config?.transcripts),
|
||||
store: params.store,
|
||||
session: stoppedSession,
|
||||
sessionDir: selectedActive ? undefined : resolvedEntry?.sessionDir,
|
||||
});
|
||||
return toolText(`Transcripts stopped: ${sessionId}\nSummary: ${summaryPath}`, {
|
||||
sessionId,
|
||||
...(providerStopError ? { providerStopError } : {}),
|
||||
summary,
|
||||
summaryPath,
|
||||
});
|
||||
const { summaryPath, intendedSummaryPath, summary, summaryExportError } =
|
||||
await summarizeAndPersist({
|
||||
config: resolveTranscriptsConfig(params.ctx.config?.transcripts),
|
||||
store: params.store,
|
||||
session: stoppedSession,
|
||||
});
|
||||
return toolText(
|
||||
`Transcripts stopped: ${sessionId}${summaryPath ? `\nSummary: ${summaryPath}` : `\nSummary export failed: ${summaryExportError}`}`,
|
||||
{
|
||||
sessionId,
|
||||
...(providerStopError ? { providerStopError } : {}),
|
||||
...(summaryExportError ? { summaryExportError } : {}),
|
||||
...(intendedSummaryPath ? { intendedSummaryPath } : {}),
|
||||
summary,
|
||||
...(summaryPath ? { summaryPath } : {}),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function importTranscripts(params: {
|
||||
@@ -233,17 +236,23 @@ async function importTranscripts(params: {
|
||||
for (const utterance of utterances) {
|
||||
await params.store.appendUtteranceForSession(session, utterance);
|
||||
}
|
||||
const { summaryPath, summary } = await summarizeAndPersist({
|
||||
config: resolveTranscriptsConfig(params.ctx.config?.transcripts),
|
||||
store: params.store,
|
||||
session,
|
||||
});
|
||||
return toolText(`Transcript imported: ${session.sessionId}\nSummary: ${summaryPath}`, {
|
||||
sessionId: session.sessionId,
|
||||
utteranceCount: utterances.length,
|
||||
summary,
|
||||
summaryPath,
|
||||
});
|
||||
const { summaryPath, intendedSummaryPath, summary, summaryExportError } =
|
||||
await summarizeAndPersist({
|
||||
config: resolveTranscriptsConfig(params.ctx.config?.transcripts),
|
||||
store: params.store,
|
||||
session,
|
||||
});
|
||||
return toolText(
|
||||
`Transcript imported: ${session.sessionId}${summaryPath ? `\nSummary: ${summaryPath}` : `\nSummary export failed: ${summaryExportError}`}`,
|
||||
{
|
||||
sessionId: session.sessionId,
|
||||
utteranceCount: utterances.length,
|
||||
...(summaryExportError ? { summaryExportError } : {}),
|
||||
...(intendedSummaryPath ? { intendedSummaryPath } : {}),
|
||||
summary,
|
||||
...(summaryPath ? { summaryPath } : {}),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function summarizeExisting(params: {
|
||||
@@ -259,17 +268,22 @@ async function summarizeExisting(params: {
|
||||
if (!entry) {
|
||||
throw new Error(`transcripts session not found: ${sessionId}`);
|
||||
}
|
||||
const { summaryPath, summary } = await summarizeAndPersist({
|
||||
config: params.config,
|
||||
store: params.store,
|
||||
session: entry.session,
|
||||
sessionDir: entry.sessionDir,
|
||||
});
|
||||
return toolText(`Transcripts summarized: ${sessionId}\nSummary: ${summaryPath}`, {
|
||||
sessionId,
|
||||
summary,
|
||||
summaryPath,
|
||||
});
|
||||
const { summaryPath, intendedSummaryPath, summary, summaryExportError } =
|
||||
await summarizeAndPersist({
|
||||
config: params.config,
|
||||
store: params.store,
|
||||
session: entry.session,
|
||||
});
|
||||
return toolText(
|
||||
`Transcripts summarized: ${sessionId}${summaryPath ? `\nSummary: ${summaryPath}` : `\nSummary export failed: ${summaryExportError}`}`,
|
||||
{
|
||||
sessionId,
|
||||
...(summaryExportError ? { summaryExportError } : {}),
|
||||
...(intendedSummaryPath ? { intendedSummaryPath } : {}),
|
||||
summary,
|
||||
...(summaryPath ? { summaryPath } : {}),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function statusTranscripts(ctx: TranscriptsRuntimeContext) {
|
||||
@@ -412,7 +426,7 @@ export function createTranscriptsAutoStartService(ctx: TranscriptsRuntimeContext
|
||||
if (!config.enabled || config.autoStart.length === 0) {
|
||||
return;
|
||||
}
|
||||
const store = new TranscriptsStore(path.join(ctx.stateDir, "transcripts"));
|
||||
const store = createStore(ctx);
|
||||
for (const entry of config.autoStart) {
|
||||
startEntry(
|
||||
{
|
||||
@@ -441,7 +455,7 @@ export function createTranscriptsAutoStartService(ctx: TranscriptsRuntimeContext
|
||||
}`,
|
||||
);
|
||||
}
|
||||
const store = new TranscriptsStore(path.join(ctx.stateDir, "transcripts"));
|
||||
const store = createStore(ctx);
|
||||
for (const sessionId of startedSessionIds) {
|
||||
await stopTranscripts({
|
||||
ctx,
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
// Register transcripts tests cover transcript command registration and file handling.
|
||||
// Transcripts CLI tests cover SQLite reads and explicit artifact materialization.
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { Command } from "commander";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { executeSqliteQuerySync, getNodeSqliteKysely } from "../../infra/kysely-sync.js";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "../../state/openclaw-state-db.generated.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
} from "../../state/openclaw-state-db.js";
|
||||
import { manualTranscriptSourceProvider } from "../../transcripts/manual-source.js";
|
||||
import type { TranscriptSessionDescriptor } from "../../transcripts/provider-types.js";
|
||||
import { TranscriptsStore } from "../../transcripts/store.js";
|
||||
@@ -16,32 +22,31 @@ async function makeStateDir(): Promise<string> {
|
||||
return await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-transcripts-cli-"));
|
||||
}
|
||||
|
||||
function storeFor(stateDir: string): TranscriptsStore {
|
||||
return new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
|
||||
});
|
||||
}
|
||||
|
||||
async function writeSession(
|
||||
stateDir: string,
|
||||
sessionId: string,
|
||||
date = "2026-05-22",
|
||||
): Promise<string> {
|
||||
const sessionDir = path.join(stateDir, "transcripts", date, sessionId);
|
||||
await fs.mkdir(sessionDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(sessionDir, "metadata.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
sessionId,
|
||||
title: "Design review",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: `${date}T10:00:00.000Z`,
|
||||
stoppedAt: `${date}T10:05:00.000Z`,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(sessionDir, "summary.md"),
|
||||
"# Design review\n\n## Action Items\n- Sam: Ship CLI\n",
|
||||
);
|
||||
return sessionDir;
|
||||
const session: TranscriptSessionDescriptor = {
|
||||
sessionId,
|
||||
title: "Design review",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: `${date}T10:00:00.000Z`,
|
||||
stoppedAt: `${date}T10:05:00.000Z`,
|
||||
};
|
||||
const store = storeFor(stateDir);
|
||||
const utterance = { text: "Action item: Ship CLI", speaker: { label: "Sam" } };
|
||||
const utterances = [utterance];
|
||||
await store.writeSession(session);
|
||||
await store.appendUtteranceForSession(session, utterance);
|
||||
await store.writeSummary(summarizeTranscripts({ session, utterances }), session);
|
||||
return store.sessionDir(session);
|
||||
}
|
||||
|
||||
async function runTranscriptsCli(args: string[]): Promise<string> {
|
||||
@@ -72,6 +77,7 @@ describe("transcripts CLI", () => {
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
if (originalStateDir === undefined) {
|
||||
delete process.env.OPENCLAW_STATE_DIR;
|
||||
} else {
|
||||
@@ -86,7 +92,7 @@ describe("transcripts CLI", () => {
|
||||
expect(program.commands.map((command) => command.name())).toContain("transcripts");
|
||||
});
|
||||
|
||||
it("lists stored transcript sessions", async () => {
|
||||
it("lists stored transcript sessions from SQLite", async () => {
|
||||
const sessionDir = await writeSession(stateDir, "design-review");
|
||||
|
||||
const output = await runTranscriptsCli(["list"]);
|
||||
@@ -96,29 +102,67 @@ describe("transcripts CLI", () => {
|
||||
expect(output).toContain(path.join(sessionDir, "summary.md"));
|
||||
});
|
||||
|
||||
it("prints summary markdown for a session", async () => {
|
||||
await writeSession(stateDir, "design-review");
|
||||
it("prints summary markdown and keeps its export current", async () => {
|
||||
const sessionDir = await writeSession(stateDir, "design-review");
|
||||
await fs.rm(sessionDir, { recursive: true, force: true });
|
||||
|
||||
const output = await runTranscriptsCli(["show", "design-review"]);
|
||||
|
||||
expect(output).toContain("# Design review");
|
||||
expect(output).toContain("Ship CLI");
|
||||
expect(output.endsWith("\n")).toBe(true);
|
||||
const jsonOutput = JSON.parse(await runTranscriptsCli(["show", "design-review", "--json"])) as {
|
||||
summary: string;
|
||||
};
|
||||
expect(jsonOutput.summary.endsWith("\n")).toBe(true);
|
||||
await expect(fs.readFile(path.join(sessionDir, "summary.md"), "utf8")).resolves.toContain(
|
||||
"Ship CLI",
|
||||
);
|
||||
});
|
||||
|
||||
it("sanitizes summaries created before the upgrade at the show boundary", async () => {
|
||||
const sessionDir = await writeSession(stateDir, "legacy-summary");
|
||||
await fs.writeFile(
|
||||
path.join(sessionDir, "summary.md"),
|
||||
"# Legacy\n\n- first\tcolumn\n- \u001b[2J\u001b[31mADMIN APPROVED\u001b[0m\n",
|
||||
it("keeps JSON inspection available before a summary exists", async () => {
|
||||
await storeFor(stateDir).writeSession({
|
||||
sessionId: "active-session",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-05-22T10:00:00.000Z",
|
||||
});
|
||||
|
||||
const jsonOutput = await runTranscriptsCli(["show", "active-session", "--json"]);
|
||||
|
||||
expect(JSON.parse(jsonOutput)).toMatchObject({
|
||||
session: { sessionId: "active-session" },
|
||||
summary: null,
|
||||
});
|
||||
await expect(runTranscriptsCli(["show", "active-session"])).rejects.toThrow(
|
||||
"summary.md not found",
|
||||
);
|
||||
});
|
||||
|
||||
it("sanitizes stored summary control bytes at the show boundary", async () => {
|
||||
await writeSession(stateDir, "legacy-summary");
|
||||
const database = openOpenClawStateDatabase({
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
|
||||
});
|
||||
const db = getNodeSqliteKysely<
|
||||
Pick<OpenClawStateKyselyDatabase, "meeting_transcript_summaries">
|
||||
>(database.db);
|
||||
executeSqliteQuerySync(
|
||||
database.db,
|
||||
db
|
||||
.updateTable("meeting_transcript_summaries")
|
||||
.set({
|
||||
markdown: "# Legacy\n\n- first\tcolumn\n- \u001b[2J\u001b[31mADMIN APPROVED\u001b[0m",
|
||||
})
|
||||
.where("session_id", "=", "legacy-summary"),
|
||||
);
|
||||
|
||||
const output = await runTranscriptsCli(["show", "legacy-summary"]);
|
||||
|
||||
expect(output).toContain("# Legacy\n\n- first\\tcolumn\n- ADMIN APPROVED\n");
|
||||
expect(output).toContain("# Legacy\n\n- first\\tcolumn\n- ADMIN APPROVED");
|
||||
expect(output).not.toContain("\u001b");
|
||||
});
|
||||
|
||||
it("show prints imported summaries without terminal control bytes", async () => {
|
||||
it("round-trips ANSI-bearing ids without terminal control bytes", async () => {
|
||||
const session: TranscriptSessionDescriptor = {
|
||||
sessionId: "ansi-\u001b[31mprovider\u001b[0m",
|
||||
title: "\u001b[31mANSI import\u001b[0m",
|
||||
@@ -126,7 +170,7 @@ describe("transcripts CLI", () => {
|
||||
startedAt: "2026-05-22T10:00:00.000Z",
|
||||
stoppedAt: "2026-05-22T10:05:00.000Z",
|
||||
};
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"));
|
||||
const store = storeFor(stateDir);
|
||||
await store.writeSession(session);
|
||||
const utterances =
|
||||
(await manualTranscriptSourceProvider.importTranscript?.({
|
||||
@@ -138,76 +182,34 @@ describe("transcripts CLI", () => {
|
||||
}
|
||||
await store.writeSummary(summarizeTranscripts({ session, utterances }), session);
|
||||
|
||||
const output = await runTranscriptsCli(["show", session.sessionId]);
|
||||
const listOutput = await runTranscriptsCli(["list"]);
|
||||
|
||||
expect(output).toContain("# ANSI import");
|
||||
expect(output).toContain("Session: ansi-provider");
|
||||
expect(output).toContain("Attacker: ADMIN APPROVED");
|
||||
expect(output).not.toContain("\u001b");
|
||||
expect(listOutput).toContain("2026-05-22/ansi--31mprovider-0m");
|
||||
expect(listOutput).toContain("ANSI import");
|
||||
expect(listOutput).not.toContain("\u001b");
|
||||
});
|
||||
|
||||
it("list selectors for ANSI-bearing session ids round-trip through show and path", async () => {
|
||||
const session: TranscriptSessionDescriptor = {
|
||||
sessionId: "ansi-\u001b[31mprovider\u001b[0m",
|
||||
title: "ANSI import",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-05-22T10:00:00.000Z",
|
||||
stoppedAt: "2026-05-22T10:05:00.000Z",
|
||||
};
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"));
|
||||
await store.writeSession(session);
|
||||
const utterances =
|
||||
(await manualTranscriptSourceProvider.importTranscript?.({
|
||||
session,
|
||||
text: "Sam: We decided to ship the CLI.",
|
||||
})) ?? [];
|
||||
await store.writeSummary(summarizeTranscripts({ session, utterances }), session);
|
||||
|
||||
const listOutput = await runTranscriptsCli(["list"]);
|
||||
const selector = listOutput.split("\t")[0] ?? "";
|
||||
expect(selector).toBe("2026-05-22/ansi--31mprovider-0m");
|
||||
|
||||
const showOutput = await runTranscriptsCli(["show", selector]);
|
||||
const pathOutput = await runTranscriptsCli(["path", selector]);
|
||||
|
||||
expect(selector).toBe("2026-05-22/ansi--31mprovider-0m");
|
||||
expect(showOutput).toContain("Session: ansi-provider");
|
||||
expect(showOutput).toContain("We decided to ship the CLI.");
|
||||
expect(pathOutput.trim()).toBe(path.join(store.sessionDir(session), "summary.md"));
|
||||
expect(showOutput).toContain("Attacker: ADMIN APPROVED");
|
||||
expect(`${listOutput}${showOutput}`).not.toContain("\u001b");
|
||||
});
|
||||
|
||||
it("list --json escapes C1 control characters while JSON.parse round-trips raw values", async () => {
|
||||
it("escapes C1 control characters in list JSON", async () => {
|
||||
const title = "CSI \u009b31m injected \u007f\u0085 title";
|
||||
const sessionDir = path.join(stateDir, "transcripts", "2026-05-22", "c1-title");
|
||||
await fs.mkdir(sessionDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(sessionDir, "metadata.json"),
|
||||
JSON.stringify({
|
||||
sessionId: "c1-title",
|
||||
title,
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-05-22T10:00:00.000Z",
|
||||
stoppedAt: "2026-05-22T10:05:00.000Z",
|
||||
}),
|
||||
);
|
||||
await storeFor(stateDir).writeSession({
|
||||
sessionId: "c1-title",
|
||||
title,
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-05-22T10:00:00.000Z",
|
||||
});
|
||||
|
||||
const output = await runTranscriptsCli(["list", "--json"]);
|
||||
|
||||
const bytes = Buffer.from(output, "utf8");
|
||||
expect(bytes.includes(Buffer.from([0xc2, 0x9b]))).toBe(false);
|
||||
expect(bytes.includes(0x7f)).toBe(false);
|
||||
expect(/[\u007f-\u009f]/.test(output)).toBe(false);
|
||||
expect(output).toContain("\\u009b");
|
||||
const parsed = JSON.parse(output) as Array<{ sessionId: string; title: string }>;
|
||||
expect(parsed).toHaveLength(1);
|
||||
expect(parsed[0]?.sessionId).toBe("c1-title");
|
||||
expect(parsed[0]?.title).toBe(title);
|
||||
expect(parsed).toEqual([expect.objectContaining({ sessionId: "c1-title", title })]);
|
||||
});
|
||||
|
||||
it("ignores unrelated corrupt metadata while reading a valid session", async () => {
|
||||
it("ignores unrelated corrupt export files", async () => {
|
||||
await writeSession(stateDir, "design-review");
|
||||
const corruptDir = path.join(stateDir, "transcripts", "corrupt");
|
||||
await fs.mkdir(corruptDir, { recursive: true });
|
||||
@@ -221,7 +223,7 @@ describe("transcripts CLI", () => {
|
||||
expect(showOutput).toContain("# Design review");
|
||||
});
|
||||
|
||||
it("requires date-qualified selectors for repeated human session ids", async () => {
|
||||
it("requires date-qualified selectors for repeated ids", async () => {
|
||||
const olderSessionDir = await writeSession(stateDir, "standup", "2026-05-21");
|
||||
await writeSession(stateDir, "standup", "2026-05-22");
|
||||
|
||||
@@ -233,11 +235,22 @@ describe("transcripts CLI", () => {
|
||||
expect(output.trim()).toBe(path.join(olderSessionDir, "summary.md"));
|
||||
});
|
||||
|
||||
it("prints the summary path by default", async () => {
|
||||
it("materializes metadata, transcript, and directory exports from SQLite", async () => {
|
||||
const sessionDir = await writeSession(stateDir, "design-review");
|
||||
await fs.rm(sessionDir, { recursive: true, force: true });
|
||||
|
||||
const output = await runTranscriptsCli(["path", "design-review"]);
|
||||
const metadataOutput = await runTranscriptsCli(["path", "design-review", "--metadata"]);
|
||||
const transcriptOutput = await runTranscriptsCli(["path", "design-review", "--transcript"]);
|
||||
const dirOutput = await runTranscriptsCli(["path", "design-review", "--dir"]);
|
||||
|
||||
expect(output.trim()).toBe(path.join(sessionDir, "summary.md"));
|
||||
expect(metadataOutput.trim()).toBe(path.join(sessionDir, "metadata.json"));
|
||||
expect(transcriptOutput.trim()).toBe(path.join(sessionDir, "transcript.jsonl"));
|
||||
expect(dirOutput.trim()).toBe(sessionDir);
|
||||
await expect(fs.readFile(path.join(sessionDir, "metadata.json"), "utf8")).resolves.toContain(
|
||||
'"sessionId": "design-review"',
|
||||
);
|
||||
await expect(fs.readFile(path.join(sessionDir, "transcript.jsonl"), "utf8")).resolves.toContain(
|
||||
'"text":"Action item: Ship CLI"',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
// `openclaw transcripts`: local state inspector for stored transcript metadata and summaries.
|
||||
import type { Dirent } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
// `openclaw transcripts`: SQLite-backed transcript inspector and artifact exporter.
|
||||
import path from "node:path";
|
||||
import type { Command } from "commander";
|
||||
import { sanitizeTerminalText } from "../../../packages/terminal-core/src/safe-text.js";
|
||||
import { resolveStateDir } from "../../config/paths.js";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
import type { TranscriptSessionDescriptor } from "../../transcripts/provider-types.js";
|
||||
import {
|
||||
TranscriptsStore,
|
||||
type TranscriptArtifactKind,
|
||||
type TranscriptsSessionEntry,
|
||||
} from "../../transcripts/store.js";
|
||||
|
||||
type TranscriptsCliOptions = {
|
||||
json?: boolean;
|
||||
@@ -18,54 +19,11 @@ type TranscriptsPathOptions = TranscriptsCliOptions & {
|
||||
transcript?: boolean;
|
||||
};
|
||||
|
||||
type StoredTranscriptsSession = {
|
||||
session: TranscriptSessionDescriptor;
|
||||
sessionDir: string;
|
||||
date: string;
|
||||
summaryPath: string;
|
||||
hasSummary: boolean;
|
||||
};
|
||||
|
||||
const TRANSCRIPTS_STATE_SUBDIR = "transcripts";
|
||||
|
||||
function safeSegment(value: string): string {
|
||||
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "session";
|
||||
}
|
||||
|
||||
function stateRootDir(): string {
|
||||
return path.join(resolveStateDir(), TRANSCRIPTS_STATE_SUBDIR);
|
||||
}
|
||||
|
||||
function dateFromSessionId(sessionId: string): string | undefined {
|
||||
return sessionId
|
||||
.match(/^transcript-(\d{4})-(\d{2})-(\d{2})T/)
|
||||
?.slice(1, 4)
|
||||
.join("-");
|
||||
}
|
||||
|
||||
function sessionDir(date: string, sessionId: string): string {
|
||||
return path.join(stateRootDir(), date, safeSegment(sessionId));
|
||||
}
|
||||
|
||||
// Selectors are date-qualified when duplicate session ids can exist across transcript days.
|
||||
function readDateFromSessionDir(sessionDirValue: string): string {
|
||||
const candidate = path.basename(path.dirname(sessionDirValue));
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(candidate)) {
|
||||
throw new Error(`invalid transcripts date directory: ${candidate}`);
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function formatSelector(entry: StoredTranscriptsSession): string {
|
||||
return `${entry.date}/${safeSegment(entry.session.sessionId)}`;
|
||||
}
|
||||
|
||||
function parseQualifiedSelector(selector: string): { date: string; sessionId: string } | null {
|
||||
const match = selector.match(/^(\d{4}-\d{2}-\d{2})\/(.+)$/);
|
||||
if (!match?.[1] || !match[2]) {
|
||||
return null;
|
||||
}
|
||||
return { date: match[1], sessionId: match[2] };
|
||||
function createStore(): TranscriptsStore {
|
||||
const stateDir = resolveStateDir();
|
||||
return new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
|
||||
});
|
||||
}
|
||||
|
||||
function writeLine(value: string): void {
|
||||
@@ -81,163 +39,33 @@ function writeJson(value: unknown): void {
|
||||
);
|
||||
}
|
||||
|
||||
function isNodeError(err: unknown, code: string): boolean {
|
||||
return Boolean(err && typeof err === "object" && "code" in err && err.code === code);
|
||||
}
|
||||
|
||||
async function pathExists(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(filePath);
|
||||
return true;
|
||||
} catch (err) {
|
||||
if (isNodeError(err, "ENOENT")) {
|
||||
return false;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function readJsonFile<T>(filePath: string): Promise<T> {
|
||||
return JSON.parse(await fs.readFile(filePath, "utf8")) as T;
|
||||
}
|
||||
|
||||
async function readStoredSession(
|
||||
sessionDirLocal: string,
|
||||
options: { ignoreInvalid?: boolean } = {},
|
||||
): Promise<StoredTranscriptsSession | null> {
|
||||
const metadataPath = path.join(sessionDirLocal, "metadata.json");
|
||||
try {
|
||||
const session = await readJsonFile<TranscriptSessionDescriptor>(metadataPath);
|
||||
const summaryPath = path.join(sessionDirLocal, "summary.md");
|
||||
return {
|
||||
session,
|
||||
sessionDir: sessionDirLocal,
|
||||
date: readDateFromSessionDir(sessionDirLocal),
|
||||
summaryPath,
|
||||
hasSummary: await pathExists(summaryPath),
|
||||
};
|
||||
} catch (err) {
|
||||
if (isNodeError(err, "ENOENT")) {
|
||||
return null;
|
||||
}
|
||||
if (options.ignoreInvalid) {
|
||||
return null;
|
||||
}
|
||||
throw new Error(`invalid transcripts metadata at ${metadataPath}: ${formatErrorMessage(err)}`, {
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function listStoredSessionDirs(): Promise<string[]> {
|
||||
let entries: Dirent[];
|
||||
try {
|
||||
entries = await fs.readdir(stateRootDir(), { withFileTypes: true });
|
||||
} catch (err) {
|
||||
if (isNodeError(err, "ENOENT")) {
|
||||
return [];
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const dirs: string[] = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
const firstLevelDir = path.join(stateRootDir(), entry.name);
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(entry.name)) {
|
||||
continue;
|
||||
}
|
||||
const nestedEntries = await fs.readdir(firstLevelDir, { withFileTypes: true });
|
||||
dirs.push(
|
||||
...nestedEntries
|
||||
.filter((nestedEntry) => nestedEntry.isDirectory())
|
||||
.map((nestedEntry) => path.join(firstLevelDir, nestedEntry.name)),
|
||||
);
|
||||
}
|
||||
return dirs;
|
||||
}
|
||||
|
||||
function assertRequestedSession(
|
||||
entry: StoredTranscriptsSession,
|
||||
sessionId: string,
|
||||
): StoredTranscriptsSession {
|
||||
if (entry.session.sessionId !== sessionId && safeSegment(entry.session.sessionId) !== sessionId) {
|
||||
throw new Error(
|
||||
`transcripts metadata mismatch for ${sessionId}: found ${entry.session.sessionId}`,
|
||||
);
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
async function requireStoredSession(selector: string): Promise<StoredTranscriptsSession> {
|
||||
const qualified = parseQualifiedSelector(selector);
|
||||
if (qualified) {
|
||||
const session = await readStoredSession(sessionDir(qualified.date, qualified.sessionId));
|
||||
if (!session) {
|
||||
throw new Error(`transcripts session not found: ${selector}`);
|
||||
}
|
||||
return assertRequestedSession(session, qualified.sessionId);
|
||||
}
|
||||
|
||||
const idDate = dateFromSessionId(selector);
|
||||
const session = idDate ? await readStoredSession(sessionDir(idDate, selector)) : null;
|
||||
if (session) {
|
||||
return assertRequestedSession(session, selector);
|
||||
}
|
||||
const sessions = await listStoredSessions();
|
||||
const matches = sessions.filter(
|
||||
(entry) =>
|
||||
entry.session.sessionId === selector || safeSegment(entry.session.sessionId) === selector,
|
||||
);
|
||||
if (matches.length === 1 && matches[0]) {
|
||||
return assertRequestedSession(matches[0], selector);
|
||||
}
|
||||
if (matches.length > 1) {
|
||||
throw new Error(
|
||||
`multiple transcripts sessions match ${selector}; use one of: ${matches
|
||||
.map(formatSelector)
|
||||
.join(", ")}`,
|
||||
);
|
||||
}
|
||||
throw new Error(`transcripts session not found: ${selector}`);
|
||||
}
|
||||
|
||||
async function listStoredSessions(): Promise<StoredTranscriptsSession[]> {
|
||||
const dirs = await listStoredSessionDirs();
|
||||
const sessions = await Promise.all(
|
||||
dirs.map((dir) =>
|
||||
readStoredSession(dir, {
|
||||
ignoreInvalid: true,
|
||||
}),
|
||||
),
|
||||
);
|
||||
return sessions
|
||||
.filter((session): session is StoredTranscriptsSession => session !== null)
|
||||
.toSorted((left, right) =>
|
||||
(right.session.startedAt ?? "").localeCompare(left.session.startedAt ?? ""),
|
||||
);
|
||||
}
|
||||
|
||||
function formatSessionLine(entry: StoredTranscriptsSession): string {
|
||||
const title = sanitizeTerminalText(entry.session.title?.trim() || "Transcripts");
|
||||
const started = sanitizeTerminalText(entry.session.startedAt || "unknown");
|
||||
const summary = sanitizeTerminalText(entry.hasSummary ? entry.summaryPath : "no summary.md");
|
||||
return `${formatSelector(entry)}\t${started}\t${title}\t${summary}`;
|
||||
}
|
||||
|
||||
function sanitizeMarkdownForTerminal(markdown: string): string {
|
||||
return markdown.split("\n").map(sanitizeTerminalText).join("\n");
|
||||
}
|
||||
|
||||
function formatSessionLine(entry: TranscriptsSessionEntry): string {
|
||||
const title = sanitizeTerminalText(entry.session.title?.trim() || "Transcripts");
|
||||
const started = sanitizeTerminalText(entry.session.startedAt || "unknown");
|
||||
const summary = sanitizeTerminalText(entry.hasSummary ? entry.summaryPath : "no summary.md");
|
||||
return `${entry.selector}\t${started}\t${title}\t${summary}`;
|
||||
}
|
||||
|
||||
async function requireStoredSession(selector: string): Promise<TranscriptsSessionEntry> {
|
||||
const session = await createStore().readSessionEntry(selector);
|
||||
if (!session) {
|
||||
throw new Error(`transcripts session not found: ${selector}`);
|
||||
}
|
||||
return session;
|
||||
}
|
||||
|
||||
async function listCommand(options: TranscriptsCliOptions): Promise<void> {
|
||||
const sessions = await listStoredSessions();
|
||||
const sessions = await createStore().listSessionEntries();
|
||||
if (options.json) {
|
||||
writeJson(
|
||||
sessions.map((entry) => ({
|
||||
sessionId: entry.session.sessionId,
|
||||
selector: formatSelector(entry),
|
||||
date: entry.date,
|
||||
selector: entry.selector,
|
||||
date: entry.selector.slice(0, 10),
|
||||
title: entry.session.title,
|
||||
startedAt: entry.session.startedAt,
|
||||
stoppedAt: entry.session.stoppedAt,
|
||||
@@ -258,47 +86,76 @@ async function listCommand(options: TranscriptsCliOptions): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
async function showCommand(sessionId: string, options: TranscriptsCliOptions): Promise<void> {
|
||||
const session = await requireStoredSession(sessionId);
|
||||
async function showCommand(sessionSelector: string, options: TranscriptsCliOptions): Promise<void> {
|
||||
const store = createStore();
|
||||
const entry = await store.readSessionEntry(sessionSelector);
|
||||
if (!entry) {
|
||||
throw new Error(`transcripts session not found: ${sessionSelector}`);
|
||||
}
|
||||
const storedSummary = await store.readSummary(entry.session);
|
||||
const materializedMarkdown =
|
||||
storedSummary.markdown === undefined
|
||||
? undefined
|
||||
: storedSummary.markdown.endsWith("\n")
|
||||
? storedSummary.markdown
|
||||
: `${storedSummary.markdown}\n`;
|
||||
// `show` is an explicit export boundary: keep the shipped summary path current.
|
||||
await store.materializeSessionArtifacts(entry.session, "summary");
|
||||
if (options.json) {
|
||||
const summary = session.hasSummary ? await fs.readFile(session.summaryPath, "utf8") : null;
|
||||
writeJson({
|
||||
session: session.session,
|
||||
selector: formatSelector(session),
|
||||
path: session.sessionDir,
|
||||
summaryPath: session.summaryPath,
|
||||
summary,
|
||||
session: entry.session,
|
||||
selector: entry.selector,
|
||||
path: entry.sessionDir,
|
||||
summaryPath: entry.summaryPath,
|
||||
summary: materializedMarkdown ?? null,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (!session.hasSummary) {
|
||||
throw new Error(`summary.md not found for transcripts session: ${sessionId}`);
|
||||
if (materializedMarkdown === undefined) {
|
||||
throw new Error(`summary.md not found for transcripts session: ${sessionSelector}`);
|
||||
}
|
||||
process.stdout.write(sanitizeMarkdownForTerminal(await fs.readFile(session.summaryPath, "utf8")));
|
||||
process.stdout.write(sanitizeMarkdownForTerminal(materializedMarkdown));
|
||||
}
|
||||
|
||||
function selectedArtifactKind(options: TranscriptsPathOptions): TranscriptArtifactKind {
|
||||
if (options.dir) {
|
||||
return "all";
|
||||
}
|
||||
if (options.metadata) {
|
||||
return "metadata";
|
||||
}
|
||||
if (options.transcript) {
|
||||
return "transcript";
|
||||
}
|
||||
return "summary";
|
||||
}
|
||||
|
||||
async function pathCommand(selector: string, options: TranscriptsPathOptions): Promise<void> {
|
||||
const session = await requireStoredSession(selector);
|
||||
const store = createStore();
|
||||
const entry = await requireStoredSession(selector);
|
||||
const kind = selectedArtifactKind(options);
|
||||
const artifacts = await store.materializeSessionArtifacts(entry.session, kind);
|
||||
const selectedPath = options.dir
|
||||
? session.sessionDir
|
||||
? artifacts.sessionDir
|
||||
: options.metadata
|
||||
? path.join(session.sessionDir, "metadata.json")
|
||||
? artifacts.metadataPath
|
||||
: options.transcript
|
||||
? path.join(session.sessionDir, "transcript.jsonl")
|
||||
: session.summaryPath;
|
||||
? artifacts.transcriptPath
|
||||
: artifacts.summaryPath;
|
||||
const exists = kind !== "summary" || artifacts.hasSummary;
|
||||
if (options.json) {
|
||||
writeJson({
|
||||
sessionId: session.session.sessionId,
|
||||
selector: formatSelector(session),
|
||||
sessionId: entry.session.sessionId,
|
||||
selector: entry.selector,
|
||||
path: selectedPath,
|
||||
exists: await pathExists(selectedPath),
|
||||
exists,
|
||||
});
|
||||
return;
|
||||
}
|
||||
writeLine(selectedPath);
|
||||
}
|
||||
|
||||
/** Register transcript list/show/path inspection commands. */
|
||||
/** Register transcript list/show/path inspection and export commands. */
|
||||
export function registerTranscriptsCli(program: Command): void {
|
||||
const transcripts = program.command("transcripts").description("Inspect stored transcripts");
|
||||
|
||||
@@ -312,7 +169,7 @@ export function registerTranscriptsCli(program: Command): void {
|
||||
|
||||
transcripts
|
||||
.command("show")
|
||||
.description("Print a transcript summary markdown file")
|
||||
.description("Print and materialize a transcript summary")
|
||||
.argument("<session>", "Transcripts session id or YYYY-MM-DD/session selector")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (sessionId: string, options: TranscriptsCliOptions) => {
|
||||
@@ -321,11 +178,11 @@ export function registerTranscriptsCli(program: Command): void {
|
||||
|
||||
transcripts
|
||||
.command("path")
|
||||
.description("Print a stored transcripts artifact path")
|
||||
.description("Materialize and print a stored transcripts artifact path")
|
||||
.argument("<session>", "Transcripts session id or YYYY-MM-DD/session selector")
|
||||
.option("--dir", "Print the session directory")
|
||||
.option("--metadata", "Print metadata.json")
|
||||
.option("--transcript", "Print transcript.jsonl")
|
||||
.option("--dir", "Materialize all artifacts and print the session directory")
|
||||
.option("--metadata", "Materialize and print metadata.json")
|
||||
.option("--transcript", "Materialize and print transcript.jsonl")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (sessionId: string, options: TranscriptsPathOptions) => {
|
||||
await pathCommand(sessionId, options);
|
||||
|
||||
@@ -75,6 +75,10 @@ import {
|
||||
detectLegacyMcpOAuthStores,
|
||||
migrateLegacyMcpOAuthStores,
|
||||
} from "./state-migrations.mcp-oauth.js";
|
||||
import {
|
||||
detectLegacyMeetingTranscripts,
|
||||
migrateLegacyMeetingTranscripts,
|
||||
} from "./state-migrations.meeting-transcripts.js";
|
||||
import { mergeNotices } from "./state-migrations.messages.js";
|
||||
import {
|
||||
detectLegacyNodeHostConfig,
|
||||
@@ -461,6 +465,11 @@ export async function detectLegacyStateMigrations(params: {
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: params.doctorOnlyStateMigrations,
|
||||
});
|
||||
const meetingTranscripts = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
env,
|
||||
doctorOnlyStateMigrations: params.doctorOnlyStateMigrations,
|
||||
});
|
||||
const restartSentinel = detectLegacyRestartSentinel({ stateDir });
|
||||
const workspace = detectLegacyWorkspaceState({
|
||||
cfg: params.cfg,
|
||||
@@ -657,6 +666,9 @@ export async function detectLegacyStateMigrations(params: {
|
||||
if (mcpOauth.hasLegacy) {
|
||||
preview.push("- MCP OAuth credentials: legacy JSON → shared SQLite state");
|
||||
}
|
||||
if (meetingTranscripts.hasLegacy) {
|
||||
preview.push("- Meeting transcripts: legacy JSON/JSONL files → shared SQLite state");
|
||||
}
|
||||
if (restartSentinel.hasLegacy) {
|
||||
preview.push("- Restart sentinel: legacy JSON → shared SQLite state");
|
||||
}
|
||||
@@ -767,6 +779,7 @@ export async function detectLegacyStateMigrations(params: {
|
||||
deviceAuth,
|
||||
deviceIdentity,
|
||||
mcpOauth,
|
||||
meetingTranscripts,
|
||||
restartSentinel,
|
||||
workspace,
|
||||
webPush,
|
||||
@@ -1046,6 +1059,12 @@ export async function runLegacyStateMigrations(params: {
|
||||
env,
|
||||
stateDir: detected.stateDir,
|
||||
});
|
||||
const meetingTranscripts = await migrateLegacyMeetingTranscripts({
|
||||
detected: detected.meetingTranscripts,
|
||||
env,
|
||||
stateDir: detected.stateDir,
|
||||
now,
|
||||
});
|
||||
const restartSentinel = await migrateLegacyRestartSentinel({
|
||||
detected: detected.restartSentinel,
|
||||
env,
|
||||
@@ -1113,6 +1132,7 @@ export async function runLegacyStateMigrations(params: {
|
||||
deviceAuth,
|
||||
deviceIdentity,
|
||||
mcpOauth,
|
||||
meetingTranscripts,
|
||||
restartSentinel,
|
||||
workspace,
|
||||
webPush,
|
||||
@@ -1142,6 +1162,7 @@ export async function runLegacyStateMigrations(params: {
|
||||
...deviceAuth.changes,
|
||||
...deviceIdentity.changes,
|
||||
...mcpOauth.changes,
|
||||
...meetingTranscripts.changes,
|
||||
...restartSentinel.changes,
|
||||
...workspace.changes,
|
||||
...webPush.changes,
|
||||
@@ -1178,6 +1199,7 @@ export async function runLegacyStateMigrations(params: {
|
||||
...deviceAuth.warnings,
|
||||
...deviceIdentity.warnings,
|
||||
...mcpOauth.warnings,
|
||||
...meetingTranscripts.warnings,
|
||||
...restartSentinel.warnings,
|
||||
...workspace.warnings,
|
||||
...webPush.warnings,
|
||||
@@ -1325,6 +1347,12 @@ export async function autoMigrateLegacyState(params: {
|
||||
stateDir: detected.stateDir,
|
||||
doctorOnlyStateMigrations: params.doctorOnlyStateMigrations,
|
||||
});
|
||||
const meetingTranscripts = await migrateLegacyMeetingTranscripts({
|
||||
detected: detected.meetingTranscripts,
|
||||
env,
|
||||
stateDir: detected.stateDir,
|
||||
now: params.now,
|
||||
});
|
||||
const hasCustomAgentDir = env.OPENCLAW_AGENT_DIR?.trim() || env.PI_CODING_AGENT_DIR?.trim();
|
||||
if (hasCustomAgentDir) {
|
||||
const pluginStateSidecar = await migrateLegacyPluginStateSidecar({
|
||||
@@ -1434,6 +1462,7 @@ export async function autoMigrateLegacyState(params: {
|
||||
updateCheck,
|
||||
deviceAuth,
|
||||
deviceIdentity,
|
||||
meetingTranscripts,
|
||||
restartSentinel,
|
||||
pluginPlans,
|
||||
];
|
||||
@@ -1458,6 +1487,7 @@ export async function autoMigrateLegacyState(params: {
|
||||
currentConversationBindings.changes.length > 0 ||
|
||||
deviceAuth.changes.length > 0 ||
|
||||
deviceIdentity.changes.length > 0 ||
|
||||
meetingTranscripts.changes.length > 0 ||
|
||||
restartSentinel.changes.length > 0 ||
|
||||
channelPairing.changes.length > 0 ||
|
||||
preSessionChannelPlans.changes.length > 0 ||
|
||||
@@ -1497,6 +1527,7 @@ export async function autoMigrateLegacyState(params: {
|
||||
...acpSessionMetadata.changes,
|
||||
...deviceAuth.changes,
|
||||
...deviceIdentity.changes,
|
||||
...meetingTranscripts.changes,
|
||||
];
|
||||
const warnings = [
|
||||
...stateDirResult.warnings,
|
||||
@@ -1507,6 +1538,7 @@ export async function autoMigrateLegacyState(params: {
|
||||
...acpSessionMetadata.warnings,
|
||||
...deviceAuth.warnings,
|
||||
...deviceIdentity.warnings,
|
||||
...meetingTranscripts.warnings,
|
||||
];
|
||||
const notices = [
|
||||
...(stateDirResult.notices ?? []),
|
||||
@@ -1523,7 +1555,8 @@ export async function autoMigrateLegacyState(params: {
|
||||
orphanKeys.changes.length > 0 ||
|
||||
acpSessionMetadata.changes.length > 0 ||
|
||||
deviceAuth.changes.length > 0 ||
|
||||
deviceIdentity.changes.length > 0,
|
||||
deviceIdentity.changes.length > 0 ||
|
||||
meetingTranscripts.changes.length > 0,
|
||||
skipped: false,
|
||||
changes,
|
||||
warnings,
|
||||
@@ -1616,6 +1649,7 @@ export async function autoMigrateLegacyState(params: {
|
||||
...currentConversationBindings.changes,
|
||||
...deviceAuth.changes,
|
||||
...deviceIdentity.changes,
|
||||
...meetingTranscripts.changes,
|
||||
...restartSentinel.changes,
|
||||
...channelPairing.changes,
|
||||
...preSessionChannelPlans.changes,
|
||||
@@ -1644,6 +1678,7 @@ export async function autoMigrateLegacyState(params: {
|
||||
...currentConversationBindings.warnings,
|
||||
...deviceAuth.warnings,
|
||||
...deviceIdentity.warnings,
|
||||
...meetingTranscripts.warnings,
|
||||
...restartSentinel.warnings,
|
||||
...channelPairing.warnings,
|
||||
...preSessionChannelPlans.warnings,
|
||||
@@ -1660,6 +1695,7 @@ export async function autoMigrateLegacyState(params: {
|
||||
updateCheck,
|
||||
deviceAuth,
|
||||
deviceIdentity,
|
||||
meetingTranscripts,
|
||||
restartSentinel,
|
||||
pluginPlans,
|
||||
];
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
|
||||
import { getNodeSqliteKysely } from "./kysely-sync.js";
|
||||
|
||||
type MeetingTranscriptMigrationDatabase = Pick<
|
||||
OpenClawStateKyselyDatabase,
|
||||
| "meeting_transcript_sessions"
|
||||
| "meeting_transcript_summaries"
|
||||
| "meeting_transcript_utterances"
|
||||
| "migration_runs"
|
||||
| "migration_sources"
|
||||
>;
|
||||
|
||||
export function migrationDb(db: DatabaseSync) {
|
||||
return getNodeSqliteKysely<MeetingTranscriptMigrationDatabase>(db);
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
// Doctor detection for legacy meeting transcript files and interrupted imports.
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js";
|
||||
import {
|
||||
hasMatchingRecordedTranscriptArtifact,
|
||||
isRecordedCanonicalTranscriptExport,
|
||||
} from "./state-migrations.meeting-transcripts-files.js";
|
||||
import type { LegacyMeetingTranscriptsDetection } from "./state-migrations.meeting-transcripts.types.js";
|
||||
|
||||
type MeetingTranscriptExportOwnership = {
|
||||
selector: string;
|
||||
sessionId: string;
|
||||
startedAt: string;
|
||||
manifest: Record<string, string>;
|
||||
pending: ReadonlySet<string>;
|
||||
};
|
||||
|
||||
type MeetingTranscriptMigrationDetectionState = {
|
||||
exportOwnership: Map<string, MeetingTranscriptExportOwnership>;
|
||||
exportOwnershipByFoldedSelector: Map<string, MeetingTranscriptExportOwnership[]>;
|
||||
pendingImportCount: number;
|
||||
};
|
||||
|
||||
const TRANSCRIPT_ARTIFACT_NAMES = new Set([
|
||||
"metadata.json",
|
||||
"summary.json",
|
||||
"summary.md",
|
||||
"transcript.jsonl",
|
||||
]);
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||
}
|
||||
|
||||
function hasLegacyArtifactsSync(directory: string): boolean {
|
||||
const entries = fs.readdirSync(directory, { withFileTypes: true });
|
||||
let found = false;
|
||||
for (const entry of entries) {
|
||||
if (!TRANSCRIPT_ARTIFACT_NAMES.has(entry.name.toLowerCase())) {
|
||||
continue;
|
||||
}
|
||||
const stat = fs.lstatSync(path.join(directory, entry.name));
|
||||
if (stat.isSymbolicLink() || !stat.isFile()) {
|
||||
throw new Error(`legacy transcript source must be a regular file: ${directory}`);
|
||||
}
|
||||
found = true;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
export function resolveMeetingTranscriptExportOwnership(params: {
|
||||
state: MeetingTranscriptMigrationDetectionState;
|
||||
selector: string;
|
||||
sessionDir: string;
|
||||
sourceRoot: string;
|
||||
}): MeetingTranscriptExportOwnership | undefined {
|
||||
const exact = params.state.exportOwnership.get(params.selector);
|
||||
if (exact) {
|
||||
return exact;
|
||||
}
|
||||
const folded = params.state.exportOwnershipByFoldedSelector.get(params.selector.toLowerCase());
|
||||
if (!folded || folded.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const metadataEntries = fs
|
||||
.readdirSync(params.sessionDir, { withFileTypes: true })
|
||||
.filter((entry) => entry.name.toLowerCase() === "metadata.json");
|
||||
if (metadataEntries.length > 0) {
|
||||
if (metadataEntries.length !== 1) {
|
||||
return undefined;
|
||||
}
|
||||
const metadataPath = path.join(params.sessionDir, metadataEntries[0]!.name);
|
||||
const stat = fs.lstatSync(metadataPath);
|
||||
if (stat.isSymbolicLink() || !stat.isFile()) {
|
||||
return undefined;
|
||||
}
|
||||
const metadata = JSON.parse(fs.readFileSync(metadataPath, "utf8")) as {
|
||||
sessionId?: unknown;
|
||||
startedAt?: unknown;
|
||||
};
|
||||
const matches = folded.filter(
|
||||
(ownership) =>
|
||||
metadata.sessionId === ownership.sessionId && metadata.startedAt === ownership.startedAt,
|
||||
);
|
||||
return matches.length === 1 ? matches[0] : undefined;
|
||||
}
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
const manifestMatches = folded.filter((ownership) => {
|
||||
try {
|
||||
const canonicalDir = path.join(params.sourceRoot, ownership.selector);
|
||||
const canonicalStat = fs.statSync(canonicalDir);
|
||||
const observedStat = fs.statSync(params.sessionDir);
|
||||
if (canonicalStat.dev !== observedStat.dev || canonicalStat.ino !== observedStat.ino) {
|
||||
return false;
|
||||
}
|
||||
return hasMatchingRecordedTranscriptArtifact({
|
||||
sessionDir: params.sessionDir,
|
||||
manifest: ownership.manifest,
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
return manifestMatches.length === 1 ? manifestMatches[0] : undefined;
|
||||
}
|
||||
|
||||
export function detectLegacyMeetingTranscripts(params: {
|
||||
stateDir: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
doctorOnlyStateMigrations?: boolean;
|
||||
}): LegacyMeetingTranscriptsDetection {
|
||||
const sourceDir = path.join(params.stateDir, "transcripts");
|
||||
if (params.doctorOnlyStateMigrations !== true) {
|
||||
return { sourceDir, hasLegacy: false, pendingImportCount: 0 };
|
||||
}
|
||||
const databaseState = readMeetingTranscriptMigrationDetectionState({
|
||||
env: { ...(params.env ?? process.env), OPENCLAW_STATE_DIR: params.stateDir },
|
||||
});
|
||||
const pendingImportCount = databaseState.pendingImportCount;
|
||||
try {
|
||||
const rootStat = fs.lstatSync(sourceDir);
|
||||
if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) {
|
||||
throw new Error(`meeting transcript root must be a regular directory: ${sourceDir}`);
|
||||
}
|
||||
const dateEntries = fs.readdirSync(sourceDir, { withFileTypes: true });
|
||||
const sourceSelectors = hasLegacyArtifactsSync(sourceDir) ? ["."] : [];
|
||||
for (const dateEntry of dateEntries) {
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(dateEntry.name)) {
|
||||
continue;
|
||||
}
|
||||
if (dateEntry.isSymbolicLink()) {
|
||||
throw new Error(`legacy transcript date directory cannot be a symlink: ${dateEntry.name}`);
|
||||
}
|
||||
if (!dateEntry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
const dateDir = path.join(sourceDir, dateEntry.name);
|
||||
if (hasLegacyArtifactsSync(dateDir)) {
|
||||
sourceSelectors.push(dateEntry.name);
|
||||
}
|
||||
for (const entry of fs.readdirSync(dateDir, { withFileTypes: true })) {
|
||||
if (entry.isSymbolicLink()) {
|
||||
throw new Error(`legacy transcript session cannot be a symlink: ${entry.name}`);
|
||||
}
|
||||
if (entry.isDirectory() && hasLegacyArtifactsSync(path.join(dateDir, entry.name))) {
|
||||
sourceSelectors.push(`${dateEntry.name}/${entry.name}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
const hasSource = sourceSelectors.some((selector) => {
|
||||
const ownership = resolveMeetingTranscriptExportOwnership({
|
||||
state: databaseState,
|
||||
selector,
|
||||
sessionDir: path.join(sourceDir, selector),
|
||||
sourceRoot: sourceDir,
|
||||
});
|
||||
return (
|
||||
!ownership ||
|
||||
!isRecordedCanonicalTranscriptExport({
|
||||
sessionDir: path.join(sourceDir, selector),
|
||||
manifest: ownership.manifest,
|
||||
pending: ownership.pending,
|
||||
})
|
||||
);
|
||||
});
|
||||
return {
|
||||
sourceDir,
|
||||
hasLegacy: hasSource || pendingImportCount > 0,
|
||||
pendingImportCount,
|
||||
};
|
||||
} catch (error) {
|
||||
if (isRecord(error) && error.code === "ENOENT") {
|
||||
return { sourceDir, hasLegacy: pendingImportCount > 0, pendingImportCount };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function readMeetingTranscriptMigrationDetectionState(params: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
}): MeetingTranscriptMigrationDetectionState {
|
||||
const databasePath = resolveOpenClawStateSqlitePath(params.env);
|
||||
if (!fs.existsSync(databasePath)) {
|
||||
return {
|
||||
exportOwnership: new Map(),
|
||||
exportOwnershipByFoldedSelector: new Map(),
|
||||
pendingImportCount: 0,
|
||||
};
|
||||
}
|
||||
const database = new DatabaseSync(databasePath, { readOnly: true });
|
||||
try {
|
||||
const tables = new Set(
|
||||
database
|
||||
.prepare(
|
||||
"SELECT name FROM sqlite_schema WHERE type = 'table' AND name IN ('meeting_transcript_sessions', 'migration_sources')",
|
||||
)
|
||||
.all()
|
||||
.map((row) => String(row.name)),
|
||||
);
|
||||
const exportOwnership = new Map<string, MeetingTranscriptExportOwnership>();
|
||||
const exportOwnershipByFoldedSelector = new Map<string, MeetingTranscriptExportOwnership[]>();
|
||||
if (tables.has("meeting_transcript_sessions")) {
|
||||
const rows = database
|
||||
.prepare(
|
||||
"SELECT session_id, started_at, selector, export_manifest_json, export_pending_json FROM meeting_transcript_sessions",
|
||||
)
|
||||
.all();
|
||||
for (const row of rows) {
|
||||
const selector = String(row.selector);
|
||||
const parsed = JSON.parse(String(row.export_manifest_json)) as unknown;
|
||||
if (isRecord(parsed)) {
|
||||
const ownership = {
|
||||
selector,
|
||||
sessionId: String(row.session_id),
|
||||
startedAt: String(row.started_at),
|
||||
manifest: parsed as Record<string, string>,
|
||||
pending: new Set(JSON.parse(String(row.export_pending_json)) as string[]),
|
||||
} satisfies MeetingTranscriptExportOwnership;
|
||||
exportOwnership.set(selector, ownership);
|
||||
const foldedSelector = selector.toLowerCase();
|
||||
const foldedOwners = exportOwnershipByFoldedSelector.get(foldedSelector) ?? [];
|
||||
foldedOwners.push(ownership);
|
||||
exportOwnershipByFoldedSelector.set(foldedSelector, foldedOwners);
|
||||
}
|
||||
}
|
||||
}
|
||||
const pendingRow = tables.has("migration_sources")
|
||||
? (database
|
||||
.prepare(
|
||||
"SELECT COUNT(*) AS count FROM migration_sources WHERE migration_kind = ? AND status = ? AND removed_source = 0",
|
||||
)
|
||||
.get("meeting-transcripts-files-v1", "imported") as { count?: unknown } | undefined)
|
||||
: undefined;
|
||||
return {
|
||||
exportOwnership,
|
||||
exportOwnershipByFoldedSelector,
|
||||
pendingImportCount: typeof pendingRow?.count === "number" ? pendingRow.count : 0,
|
||||
};
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,714 @@
|
||||
// Filesystem preflight and archive helpers for legacy meeting transcripts.
|
||||
import { createHash } from "node:crypto";
|
||||
import fsSync, { createReadStream } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import type {
|
||||
TranscriptSessionDescriptor,
|
||||
TranscriptUtterance,
|
||||
} from "../transcripts/provider-types.js";
|
||||
import type { TranscriptsSummary } from "../transcripts/summary.js";
|
||||
import { renderTranscriptsMarkdown } from "../transcripts/summary.js";
|
||||
import { sha256File, sha256Hex } from "./crypto-digest.js";
|
||||
import { assertNoSymlinkParents } from "./fs-safe-advanced.js";
|
||||
|
||||
const TRANSCRIPT_EXPORT_FILE_NAMES = new Set([
|
||||
"metadata.json",
|
||||
"summary.json",
|
||||
"summary.md",
|
||||
"transcript.jsonl",
|
||||
]);
|
||||
|
||||
export const LEGACY_UTTERANCE_INSERT_CHUNK_SIZE = 64;
|
||||
const LEGACY_UTTERANCE_STAGE_BATCH_SIZE = 256;
|
||||
|
||||
export type LegacyMeetingTranscriptSnapshot = {
|
||||
sourceDir: string;
|
||||
relativeDir: string;
|
||||
stageKey: string;
|
||||
session: TranscriptSessionDescriptor;
|
||||
utteranceCount: number;
|
||||
summary?: TranscriptsSummary;
|
||||
markdown?: string;
|
||||
sourceHash: string;
|
||||
sourceSizeBytes: number;
|
||||
};
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||
}
|
||||
|
||||
function sha256FileSync(filePath: string): string {
|
||||
const digest = createHash("sha256");
|
||||
const descriptor = fsSync.openSync(filePath, "r");
|
||||
const buffer = Buffer.allocUnsafe(64 * 1024);
|
||||
try {
|
||||
while (true) {
|
||||
const bytesRead = fsSync.readSync(descriptor, buffer, 0, buffer.length, null);
|
||||
if (bytesRead === 0) {
|
||||
break;
|
||||
}
|
||||
digest.update(buffer.subarray(0, bytesRead));
|
||||
}
|
||||
} finally {
|
||||
fsSync.closeSync(descriptor);
|
||||
}
|
||||
return digest.digest("hex");
|
||||
}
|
||||
|
||||
export function isRecordedCanonicalTranscriptExport(params: {
|
||||
sessionDir: string;
|
||||
manifest: Readonly<Record<string, string>>;
|
||||
pending?: ReadonlySet<string>;
|
||||
}): boolean {
|
||||
const entries = fsSync.readdirSync(params.sessionDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const canonicalName = entry.name.toLowerCase();
|
||||
if (!TRANSCRIPT_EXPORT_FILE_NAMES.has(canonicalName)) {
|
||||
continue;
|
||||
}
|
||||
const filePath = path.join(params.sessionDir, entry.name);
|
||||
const stat = fsSync.lstatSync(filePath);
|
||||
if (stat.isSymbolicLink() || !stat.isFile()) {
|
||||
return false;
|
||||
}
|
||||
const expectedHash = params.manifest[canonicalName];
|
||||
if (
|
||||
params.pending?.has(canonicalName) !== true &&
|
||||
(!expectedHash || sha256FileSync(filePath) !== expectedHash)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function hasMatchingRecordedTranscriptArtifact(params: {
|
||||
sessionDir: string;
|
||||
manifest: Readonly<Record<string, string>>;
|
||||
}): boolean {
|
||||
for (const entry of fsSync.readdirSync(params.sessionDir, { withFileTypes: true })) {
|
||||
const canonicalName = entry.name.toLowerCase();
|
||||
const expectedHash = params.manifest[canonicalName];
|
||||
if (!TRANSCRIPT_EXPORT_FILE_NAMES.has(canonicalName) || !expectedHash) {
|
||||
continue;
|
||||
}
|
||||
const filePath = path.join(params.sessionDir, entry.name);
|
||||
const stat = fsSync.lstatSync(filePath);
|
||||
if (!stat.isSymbolicLink() && stat.isFile() && sha256FileSync(filePath) === expectedHash) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function validateMeetingTranscriptRoot(
|
||||
rootDir: string,
|
||||
options: { allowMissing?: boolean } = {},
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
const stat = await fs.lstat(rootDir);
|
||||
if (stat.isSymbolicLink() || !stat.isDirectory()) {
|
||||
throw new Error(`meeting transcript root must be a regular directory: ${rootDir}`);
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isRecord(error) && error.code === "ENOENT" && options.allowMissing === true) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function parseSession(value: unknown, sourcePath: string): TranscriptSessionDescriptor {
|
||||
if (!isRecord(value) || typeof value.sessionId !== "string" || !value.sessionId) {
|
||||
throw new Error(`invalid transcripts metadata sessionId at ${sourcePath}`);
|
||||
}
|
||||
if (typeof value.startedAt !== "string" || !value.startedAt) {
|
||||
throw new Error(`invalid transcripts metadata startedAt at ${sourcePath}`);
|
||||
}
|
||||
if (!isRecord(value.source) || typeof value.source.providerId !== "string") {
|
||||
throw new Error(`invalid transcripts metadata source at ${sourcePath}`);
|
||||
}
|
||||
if (value.title !== undefined && typeof value.title !== "string") {
|
||||
throw new Error(`invalid transcripts metadata title at ${sourcePath}`);
|
||||
}
|
||||
if (value.stoppedAt !== undefined && typeof value.stoppedAt !== "string") {
|
||||
throw new Error(`invalid transcripts metadata stoppedAt at ${sourcePath}`);
|
||||
}
|
||||
if (value.metadata !== undefined && !isRecord(value.metadata)) {
|
||||
throw new Error(`invalid transcripts metadata payload at ${sourcePath}`);
|
||||
}
|
||||
return value as TranscriptSessionDescriptor;
|
||||
}
|
||||
|
||||
function parseUtterance(
|
||||
value: unknown,
|
||||
sourcePath: string,
|
||||
lineNumber: number,
|
||||
): TranscriptUtterance {
|
||||
if (!isRecord(value) || typeof value.text !== "string") {
|
||||
throw new Error(`invalid transcript utterance at ${sourcePath}:${lineNumber}`);
|
||||
}
|
||||
if (
|
||||
value.speaker !== undefined &&
|
||||
(!isRecord(value.speaker) || typeof value.speaker.label !== "string")
|
||||
) {
|
||||
throw new Error(`invalid transcript speaker at ${sourcePath}:${lineNumber}`);
|
||||
}
|
||||
return value as TranscriptUtterance;
|
||||
}
|
||||
|
||||
function parseSummary(value: unknown, sourcePath: string): TranscriptsSummary {
|
||||
if (
|
||||
!isRecord(value) ||
|
||||
typeof value.sessionId !== "string" ||
|
||||
typeof value.title !== "string" ||
|
||||
typeof value.generatedAt !== "string" ||
|
||||
typeof value.overview !== "string" ||
|
||||
!Array.isArray(value.transcript) ||
|
||||
!Array.isArray(value.decisions) ||
|
||||
!Array.isArray(value.actionItems) ||
|
||||
!Array.isArray(value.risks) ||
|
||||
!Number.isSafeInteger(value.utteranceCount) ||
|
||||
(value.utteranceCount as number) < 0
|
||||
) {
|
||||
throw new Error(`invalid transcripts summary at ${sourcePath}`);
|
||||
}
|
||||
return value as unknown as TranscriptsSummary;
|
||||
}
|
||||
|
||||
function legacyTranscriptRelativeDir(session: TranscriptSessionDescriptor): string {
|
||||
const date = session.startedAt.match(/^(\d{4}-\d{2}-\d{2})T/)?.[1];
|
||||
if (!date) {
|
||||
throw new Error(`legacy transcript startedAt has no date: ${session.startedAt}`);
|
||||
}
|
||||
const legacySegment =
|
||||
session.sessionId.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "session";
|
||||
return path.normalize(path.join(date, legacySegment));
|
||||
}
|
||||
|
||||
async function optionalRegularFile(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
const stat = await fs.lstat(filePath);
|
||||
if (stat.isSymbolicLink() || !stat.isFile()) {
|
||||
throw new Error(`legacy transcript source must be a regular file: ${filePath}`);
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (isRecord(error) && error.code === "ENOENT") {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function openLegacyMeetingTranscriptStage(databasePath: string): DatabaseSync {
|
||||
const database = new DatabaseSync(databasePath);
|
||||
database.exec(`
|
||||
PRAGMA journal_mode = WAL;
|
||||
CREATE TABLE staged_utterances (
|
||||
stage_key TEXT NOT NULL,
|
||||
sequence INTEGER NOT NULL,
|
||||
utterance_json TEXT NOT NULL,
|
||||
PRIMARY KEY (stage_key, sequence)
|
||||
) STRICT;
|
||||
`);
|
||||
return database;
|
||||
}
|
||||
|
||||
async function stageUtterances(params: {
|
||||
filePath: string;
|
||||
stageDatabase: DatabaseSync;
|
||||
stageKey: string;
|
||||
}): Promise<number> {
|
||||
const filePath = params.filePath;
|
||||
if (!(await optionalRegularFile(filePath))) {
|
||||
return 0;
|
||||
}
|
||||
const stream = createReadStream(filePath, { encoding: "utf8" });
|
||||
const lines = createInterface({ input: stream, crlfDelay: Infinity });
|
||||
let lineNumber = 0;
|
||||
let sequence = 0;
|
||||
let pending: string[] = [];
|
||||
const insert = params.stageDatabase.prepare(
|
||||
"INSERT INTO staged_utterances (stage_key, sequence, utterance_json) VALUES (?, ?, ?)",
|
||||
);
|
||||
const flush = () => {
|
||||
if (pending.length === 0) {
|
||||
return;
|
||||
}
|
||||
params.stageDatabase.exec("BEGIN IMMEDIATE");
|
||||
try {
|
||||
for (const utteranceJson of pending) {
|
||||
insert.run(params.stageKey, sequence, utteranceJson);
|
||||
sequence += 1;
|
||||
}
|
||||
params.stageDatabase.exec("COMMIT");
|
||||
pending = [];
|
||||
} catch (error) {
|
||||
params.stageDatabase.exec("ROLLBACK");
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
try {
|
||||
for await (const line of lines) {
|
||||
lineNumber += 1;
|
||||
if (!line.trim()) {
|
||||
continue;
|
||||
}
|
||||
const utterance = parseUtterance(JSON.parse(line) as unknown, filePath, lineNumber);
|
||||
pending.push(JSON.stringify(utterance));
|
||||
if (pending.length >= LEGACY_UTTERANCE_STAGE_BATCH_SIZE) {
|
||||
flush();
|
||||
}
|
||||
}
|
||||
flush();
|
||||
} finally {
|
||||
lines.close();
|
||||
stream.destroy();
|
||||
}
|
||||
return sequence;
|
||||
}
|
||||
|
||||
export function readStagedMeetingTranscriptUtterances(params: {
|
||||
stageDatabase: DatabaseSync;
|
||||
stageKey: string;
|
||||
start: number;
|
||||
limit: number;
|
||||
}): TranscriptUtterance[] {
|
||||
return params.stageDatabase
|
||||
.prepare(
|
||||
"SELECT utterance_json FROM staged_utterances WHERE stage_key = ? AND sequence >= ? ORDER BY sequence ASC LIMIT ?",
|
||||
)
|
||||
.all(params.stageKey, params.start, params.limit)
|
||||
.map((row) => JSON.parse(String(row.utterance_json)) as TranscriptUtterance);
|
||||
}
|
||||
|
||||
async function snapshotFile(filePath: string): Promise<{
|
||||
hash?: string;
|
||||
sizeBytes: number;
|
||||
}> {
|
||||
if (!(await optionalRegularFile(filePath))) {
|
||||
return { sizeBytes: 0 };
|
||||
}
|
||||
const stat = await fs.stat(filePath);
|
||||
return { hash: await sha256File(filePath), sizeBytes: stat.size };
|
||||
}
|
||||
|
||||
async function snapshotSourceFiles(files: string[]) {
|
||||
return await Promise.all(files.map(snapshotFile));
|
||||
}
|
||||
|
||||
function sourceFilesHash(
|
||||
files: string[],
|
||||
snapshots: Array<{ hash?: string; sizeBytes: number }>,
|
||||
): string {
|
||||
return sha256Hex(
|
||||
snapshots
|
||||
.map((snapshot, index) => `${path.basename(files[index] ?? "")}\0${snapshot.hash ?? "-"}`)
|
||||
.join("\n"),
|
||||
);
|
||||
}
|
||||
|
||||
export async function snapshotLegacyMeetingTranscriptSession(params: {
|
||||
rootDir: string;
|
||||
relativeDir: string;
|
||||
stageDatabase: DatabaseSync;
|
||||
}): Promise<LegacyMeetingTranscriptSnapshot> {
|
||||
const sourceDir = path.join(params.rootDir, params.relativeDir);
|
||||
const sourceStat = await fs.lstat(sourceDir);
|
||||
if (sourceStat.isSymbolicLink() || !sourceStat.isDirectory()) {
|
||||
throw new Error(`legacy transcript session must be a regular directory: ${sourceDir}`);
|
||||
}
|
||||
const metadataPath = path.join(sourceDir, "metadata.json");
|
||||
const transcriptPath = path.join(sourceDir, "transcript.jsonl");
|
||||
const summaryJsonPath = path.join(sourceDir, "summary.json");
|
||||
const summaryMarkdownPath = path.join(sourceDir, "summary.md");
|
||||
const files = [metadataPath, transcriptPath, summaryJsonPath, summaryMarkdownPath];
|
||||
const beforeSnapshots = await snapshotSourceFiles(files);
|
||||
if (!beforeSnapshots[0]?.hash) {
|
||||
throw new Error(`legacy transcript session is missing metadata.json: ${sourceDir}`);
|
||||
}
|
||||
const session = parseSession(JSON.parse(await fs.readFile(metadataPath, "utf8")), metadataPath);
|
||||
const expectedRelativeDir = legacyTranscriptRelativeDir(session);
|
||||
if (path.normalize(params.relativeDir) !== expectedRelativeDir) {
|
||||
throw new Error(
|
||||
`legacy transcript selector mismatch at ${sourceDir}: expected ${expectedRelativeDir}`,
|
||||
);
|
||||
}
|
||||
const utteranceCount = await stageUtterances({
|
||||
filePath: transcriptPath,
|
||||
stageDatabase: params.stageDatabase,
|
||||
stageKey: params.relativeDir,
|
||||
});
|
||||
const hasSummaryJson = await optionalRegularFile(summaryJsonPath);
|
||||
const hasSummaryMarkdown = await optionalRegularFile(summaryMarkdownPath);
|
||||
const summary = hasSummaryJson
|
||||
? parseSummary(JSON.parse(await fs.readFile(summaryJsonPath, "utf8")), summaryJsonPath)
|
||||
: undefined;
|
||||
const markdown = hasSummaryMarkdown
|
||||
? await fs.readFile(summaryMarkdownPath, "utf8")
|
||||
: summary
|
||||
? renderTranscriptsMarkdown(summary)
|
||||
: undefined;
|
||||
if (summary && summary.sessionId !== session.sessionId) {
|
||||
throw new Error(`legacy transcript summary session mismatch at ${summaryJsonPath}`);
|
||||
}
|
||||
|
||||
const fileSnapshots = await snapshotSourceFiles(files);
|
||||
if (
|
||||
fileSnapshots.some(
|
||||
(snapshot, index) =>
|
||||
snapshot.hash !== beforeSnapshots[index]?.hash ||
|
||||
snapshot.sizeBytes !== beforeSnapshots[index]?.sizeBytes,
|
||||
)
|
||||
) {
|
||||
throw new Error(`legacy transcript files changed while being staged: ${sourceDir}`);
|
||||
}
|
||||
const sourceHash = sourceFilesHash(files, fileSnapshots);
|
||||
return {
|
||||
sourceDir,
|
||||
relativeDir: params.relativeDir,
|
||||
stageKey: params.relativeDir,
|
||||
session,
|
||||
utteranceCount,
|
||||
summary,
|
||||
markdown,
|
||||
sourceHash,
|
||||
sourceSizeBytes: fileSnapshots.reduce((total, file) => total + file.sizeBytes, 0),
|
||||
};
|
||||
}
|
||||
|
||||
async function hasLegacyTranscriptArtifacts(directory: string): Promise<boolean> {
|
||||
const entries = await fs.readdir(directory, { withFileTypes: true });
|
||||
let found = false;
|
||||
for (const entry of entries) {
|
||||
if (!TRANSCRIPT_EXPORT_FILE_NAMES.has(entry.name.toLowerCase())) {
|
||||
continue;
|
||||
}
|
||||
const stat = await fs.lstat(path.join(directory, entry.name));
|
||||
if (stat.isSymbolicLink() || !stat.isFile()) {
|
||||
throw new Error(`legacy transcript source must be a regular file: ${directory}`);
|
||||
}
|
||||
found = true;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
async function listLegacyMeetingTranscriptDirs(
|
||||
rootDir: string,
|
||||
mode: "artifacts" | "sessions",
|
||||
): Promise<string[]> {
|
||||
if (!(await validateMeetingTranscriptRoot(rootDir, { allowMissing: true }))) {
|
||||
return [];
|
||||
}
|
||||
let dateEntries;
|
||||
try {
|
||||
dateEntries = await fs.readdir(rootDir, { withFileTypes: true });
|
||||
} catch (error) {
|
||||
if (isRecord(error) && error.code === "ENOENT") {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const include = async (directory: string) =>
|
||||
mode === "sessions"
|
||||
? await optionalRegularFile(path.join(directory, "metadata.json"))
|
||||
: await hasLegacyTranscriptArtifacts(directory);
|
||||
const sessions: string[] = [];
|
||||
if (await include(rootDir)) {
|
||||
sessions.push(".");
|
||||
}
|
||||
for (const dateEntry of dateEntries.toSorted((a, b) => a.name.localeCompare(b.name))) {
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(dateEntry.name)) {
|
||||
continue;
|
||||
}
|
||||
if (dateEntry.isSymbolicLink()) {
|
||||
throw new Error(`legacy transcript date directory cannot be a symlink: ${dateEntry.name}`);
|
||||
}
|
||||
if (!dateEntry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
const dateDir = path.join(rootDir, dateEntry.name);
|
||||
if (await include(dateDir)) {
|
||||
sessions.push(dateEntry.name);
|
||||
}
|
||||
const sessionEntries = await fs.readdir(dateDir, { withFileTypes: true });
|
||||
for (const sessionEntry of sessionEntries.toSorted((a, b) => a.name.localeCompare(b.name))) {
|
||||
if (sessionEntry.isSymbolicLink()) {
|
||||
throw new Error(`legacy transcript session cannot be a symlink: ${sessionEntry.name}`);
|
||||
}
|
||||
if (sessionEntry.isDirectory() && (await include(path.join(dateDir, sessionEntry.name)))) {
|
||||
sessions.push(path.join(dateEntry.name, sessionEntry.name));
|
||||
}
|
||||
}
|
||||
}
|
||||
return sessions;
|
||||
}
|
||||
|
||||
export async function listLegacyMeetingTranscriptSessionDirs(rootDir: string): Promise<string[]> {
|
||||
return await listLegacyMeetingTranscriptDirs(rootDir, "sessions");
|
||||
}
|
||||
|
||||
export async function listLegacyMeetingTranscriptArtifactDirs(rootDir: string): Promise<string[]> {
|
||||
return await listLegacyMeetingTranscriptDirs(rootDir, "artifacts");
|
||||
}
|
||||
|
||||
export async function archivePartialMeetingTranscriptArtifacts(params: {
|
||||
sourceRoot: string;
|
||||
relativeDirs: string[];
|
||||
recoveryRoot: string;
|
||||
}): Promise<void> {
|
||||
const moves: Array<{ source: string; destination: string }> = [];
|
||||
const sourceDirs = new Set<string>();
|
||||
for (const relativeDir of params.relativeDirs) {
|
||||
const sourceDir = path.join(params.sourceRoot, relativeDir);
|
||||
if (relativeDir !== ".") {
|
||||
sourceDirs.add(sourceDir);
|
||||
}
|
||||
const destinationDir = path.join(params.recoveryRoot, relativeDir);
|
||||
for (const entry of await fs.readdir(sourceDir, { withFileTypes: true })) {
|
||||
if (!TRANSCRIPT_EXPORT_FILE_NAMES.has(entry.name.toLowerCase())) {
|
||||
continue;
|
||||
}
|
||||
const source = path.join(sourceDir, entry.name);
|
||||
const stat = await fs.lstat(source);
|
||||
if (stat.isSymbolicLink() || !stat.isFile()) {
|
||||
throw new Error(`legacy transcript source must be a regular file: ${sourceDir}`);
|
||||
}
|
||||
const destination = path.join(destinationDir, entry.name);
|
||||
try {
|
||||
await fs.lstat(destination);
|
||||
throw new Error(`partial transcript recovery destination already exists: ${destination}`);
|
||||
} catch (error) {
|
||||
if (!(isRecord(error) && error.code === "ENOENT")) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
moves.push({ source, destination });
|
||||
}
|
||||
}
|
||||
for (const destinationDir of new Set(moves.map((move) => path.dirname(move.destination)))) {
|
||||
await fs.mkdir(destinationDir, { recursive: true });
|
||||
}
|
||||
const moved: Array<{ source: string; destination: string }> = [];
|
||||
try {
|
||||
for (const move of moves) {
|
||||
await fs.rename(move.source, move.destination);
|
||||
moved.push(move);
|
||||
}
|
||||
} catch (error) {
|
||||
const rollbackErrors: string[] = [];
|
||||
for (const move of moved.toReversed()) {
|
||||
try {
|
||||
await fs.rename(move.destination, move.source);
|
||||
} catch (rollbackError) {
|
||||
rollbackErrors.push(String(rollbackError));
|
||||
}
|
||||
}
|
||||
if (rollbackErrors.length > 0) {
|
||||
throw new Error(
|
||||
`partial transcript recovery failed and rollback was incomplete; inspect ${params.recoveryRoot}: ${String(error)}; rollback errors: ${rollbackErrors.join("; ")}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
// Artifact moves are already committed; empty-directory cleanup is best effort
|
||||
// so an unremovable harmless directory cannot hide the reported recovery move.
|
||||
for (const sourceDir of [...sourceDirs].toSorted((a, b) => b.length - a.length)) {
|
||||
await fs.rmdir(sourceDir).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
export async function rehashLegacyMeetingTranscriptSnapshots(
|
||||
snapshots: LegacyMeetingTranscriptSnapshot[],
|
||||
): Promise<boolean> {
|
||||
for (const snapshot of snapshots) {
|
||||
const files = ["metadata.json", "transcript.jsonl", "summary.json", "summary.md"].map(
|
||||
(fileName) => path.join(snapshot.sourceDir, fileName),
|
||||
);
|
||||
const fileSnapshots = await snapshotSourceFiles(files);
|
||||
const currentHash = sourceFilesHash(files, fileSnapshots);
|
||||
if (currentHash !== snapshot.sourceHash) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function archiveLegacyMeetingTranscriptSnapshots(params: {
|
||||
sourceRoot: string;
|
||||
snapshots: LegacyMeetingTranscriptSnapshot[];
|
||||
expectedRelativeDirs: string[];
|
||||
canonicalRelativeDirs: string[];
|
||||
archiveRoot: string;
|
||||
}): Promise<string> {
|
||||
await validateMeetingTranscriptRoot(params.sourceRoot);
|
||||
const currentRelativeDirs = await listLegacyMeetingTranscriptSessionDirs(params.sourceRoot);
|
||||
const expectedRelativeDirs = params.expectedRelativeDirs.toSorted((a, b) => a.localeCompare(b));
|
||||
if (JSON.stringify(currentRelativeDirs) !== JSON.stringify(expectedRelativeDirs)) {
|
||||
throw new Error("legacy transcript session tree changed before archive");
|
||||
}
|
||||
await fs.rename(params.sourceRoot, params.archiveRoot);
|
||||
try {
|
||||
const archivedSnapshots = params.snapshots.map((snapshot) => ({
|
||||
...snapshot,
|
||||
sourceDir: path.join(
|
||||
params.archiveRoot,
|
||||
path.relative(params.sourceRoot, snapshot.sourceDir) || ".",
|
||||
),
|
||||
}));
|
||||
if (!(await rehashLegacyMeetingTranscriptSnapshots(archivedSnapshots))) {
|
||||
throw new Error("legacy transcript files changed at the archive boundary");
|
||||
}
|
||||
await restoreCanonicalMeetingTranscriptExports({
|
||||
sourceRoot: params.sourceRoot,
|
||||
archiveRoot: params.archiveRoot,
|
||||
migratedSourcePaths: params.snapshots.map((snapshot) => snapshot.sourceDir),
|
||||
canonicalRelativeDirs: params.canonicalRelativeDirs,
|
||||
});
|
||||
} catch (error) {
|
||||
throw new LegacyMeetingTranscriptArchiveMovedError(error);
|
||||
}
|
||||
return params.archiveRoot;
|
||||
}
|
||||
|
||||
export class LegacyMeetingTranscriptArchiveMovedError extends Error {
|
||||
constructor(cause: unknown) {
|
||||
super(
|
||||
`legacy transcript source moved but canonical export restoration failed: ${String(cause)}`,
|
||||
);
|
||||
this.name = "LegacyMeetingTranscriptArchiveMovedError";
|
||||
}
|
||||
}
|
||||
|
||||
export async function restoreCanonicalMeetingTranscriptExports(params: {
|
||||
sourceRoot: string;
|
||||
archiveRoot: string;
|
||||
migratedSourcePaths: string[];
|
||||
canonicalRelativeDirs: string[];
|
||||
}): Promise<void> {
|
||||
await validateMeetingTranscriptRoot(params.archiveRoot);
|
||||
if (!(await validateMeetingTranscriptRoot(params.sourceRoot, { allowMissing: true }))) {
|
||||
await fs.mkdir(params.sourceRoot, { recursive: true });
|
||||
await validateMeetingTranscriptRoot(params.sourceRoot);
|
||||
}
|
||||
const migratedRelativeDirs = new Set(
|
||||
params.migratedSourcePaths.map(
|
||||
(sourcePath) => path.relative(params.sourceRoot, sourcePath) || ".",
|
||||
),
|
||||
);
|
||||
for (const relativeDir of params.canonicalRelativeDirs) {
|
||||
const archiveRelative = path.relative(
|
||||
path.resolve(params.archiveRoot),
|
||||
path.resolve(params.archiveRoot, relativeDir),
|
||||
);
|
||||
const sourceRelative = path.relative(
|
||||
path.resolve(params.sourceRoot),
|
||||
path.resolve(params.sourceRoot, relativeDir),
|
||||
);
|
||||
if (
|
||||
!archiveRelative ||
|
||||
archiveRelative.startsWith("..") ||
|
||||
path.isAbsolute(archiveRelative) ||
|
||||
!sourceRelative ||
|
||||
sourceRelative.startsWith("..") ||
|
||||
path.isAbsolute(sourceRelative)
|
||||
) {
|
||||
throw new Error(`canonical transcript export path escaped its root: ${relativeDir}`);
|
||||
}
|
||||
if (migratedRelativeDirs.has(relativeDir)) {
|
||||
continue;
|
||||
}
|
||||
const source = path.join(params.archiveRoot, relativeDir);
|
||||
const destination = path.join(params.sourceRoot, relativeDir);
|
||||
try {
|
||||
const sourceStat = await fs.lstat(source);
|
||||
if (sourceStat.isSymbolicLink() || !sourceStat.isDirectory()) {
|
||||
throw new Error(`canonical transcript export source is not a directory: ${source}`);
|
||||
}
|
||||
await assertNoSymlinkParents({
|
||||
rootDir: params.archiveRoot,
|
||||
targetPath: source,
|
||||
allowMissing: false,
|
||||
messagePrefix: "Canonical transcript export source",
|
||||
});
|
||||
} catch (error) {
|
||||
if (!(isRecord(error) && error.code === "ENOENT")) {
|
||||
throw error;
|
||||
}
|
||||
const destinationStat = await fs.lstat(destination);
|
||||
if (destinationStat.isSymbolicLink() || !destinationStat.isDirectory()) {
|
||||
throw new Error(
|
||||
`canonical transcript export destination is not a directory: ${destination}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
await assertNoSymlinkParents({
|
||||
rootDir: params.sourceRoot,
|
||||
targetPath: destination,
|
||||
allowMissing: false,
|
||||
messagePrefix: "Canonical transcript export destination",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const destinationStat = await fs.lstat(destination);
|
||||
if (destinationStat.isSymbolicLink() || !destinationStat.isDirectory()) {
|
||||
throw new Error(
|
||||
`canonical transcript export destination is not a directory: ${destination}`,
|
||||
);
|
||||
}
|
||||
await assertNoSymlinkParents({
|
||||
rootDir: params.sourceRoot,
|
||||
targetPath: destination,
|
||||
allowMissing: false,
|
||||
messagePrefix: "Canonical transcript export destination",
|
||||
});
|
||||
const readMetadata = async (directory: string) =>
|
||||
parseSession(
|
||||
JSON.parse(await fs.readFile(path.join(directory, "metadata.json"), "utf8")),
|
||||
path.join(directory, "metadata.json"),
|
||||
);
|
||||
const [sourceMetadata, destinationMetadata] = await Promise.all([
|
||||
readMetadata(source),
|
||||
readMetadata(destination),
|
||||
]);
|
||||
if (
|
||||
sourceMetadata.sessionId !== destinationMetadata.sessionId ||
|
||||
sourceMetadata.startedAt !== destinationMetadata.startedAt
|
||||
) {
|
||||
throw new Error(`canonical transcript export destination changed identity: ${destination}`);
|
||||
}
|
||||
continue;
|
||||
} catch (error) {
|
||||
if (!(isRecord(error) && error.code === "ENOENT")) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
await assertNoSymlinkParents({
|
||||
rootDir: params.sourceRoot,
|
||||
targetPath: destination,
|
||||
allowMissing: true,
|
||||
messagePrefix: "Canonical transcript export destination",
|
||||
});
|
||||
await fs.mkdir(path.dirname(destination), { recursive: true });
|
||||
await fs.rename(source, destination);
|
||||
}
|
||||
}
|
||||
|
||||
export async function archiveDivergentMeetingTranscriptExport(params: {
|
||||
sourceRoot: string;
|
||||
relativeDir: string;
|
||||
recoveryRoot: string;
|
||||
}): Promise<string> {
|
||||
const source = path.join(params.sourceRoot, params.relativeDir);
|
||||
const destination = path.join(params.recoveryRoot, params.relativeDir);
|
||||
await fs.mkdir(path.dirname(destination), { recursive: true });
|
||||
await fs.rename(source, destination);
|
||||
return destination;
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { runOpenClawStateWriteTransaction } from "../state/openclaw-state-db.js";
|
||||
import {
|
||||
safeTranscriptPathSegment,
|
||||
transcriptSessionExportKey,
|
||||
transcriptSessionSelector,
|
||||
} from "../transcripts/store.js";
|
||||
import { sha256Hex } from "./crypto-digest.js";
|
||||
import { executeSqliteQuerySync } from "./kysely-sync.js";
|
||||
import { migrationDb } from "./state-migrations.meeting-transcripts-database.js";
|
||||
import {
|
||||
LEGACY_UTTERANCE_INSERT_CHUNK_SIZE,
|
||||
readStagedMeetingTranscriptUtterances,
|
||||
type LegacyMeetingTranscriptSnapshot,
|
||||
} from "./state-migrations.meeting-transcripts-files.js";
|
||||
|
||||
function sourceKey(sourceDir: string): string {
|
||||
return `meeting-transcripts:${sha256Hex(path.resolve(sourceDir))}`;
|
||||
}
|
||||
|
||||
export function insertMeetingTranscriptSnapshots(params: {
|
||||
snapshots: LegacyMeetingTranscriptSnapshot[];
|
||||
runId: string;
|
||||
now: number;
|
||||
archiveRoot: string;
|
||||
canonicalRelativeDirs: string[];
|
||||
stageDatabase: DatabaseSync;
|
||||
env: NodeJS.ProcessEnv;
|
||||
stateDir: string;
|
||||
}): void {
|
||||
runOpenClawStateWriteTransaction(
|
||||
({ db: database }) => {
|
||||
const db = migrationDb(database);
|
||||
// Run-wide metadata is stored once here; per-source receipts below keep
|
||||
// only their selector so large migrations remain linear in session count.
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db.insertInto("migration_runs").values({
|
||||
id: params.runId,
|
||||
started_at: params.now,
|
||||
finished_at: null,
|
||||
status: "imported",
|
||||
report_json: JSON.stringify({
|
||||
format: "meeting-transcripts-files-v1",
|
||||
sessions: params.snapshots.length,
|
||||
utterances: params.snapshots.reduce(
|
||||
(total, snapshot) => total + snapshot.utteranceCount,
|
||||
0,
|
||||
),
|
||||
archiveRoot: params.archiveRoot,
|
||||
canonicalRelativeDirs: params.canonicalRelativeDirs,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
for (const snapshot of params.snapshots) {
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db.insertInto("meeting_transcript_sessions").values({
|
||||
session_id: snapshot.session.sessionId,
|
||||
started_at: snapshot.session.startedAt,
|
||||
selector: transcriptSessionSelector(snapshot.session),
|
||||
export_key: transcriptSessionExportKey(snapshot.session),
|
||||
session_slug: safeTranscriptPathSegment(snapshot.session.sessionId),
|
||||
provider_id: snapshot.session.source.providerId,
|
||||
title: snapshot.session.title ?? null,
|
||||
source_json: JSON.stringify(snapshot.session.source),
|
||||
stopped_at: snapshot.session.stoppedAt ?? null,
|
||||
metadata_json: snapshot.session.metadata
|
||||
? JSON.stringify(snapshot.session.metadata)
|
||||
: null,
|
||||
export_manifest_json: "{}",
|
||||
export_pending_json: "[]",
|
||||
next_utterance_seq: snapshot.utteranceCount,
|
||||
created_at_ms: params.now,
|
||||
updated_at_ms: params.now,
|
||||
}),
|
||||
);
|
||||
if (snapshot.utteranceCount > 0) {
|
||||
for (
|
||||
let start = 0;
|
||||
start < snapshot.utteranceCount;
|
||||
start += LEGACY_UTTERANCE_INSERT_CHUNK_SIZE
|
||||
) {
|
||||
const chunk = readStagedMeetingTranscriptUtterances({
|
||||
stageDatabase: params.stageDatabase,
|
||||
stageKey: snapshot.stageKey,
|
||||
start,
|
||||
limit: LEGACY_UTTERANCE_INSERT_CHUNK_SIZE,
|
||||
});
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db.insertInto("meeting_transcript_utterances").values(
|
||||
chunk.map((utterance, offset) => ({
|
||||
session_id: snapshot.session.sessionId,
|
||||
session_started_at: snapshot.session.startedAt,
|
||||
sequence: start + offset,
|
||||
utterance_id: utterance.id ?? null,
|
||||
started_at: utterance.startedAt ?? null,
|
||||
ended_at: utterance.endedAt ?? null,
|
||||
speaker_id: utterance.speaker?.id ?? null,
|
||||
speaker_label: utterance.speaker?.label ?? null,
|
||||
text: utterance.text,
|
||||
final: utterance.final === undefined ? null : utterance.final ? 1 : 0,
|
||||
metadata_json: utterance.metadata ? JSON.stringify(utterance.metadata) : null,
|
||||
})),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (snapshot.summary !== undefined || snapshot.markdown !== undefined) {
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db.insertInto("meeting_transcript_summaries").values({
|
||||
session_id: snapshot.session.sessionId,
|
||||
session_started_at: snapshot.session.startedAt,
|
||||
generated_at: snapshot.summary?.generatedAt ?? null,
|
||||
summary_json: snapshot.summary ? JSON.stringify(snapshot.summary) : null,
|
||||
markdown: snapshot.markdown ?? null,
|
||||
utterance_count: snapshot.summary?.utteranceCount ?? snapshot.utteranceCount,
|
||||
}),
|
||||
);
|
||||
}
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db
|
||||
.insertInto("migration_sources")
|
||||
.values({
|
||||
source_key: sourceKey(snapshot.sourceDir),
|
||||
migration_kind: "meeting-transcripts-files-v1",
|
||||
source_path: snapshot.sourceDir,
|
||||
target_table: "meeting_transcript_sessions",
|
||||
source_sha256: snapshot.sourceHash,
|
||||
source_size_bytes: snapshot.sourceSizeBytes,
|
||||
source_record_count: snapshot.utteranceCount,
|
||||
last_run_id: params.runId,
|
||||
status: "imported",
|
||||
imported_at: params.now,
|
||||
removed_source: 0,
|
||||
report_json: JSON.stringify({
|
||||
selector: transcriptSessionSelector(snapshot.session),
|
||||
}),
|
||||
})
|
||||
.onConflict((conflict) =>
|
||||
conflict.column("source_key").doUpdateSet({
|
||||
source_sha256: snapshot.sourceHash,
|
||||
source_size_bytes: snapshot.sourceSizeBytes,
|
||||
source_record_count: snapshot.utteranceCount,
|
||||
last_run_id: params.runId,
|
||||
status: "imported",
|
||||
imported_at: params.now,
|
||||
removed_source: 0,
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
},
|
||||
{ env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir } },
|
||||
{ operationLabel: "meeting-transcripts.legacy-import" },
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
// Verifies staged legacy transcript rows against the committed canonical store.
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { TranscriptUtterance } from "../transcripts/provider-types.js";
|
||||
import { transcriptSessionSelector, TranscriptsStore } from "../transcripts/store.js";
|
||||
import {
|
||||
LEGACY_UTTERANCE_INSERT_CHUNK_SIZE,
|
||||
readStagedMeetingTranscriptUtterances,
|
||||
type LegacyMeetingTranscriptSnapshot,
|
||||
} from "./state-migrations.meeting-transcripts-files.js";
|
||||
|
||||
type StoredUtteranceRow = {
|
||||
ended_at: string | null;
|
||||
final: number | null;
|
||||
metadata_json: string | null;
|
||||
session_id: string;
|
||||
speaker_id: string | null;
|
||||
speaker_label: string | null;
|
||||
started_at: string | null;
|
||||
text: string;
|
||||
utterance_id: string | null;
|
||||
};
|
||||
|
||||
function canonicalJson(value: unknown): string {
|
||||
if (value === undefined) {
|
||||
return "undefined";
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return `[${value.map(canonicalJson).join(",")}]`;
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
const record = value as Record<string, unknown>;
|
||||
return `{${Object.keys(record)
|
||||
.toSorted()
|
||||
.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value) ?? "undefined";
|
||||
}
|
||||
|
||||
function storedUtteranceFromRow(row: StoredUtteranceRow): TranscriptUtterance {
|
||||
const utterance: TranscriptUtterance = { sessionId: row.session_id, text: row.text };
|
||||
if (row.utterance_id !== null) {
|
||||
utterance.id = row.utterance_id;
|
||||
}
|
||||
if (row.started_at !== null) {
|
||||
utterance.startedAt = row.started_at;
|
||||
}
|
||||
if (row.ended_at !== null) {
|
||||
utterance.endedAt = row.ended_at;
|
||||
}
|
||||
if (row.speaker_label !== null) {
|
||||
const speaker: NonNullable<TranscriptUtterance["speaker"]> = { label: row.speaker_label };
|
||||
if (row.speaker_id !== null) {
|
||||
speaker.id = row.speaker_id;
|
||||
}
|
||||
utterance.speaker = speaker;
|
||||
}
|
||||
if (row.final !== null) {
|
||||
utterance.final = row.final === 1;
|
||||
}
|
||||
if (row.metadata_json) {
|
||||
utterance.metadata = JSON.parse(row.metadata_json) as Record<string, unknown>;
|
||||
}
|
||||
return utterance;
|
||||
}
|
||||
|
||||
export async function verifyImportedMeetingTranscriptSnapshots(params: {
|
||||
store: TranscriptsStore;
|
||||
snapshots: LegacyMeetingTranscriptSnapshot[];
|
||||
stageDatabase: DatabaseSync;
|
||||
database: DatabaseSync;
|
||||
}): Promise<void> {
|
||||
const selectUtterances = params.database.prepare(`
|
||||
SELECT
|
||||
ended_at,
|
||||
final,
|
||||
metadata_json,
|
||||
session_id,
|
||||
speaker_id,
|
||||
speaker_label,
|
||||
started_at,
|
||||
text,
|
||||
utterance_id
|
||||
FROM meeting_transcript_utterances
|
||||
WHERE session_id = ? AND session_started_at = ?
|
||||
ORDER BY sequence ASC
|
||||
LIMIT ? OFFSET ?
|
||||
`);
|
||||
for (const snapshot of params.snapshots) {
|
||||
const session = await params.store.readSession(transcriptSessionSelector(snapshot.session));
|
||||
if (!session || canonicalJson(session) !== canonicalJson(snapshot.session)) {
|
||||
throw new Error(`meeting transcript import verification failed: ${snapshot.relativeDir}`);
|
||||
}
|
||||
for (
|
||||
let start = 0;
|
||||
start < snapshot.utteranceCount;
|
||||
start += LEGACY_UTTERANCE_INSERT_CHUNK_SIZE
|
||||
) {
|
||||
const expected = readStagedMeetingTranscriptUtterances({
|
||||
stageDatabase: params.stageDatabase,
|
||||
stageKey: snapshot.stageKey,
|
||||
start,
|
||||
limit: LEGACY_UTTERANCE_INSERT_CHUNK_SIZE,
|
||||
});
|
||||
const actual = selectUtterances
|
||||
.all(
|
||||
snapshot.session.sessionId,
|
||||
snapshot.session.startedAt,
|
||||
LEGACY_UTTERANCE_INSERT_CHUNK_SIZE,
|
||||
start,
|
||||
)
|
||||
.map((row) => storedUtteranceFromRow(row as StoredUtteranceRow));
|
||||
if (canonicalJson(actual) !== canonicalJson(expected)) {
|
||||
throw new Error(`meeting transcript import verification failed: ${snapshot.relativeDir}`);
|
||||
}
|
||||
}
|
||||
const summary = await params.store.readSummary(session);
|
||||
if (
|
||||
canonicalJson(summary.summary) !== canonicalJson(snapshot.summary) ||
|
||||
canonicalJson(summary.markdown?.trimEnd()) !== canonicalJson(snapshot.markdown?.trimEnd())
|
||||
) {
|
||||
throw new Error(`meeting transcript summary verification failed: ${snapshot.relativeDir}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,995 @@
|
||||
import fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import { TranscriptsStore } from "../transcripts/store.js";
|
||||
import { summarizeTranscripts } from "../transcripts/summary.js";
|
||||
import { restoreCanonicalMeetingTranscriptExports } from "./state-migrations.meeting-transcripts-files.js";
|
||||
import {
|
||||
detectLegacyMeetingTranscripts,
|
||||
migrateLegacyMeetingTranscripts,
|
||||
} from "./state-migrations.meeting-transcripts.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
afterEach(() => closeOpenClawStateDatabaseForTest());
|
||||
|
||||
async function seedLegacySession(params: {
|
||||
stateDir: string;
|
||||
sessionId: string;
|
||||
date?: string;
|
||||
invalidTranscript?: boolean;
|
||||
utteranceCount?: number;
|
||||
emptyMarkdown?: boolean;
|
||||
omitSummaryJson?: boolean;
|
||||
}): Promise<string> {
|
||||
const date = params.date ?? "2026-07-01";
|
||||
const sessionDir = path.join(params.stateDir, "transcripts", date, params.sessionId);
|
||||
await fs.mkdir(sessionDir, { recursive: true });
|
||||
const session = {
|
||||
sessionId: params.sessionId,
|
||||
title: "Design review",
|
||||
source: { providerId: "manual-transcript", meetingUrl: "https://meet.example.invalid/room" },
|
||||
startedAt: `${date}T10:00:00.000Z`,
|
||||
stoppedAt: `${date}T10:30:00.000Z`,
|
||||
};
|
||||
await fs.writeFile(
|
||||
path.join(sessionDir, "metadata.json"),
|
||||
`${JSON.stringify(session, null, 2)}\n`,
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(sessionDir, "transcript.jsonl"),
|
||||
params.invalidTranscript
|
||||
? "{invalid\n"
|
||||
: Array.from({ length: params.utteranceCount ?? 2 }, (_, index) =>
|
||||
JSON.stringify({
|
||||
id: `u-${index + 1}`,
|
||||
sessionId: params.sessionId,
|
||||
speaker: { label: index % 2 === 0 ? "Alex" : "Sam" },
|
||||
text: index === 0 ? "First line" : index === 1 ? "Second line" : `Line ${index + 1}`,
|
||||
final: true,
|
||||
}),
|
||||
).join("\n") + "\n",
|
||||
);
|
||||
const summary = {
|
||||
sessionId: params.sessionId,
|
||||
title: "Design review",
|
||||
generatedAt: `${date}T10:31:00.000Z`,
|
||||
overview: "First line. Second line.",
|
||||
transcript: ["Alex: First line", "Sam: Second line"],
|
||||
decisions: [],
|
||||
actionItems: [],
|
||||
risks: [],
|
||||
utteranceCount: params.utteranceCount ?? 2,
|
||||
};
|
||||
if (!params.omitSummaryJson) {
|
||||
await fs.writeFile(
|
||||
path.join(sessionDir, "summary.json"),
|
||||
`${JSON.stringify(summary, null, 2)}\n`,
|
||||
);
|
||||
}
|
||||
await fs.writeFile(
|
||||
path.join(sessionDir, "summary.md"),
|
||||
params.emptyMarkdown ? "" : "# Design review\n\nFirst line.\n",
|
||||
);
|
||||
return sessionDir;
|
||||
}
|
||||
|
||||
function databaseEnv(stateDir: string): NodeJS.ProcessEnv {
|
||||
return { ...process.env, OPENCLAW_STATE_DIR: stateDir };
|
||||
}
|
||||
|
||||
describe("meeting transcript Doctor migration", () => {
|
||||
it("is doctor-only", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
await seedLegacySession({ stateDir, sessionId: "design-review" });
|
||||
|
||||
expect(detectLegacyMeetingTranscripts({ stateDir })).toMatchObject({ hasLegacy: false });
|
||||
expect(
|
||||
detectLegacyMeetingTranscripts({ stateDir, doctorOnlyStateMigrations: true }),
|
||||
).toMatchObject({ hasLegacy: true });
|
||||
});
|
||||
|
||||
it("surfaces filesystem errors during detection", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
await fs.writeFile(path.join(stateDir, "transcripts"), "not a directory");
|
||||
|
||||
expect(() =>
|
||||
detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"rejects a symlinked transcript root before migration",
|
||||
async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const externalRoot = tempDirs.make("openclaw-meeting-transcripts-external-");
|
||||
await fs.symlink(externalRoot, path.join(stateDir, "transcripts"), "dir");
|
||||
|
||||
expect(() =>
|
||||
detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
}),
|
||||
).toThrow("regular directory");
|
||||
},
|
||||
);
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"rejects symlinked date and session directories during detection",
|
||||
async () => {
|
||||
for (const target of ["date", "session"] as const) {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const externalRoot = tempDirs.make("openclaw-meeting-transcripts-external-");
|
||||
const transcriptsDir = path.join(stateDir, "transcripts");
|
||||
await fs.mkdir(transcriptsDir, { recursive: true });
|
||||
if (target === "date") {
|
||||
await fs.symlink(externalRoot, path.join(transcriptsDir, "2026-07-01"), "dir");
|
||||
} else {
|
||||
const dateDir = path.join(transcriptsDir, "2026-07-01");
|
||||
await fs.mkdir(dateDir);
|
||||
await fs.symlink(externalRoot, path.join(dateDir, "linked-session"), "dir");
|
||||
}
|
||||
|
||||
expect(() =>
|
||||
detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
}),
|
||||
).toThrow("cannot be a symlink");
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("imports, verifies, receipts, archives, and reopens SQLite-only", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const sourceDir = await seedLegacySession({ stateDir, sessionId: "design-review" });
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
|
||||
const result = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
now: () => Date.parse("2026-07-02T00:00:00.000Z"),
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.changes.join("\n")).toContain("2 utterances");
|
||||
await expect(fs.stat(sourceDir)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
const archives = (await fs.readdir(stateDir)).filter((entry) =>
|
||||
entry.startsWith("transcripts.migrated-2026-07-02T00-00-00-000Z"),
|
||||
);
|
||||
expect(archives).toHaveLength(1);
|
||||
|
||||
const database = openOpenClawStateDatabase({ env: databaseEnv(stateDir) }).db;
|
||||
expect(
|
||||
database
|
||||
.prepare(
|
||||
"SELECT status, removed_source, source_record_count FROM migration_sources WHERE migration_kind = ?",
|
||||
)
|
||||
.get("meeting-transcripts-files-v1"),
|
||||
).toEqual({ status: "archived", removed_source: 1, source_record_count: 2 });
|
||||
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: databaseEnv(stateDir),
|
||||
});
|
||||
const session = await store.readSession("design-review");
|
||||
expect(session).toMatchObject({ title: "Design review" });
|
||||
await expect(store.readUtterancesForSession(session!)).resolves.toEqual([
|
||||
expect.objectContaining({ id: "u-1", text: "First line" }),
|
||||
expect.objectContaining({ id: "u-2", text: "Second line" }),
|
||||
]);
|
||||
await expect(store.readSummary(session!)).resolves.toMatchObject({
|
||||
markdown: "# Design review\n\nFirst line.\n",
|
||||
});
|
||||
});
|
||||
|
||||
it("imports shipped dot-only session layouts into reserved SQLite selectors", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
for (const sessionId of [".", "..", "session"]) {
|
||||
await seedLegacySession({ stateDir, sessionId });
|
||||
}
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
expect(detected.hasLegacy).toBe(true);
|
||||
|
||||
const result = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: databaseEnv(stateDir),
|
||||
});
|
||||
const expectedSlugs = new Map([
|
||||
[".", "%2E"],
|
||||
["..", "%2E%2E"],
|
||||
["session", "session"],
|
||||
]);
|
||||
for (const [sessionId, expectedSlug] of expectedSlugs) {
|
||||
const session = await store.readSession(sessionId);
|
||||
expect(session?.sessionId).toBe(sessionId);
|
||||
expect(store.sessionDir(session!)).toBe(
|
||||
path.join(stateDir, "transcripts", "2026-07-01", expectedSlug),
|
||||
);
|
||||
const artifacts = await store.materializeSessionArtifacts(session!, "transcript");
|
||||
const lines = (await fs.readFile(artifacts.transcriptPath, "utf8")).trim().split("\n");
|
||||
expect(lines.map((line) => JSON.parse(line).text)).toEqual(["First line", "Second line"]);
|
||||
}
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32" && process.platform !== "darwin")(
|
||||
"imports case-distinct sessions from a case-sensitive legacy tree",
|
||||
async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
await seedLegacySession({ stateDir, sessionId: "Capital" });
|
||||
await seedLegacySession({ stateDir, sessionId: "capital" });
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
|
||||
const result = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: databaseEnv(stateDir),
|
||||
});
|
||||
await expect(store.readSession("2026-07-01/Capital")).resolves.toMatchObject({
|
||||
sessionId: "Capital",
|
||||
});
|
||||
await expect(store.readSession("2026-07-01/capital")).resolves.toMatchObject({
|
||||
sessionId: "capital",
|
||||
});
|
||||
const upper = (await store.readSession("2026-07-01/Capital"))!;
|
||||
const lower = (await store.readSession("2026-07-01/capital"))!;
|
||||
const upperArtifacts = await store.materializeSessionArtifacts(upper, "metadata");
|
||||
await store.materializeSessionArtifacts(lower, "metadata");
|
||||
await fs.rename(
|
||||
upperArtifacts.sessionDir,
|
||||
path.join(stateDir, "transcripts", "2026-07-01", "CAPITAL"),
|
||||
);
|
||||
expect(
|
||||
detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
env: databaseEnv(stateDir),
|
||||
doctorOnlyStateMigrations: true,
|
||||
}),
|
||||
).toMatchObject({ hasLegacy: false });
|
||||
},
|
||||
);
|
||||
|
||||
it.runIf(process.platform !== "win32" && process.platform !== "darwin")(
|
||||
"does not assign a case-distinct legacy directory to an existing SQLite session",
|
||||
async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: databaseEnv(stateDir),
|
||||
});
|
||||
await store.writeSession({
|
||||
sessionId: "Capital",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-07-01T09:00:00.000Z",
|
||||
});
|
||||
await seedLegacySession({ stateDir, sessionId: "capital" });
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
env: databaseEnv(stateDir),
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
|
||||
const result = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
await expect(store.readSession("2026-07-01/Capital")).resolves.toMatchObject({
|
||||
sessionId: "Capital",
|
||||
});
|
||||
await expect(store.readSession("2026-07-01/capital")).resolves.toMatchObject({
|
||||
sessionId: "capital",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("preflights the whole tree before importing anything", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const validDir = await seedLegacySession({ stateDir, sessionId: "valid" });
|
||||
const invalidDir = await seedLegacySession({
|
||||
stateDir,
|
||||
sessionId: "invalid",
|
||||
invalidTranscript: true,
|
||||
});
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
|
||||
const result = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
});
|
||||
|
||||
expect(result.changes).toEqual([]);
|
||||
expect(result.warnings.join("\n")).toContain("Failed migrating meeting transcripts");
|
||||
await expect(fs.stat(validDir)).resolves.toBeDefined();
|
||||
await expect(fs.stat(invalidDir)).resolves.toBeDefined();
|
||||
const database = openOpenClawStateDatabase({ env: databaseEnv(stateDir) }).db;
|
||||
expect(
|
||||
database.prepare("SELECT COUNT(*) AS count FROM meeting_transcript_sessions").get(),
|
||||
).toEqual({ count: 0 });
|
||||
});
|
||||
|
||||
it("archives metadata-less interrupted session directories without blocking import", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
await seedLegacySession({ stateDir, sessionId: "complete-session" });
|
||||
const incompleteRelativeDir = path.join("2026-07-01", "incomplete-session");
|
||||
await fs.mkdir(path.join(stateDir, "transcripts", incompleteRelativeDir));
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
|
||||
const result = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
now: () => Date.parse("2026-07-02T00:00:00.000Z"),
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
await expect(
|
||||
fs.stat(
|
||||
path.join(stateDir, "transcripts.migrated-2026-07-02T00-00-00-000Z", incompleteRelativeDir),
|
||||
),
|
||||
).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it("detects and recovers a partial-only legacy transcript tree", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const partialDir = path.join(stateDir, "transcripts", "2026-07-01", "partial-only");
|
||||
await fs.mkdir(partialDir, { recursive: true });
|
||||
await fs.writeFile(path.join(partialDir, "transcript.jsonl"), '{"text":"partial"}\n');
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
expect(detected.hasLegacy).toBe(true);
|
||||
|
||||
const result = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
now: () => Date.parse("2026-07-02T00:00:00.000Z"),
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.changes.join("\n")).toContain("incomplete meeting transcript directory");
|
||||
await expect(fs.stat(path.join(partialDir, "transcript.jsonl"))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
await expect(
|
||||
fs.stat(
|
||||
path.join(
|
||||
stateDir,
|
||||
"transcripts.partials-recovered-2026-07-02T00-00-00-000Z",
|
||||
"2026-07-01",
|
||||
"partial-only",
|
||||
"transcript.jsonl",
|
||||
),
|
||||
),
|
||||
).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"preflights every partial artifact before moving any source",
|
||||
async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const externalDir = tempDirs.make("openclaw-meeting-transcripts-external-");
|
||||
const partialDir = path.join(stateDir, "transcripts", "2026-07-01", "partial-invalid");
|
||||
await fs.mkdir(partialDir, { recursive: true });
|
||||
await fs.writeFile(path.join(partialDir, "summary.md"), "keep me\n");
|
||||
await fs.writeFile(path.join(externalDir, "transcript.jsonl"), '{"text":"outside"}\n');
|
||||
await fs.symlink(
|
||||
path.join(externalDir, "transcript.jsonl"),
|
||||
path.join(partialDir, "transcript.jsonl"),
|
||||
);
|
||||
expect(() =>
|
||||
detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
}),
|
||||
).toThrow("regular file");
|
||||
await expect(fs.readFile(path.join(partialDir, "summary.md"), "utf8")).resolves.toBe(
|
||||
"keep me\n",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("rolls back when a session appears between verification and archive", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const sourceDir = await seedLegacySession({ stateDir, sessionId: "verified" });
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
|
||||
const result = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
testHooks: {
|
||||
afterImport: () => {
|
||||
const lateDir = path.join(stateDir, "transcripts", "2026-07-03", "late-session");
|
||||
fsSync.mkdirSync(lateDir, { recursive: true });
|
||||
fsSync.writeFileSync(
|
||||
path.join(lateDir, "metadata.json"),
|
||||
JSON.stringify({
|
||||
sessionId: "late-session",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-07-03T10:00:00.000Z",
|
||||
}),
|
||||
);
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.changes).toEqual([]);
|
||||
expect(result.warnings.join("\n")).toContain("session tree changed before archive");
|
||||
await expect(fs.stat(sourceDir)).resolves.toBeDefined();
|
||||
await expect(
|
||||
fs.stat(path.join(stateDir, "transcripts", "2026-07-03", "late-session")),
|
||||
).resolves.toBeDefined();
|
||||
const database = openOpenClawStateDatabase({ env: databaseEnv(stateDir) }).db;
|
||||
expect(
|
||||
database.prepare("SELECT COUNT(*) AS count FROM meeting_transcript_sessions").get(),
|
||||
).toEqual({ count: 0 });
|
||||
});
|
||||
|
||||
it("does not mistake a colliding archive destination for a completed move", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const sourceDir = await seedLegacySession({ stateDir, sessionId: "archive-collision" });
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
const archiveRoot = path.join(stateDir, "transcripts.migrated-2026-07-02T00-00-00-000Z");
|
||||
|
||||
const result = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
now: () => Date.parse("2026-07-02T00:00:00.000Z"),
|
||||
testHooks: {
|
||||
afterImport: () => {
|
||||
fsSync.mkdirSync(archiveRoot);
|
||||
fsSync.writeFileSync(path.join(archiveRoot, "unrelated"), "keep");
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.changes).toEqual([]);
|
||||
expect(result.warnings.join("\n")).toContain("Failed archiving verified legacy");
|
||||
await expect(fs.stat(sourceDir)).resolves.toBeDefined();
|
||||
await expect(fs.stat(archiveRoot)).resolves.toBeDefined();
|
||||
const database = openOpenClawStateDatabase({ env: databaseEnv(stateDir) }).db;
|
||||
expect(database.prepare("SELECT COUNT(*) AS count FROM migration_sources").get()).toEqual({
|
||||
count: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("restores idempotently when a canonical exporter recreated the destination", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const sourceRoot = path.join(stateDir, "transcripts");
|
||||
const archivedStateDir = tempDirs.make("openclaw-meeting-transcripts-archive-");
|
||||
const archivedSessionDir = await seedLegacySession({
|
||||
stateDir: archivedStateDir,
|
||||
sessionId: "recreated-export",
|
||||
});
|
||||
const archiveRoot = path.join(archivedStateDir, "transcripts");
|
||||
const destination = path.join(sourceRoot, "2026-07-01", "recreated-export");
|
||||
await fs.mkdir(path.dirname(destination), { recursive: true });
|
||||
await fs.cp(archivedSessionDir, destination, { recursive: true });
|
||||
|
||||
await expect(
|
||||
restoreCanonicalMeetingTranscriptExports({
|
||||
sourceRoot,
|
||||
archiveRoot,
|
||||
migratedSourcePaths: [],
|
||||
canonicalRelativeDirs: [path.join("2026-07-01", "recreated-export")],
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
await expect(fs.stat(path.join(destination, "metadata.json"))).resolves.toBeDefined();
|
||||
await expect(fs.stat(path.join(archivedSessionDir, "metadata.json"))).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it("rejects canonical restore paths that normalize outside their roots", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const sourceRoot = path.join(stateDir, "transcripts");
|
||||
const archiveRoot = path.join(stateDir, "transcripts.migrated-test");
|
||||
await fs.mkdir(archiveRoot, { recursive: true });
|
||||
|
||||
await expect(
|
||||
restoreCanonicalMeetingTranscriptExports({
|
||||
sourceRoot,
|
||||
archiveRoot,
|
||||
migratedSourcePaths: [],
|
||||
canonicalRelativeDirs: [path.join("safe", "..", "..", "outside")],
|
||||
}),
|
||||
).rejects.toThrow("escaped its root");
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"rejects symlinked ancestors during canonical export restore",
|
||||
async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const archiveRoot = path.join(stateDir, "transcripts.migrated-test");
|
||||
const externalRoot = tempDirs.make("openclaw-meeting-transcripts-external-");
|
||||
await fs.mkdir(path.join(archiveRoot), { recursive: true });
|
||||
await seedLegacySession({ stateDir: externalRoot, sessionId: "linked-export" });
|
||||
await fs.symlink(
|
||||
path.join(externalRoot, "transcripts", "2026-07-01"),
|
||||
path.join(archiveRoot, "2026-07-01"),
|
||||
"dir",
|
||||
);
|
||||
|
||||
await expect(
|
||||
restoreCanonicalMeetingTranscriptExports({
|
||||
sourceRoot: path.join(stateDir, "transcripts"),
|
||||
archiveRoot,
|
||||
migratedSourcePaths: [],
|
||||
canonicalRelativeDirs: [path.join("2026-07-01", "linked-export")],
|
||||
}),
|
||||
).rejects.toThrow(/symlink/i);
|
||||
},
|
||||
);
|
||||
|
||||
it("chunks large transcript imports while preserving exact order", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
await seedLegacySession({
|
||||
stateDir,
|
||||
sessionId: "long-meeting",
|
||||
utteranceCount: 530,
|
||||
});
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
|
||||
const result = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: databaseEnv(stateDir),
|
||||
});
|
||||
const session = await store.readSession("long-meeting");
|
||||
const utterances = await store.readUtterancesForSession(session!);
|
||||
expect(utterances).toHaveLength(530);
|
||||
expect(utterances[0]).toMatchObject({ id: "u-1", text: "First line" });
|
||||
expect(utterances.at(-1)).toMatchObject({ id: "u-530", text: "Line 530" });
|
||||
});
|
||||
|
||||
it("preserves an existing empty markdown summary", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
await seedLegacySession({
|
||||
stateDir,
|
||||
sessionId: "empty-summary",
|
||||
emptyMarkdown: true,
|
||||
omitSummaryJson: true,
|
||||
});
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
|
||||
const result = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: databaseEnv(stateDir),
|
||||
});
|
||||
const session = await store.readSession("empty-summary");
|
||||
await expect(store.readSummary(session!)).resolves.toEqual({ markdown: "" });
|
||||
});
|
||||
|
||||
it("resumes an interruption after the import commit", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: databaseEnv(stateDir),
|
||||
});
|
||||
const nativeSession = {
|
||||
sessionId: "modified-before-interruption",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-07-02T10:00:00.000Z",
|
||||
};
|
||||
await store.writeSession(nativeSession);
|
||||
const nativeArtifacts = await store.materializeSessionArtifacts(nativeSession, "metadata");
|
||||
await fs.appendFile(nativeArtifacts.metadataPath, " ");
|
||||
await seedLegacySession({ stateDir, sessionId: "interrupted-import" });
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
|
||||
const interrupted = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
testHooks: {
|
||||
afterImport: () => {
|
||||
throw new Error("interrupted");
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(interrupted.warnings.join("\n")).toContain("interrupted");
|
||||
expect(interrupted.changes.join("\n")).toContain("modified meeting transcript export");
|
||||
|
||||
const pending = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
env: databaseEnv(stateDir),
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
expect(pending.pendingImportCount).toBe(1);
|
||||
const resumed = await migrateLegacyMeetingTranscripts({
|
||||
detected: pending,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
});
|
||||
|
||||
expect(resumed.warnings).toEqual([]);
|
||||
expect(resumed.changes.join("\n")).toContain("Resumed and archived");
|
||||
});
|
||||
|
||||
it("refuses to archive a new legacy session added after a pending import", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
await seedLegacySession({ stateDir, sessionId: "pending-original" });
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
testHooks: {
|
||||
afterImport: () => {
|
||||
throw new Error("interrupted");
|
||||
},
|
||||
},
|
||||
});
|
||||
const lateDir = await seedLegacySession({ stateDir, sessionId: "pending-late" });
|
||||
|
||||
const pending = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
env: databaseEnv(stateDir),
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
const resumed = await migrateLegacyMeetingTranscripts({
|
||||
detected: pending,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
});
|
||||
|
||||
expect(resumed.changes).toEqual([]);
|
||||
expect(resumed.warnings.join("\n")).toContain("session tree changed before archive");
|
||||
await expect(fs.stat(lateDir)).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it("finalizes receipts after interruption following the archive move", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: databaseEnv(stateDir),
|
||||
});
|
||||
const nativeSession = {
|
||||
sessionId: "native-export-during-resume",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-07-02T10:00:00.000Z",
|
||||
};
|
||||
await store.writeSession(nativeSession);
|
||||
await store.materializeSessionArtifacts(nativeSession, "metadata");
|
||||
await seedLegacySession({ stateDir, sessionId: "interrupted-archive" });
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
|
||||
const interrupted = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
testHooks: {
|
||||
afterArchive: () => {
|
||||
throw new Error("interrupted");
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(interrupted.warnings.join("\n")).toContain("interrupted");
|
||||
|
||||
const pending = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
env: databaseEnv(stateDir),
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
expect(pending).toMatchObject({ hasLegacy: true, pendingImportCount: 1 });
|
||||
const resumed = await migrateLegacyMeetingTranscripts({
|
||||
detected: pending,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
});
|
||||
|
||||
expect(resumed.warnings).toEqual([]);
|
||||
expect(resumed.changes.join("\n")).toContain("Finalized interrupted");
|
||||
await expect(fs.stat(store.sessionDir(nativeSession))).resolves.toBeDefined();
|
||||
const database = openOpenClawStateDatabase({ env: databaseEnv(stateDir) }).db;
|
||||
expect(
|
||||
database
|
||||
.prepare("SELECT status, removed_source FROM migration_sources WHERE migration_kind = ?")
|
||||
.get("meeting-transcripts-files-v1"),
|
||||
).toEqual({ status: "archived", removed_source: 1 });
|
||||
});
|
||||
|
||||
it("does not reimport or archive explicit canonical exports", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
await seedLegacySession({ stateDir, sessionId: "design-review" });
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
await migrateLegacyMeetingTranscripts({ detected, env: databaseEnv(stateDir), stateDir });
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: databaseEnv(stateDir),
|
||||
});
|
||||
const session = await store.readSession("design-review");
|
||||
await store.materializeSessionArtifacts(session!, "all");
|
||||
|
||||
const rerunDetected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
const rerun = await migrateLegacyMeetingTranscripts({
|
||||
detected: rerunDetected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
});
|
||||
|
||||
expect(rerun).toEqual({ changes: [], warnings: [] });
|
||||
await expect(
|
||||
fs.stat(path.join(stateDir, "transcripts", "2026-07-01", "design-review")),
|
||||
).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it("does not classify an interrupted export of an advancing transcript as legacy", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: databaseEnv(stateDir),
|
||||
});
|
||||
const session = {
|
||||
sessionId: "advancing-export",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-07-02T10:00:00.000Z",
|
||||
};
|
||||
await store.writeSession(session);
|
||||
await store.appendUtteranceForSession(session, { text: "first" });
|
||||
await store.materializeSessionArtifacts(session, "transcript");
|
||||
await store.appendUtteranceForSession(session, { text: "second" });
|
||||
|
||||
const database = openOpenClawStateDatabase({ env: databaseEnv(stateDir) }).db;
|
||||
database
|
||||
.prepare(
|
||||
"UPDATE meeting_transcript_sessions SET export_pending_json = ? WHERE session_id = ?",
|
||||
)
|
||||
.run('["transcript.jsonl"]', session.sessionId);
|
||||
await fs.writeFile(path.join(store.sessionDir(session), "transcript.jsonl"), '{"text":');
|
||||
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
env: databaseEnv(stateDir),
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
expect(detected).toMatchObject({ hasLegacy: false });
|
||||
await expect(
|
||||
migrateLegacyMeetingTranscripts({ detected, env: databaseEnv(stateDir), stateDir }),
|
||||
).resolves.toEqual({ changes: [], warnings: [] });
|
||||
|
||||
await store.materializeSessionArtifacts(session, "transcript");
|
||||
const exported = await fs.readFile(
|
||||
path.join(store.sessionDir(session), "transcript.jsonl"),
|
||||
"utf8",
|
||||
);
|
||||
expect(
|
||||
exported
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line).text),
|
||||
).toEqual(["first", "second"]);
|
||||
});
|
||||
|
||||
it("migrates legacy sessions while preserving coexisting SQLite exports", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: databaseEnv(stateDir),
|
||||
});
|
||||
const nativeSession = {
|
||||
sessionId: "native-export",
|
||||
title: "Native export",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-07-02T10:00:00.000Z",
|
||||
};
|
||||
const nativeUtterance = { text: "SQLite-native line" };
|
||||
const nativeUtterances = [nativeUtterance];
|
||||
await store.writeSession(nativeSession);
|
||||
await store.appendUtteranceForSession(nativeSession, nativeUtterance);
|
||||
await store.writeSummary(
|
||||
summarizeTranscripts({ session: nativeSession, utterances: nativeUtterances }),
|
||||
nativeSession,
|
||||
);
|
||||
await store.materializeSessionArtifacts(nativeSession, "all");
|
||||
const nativeExport = path.join(
|
||||
stateDir,
|
||||
"transcripts",
|
||||
"2026-07-02",
|
||||
"native-export",
|
||||
"metadata.json",
|
||||
);
|
||||
const nativeTranscriptExport = path.join(path.dirname(nativeExport), "transcript.jsonl");
|
||||
await fs.rm(nativeExport);
|
||||
expect(
|
||||
detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
env: databaseEnv(stateDir),
|
||||
doctorOnlyStateMigrations: true,
|
||||
}),
|
||||
).toMatchObject({ hasLegacy: false });
|
||||
await seedLegacySession({ stateDir, sessionId: "legacy-alongside-export" });
|
||||
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
env: databaseEnv(stateDir),
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
const result = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
});
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
await expect(fs.stat(nativeExport)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(fs.stat(nativeTranscriptExport)).resolves.toBeDefined();
|
||||
await expect(store.readSession("native-export")).resolves.toMatchObject({
|
||||
sessionId: "native-export",
|
||||
});
|
||||
await expect(store.readSession("legacy-alongside-export")).resolves.toMatchObject({
|
||||
sessionId: "legacy-alongside-export",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not trust a DB selector when exported bytes diverge", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: databaseEnv(stateDir),
|
||||
});
|
||||
const session = {
|
||||
sessionId: "modified-export",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-07-02T10:00:00.000Z",
|
||||
};
|
||||
await store.writeSession(session);
|
||||
await store.materializeSessionArtifacts(session, "metadata");
|
||||
await fs.appendFile(
|
||||
path.join(stateDir, "transcripts", "2026-07-02", "modified-export", "metadata.json"),
|
||||
" ",
|
||||
);
|
||||
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
env: databaseEnv(stateDir),
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
expect(detected).toMatchObject({ hasLegacy: true });
|
||||
|
||||
const recovered = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
});
|
||||
|
||||
expect(recovered.warnings).toEqual([]);
|
||||
expect(recovered.changes.join("\n")).toContain("modified meeting transcript export");
|
||||
const metadata = await fs.readFile(
|
||||
path.join(stateDir, "transcripts", "2026-07-02", "modified-export", "metadata.json"),
|
||||
"utf8",
|
||||
);
|
||||
expect(metadata.endsWith("\n")).toBe(true);
|
||||
expect(
|
||||
(await fs.readdir(stateDir)).some((entry) =>
|
||||
entry.startsWith("transcripts.exports-recovered-"),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("resolves a case-renamed export directory by metadata identity", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-meeting-transcripts-doctor-");
|
||||
const store = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: databaseEnv(stateDir),
|
||||
});
|
||||
const session = {
|
||||
sessionId: "Capital",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-07-02T10:00:00.000Z",
|
||||
};
|
||||
await store.writeSession(session);
|
||||
await store.appendUtteranceForSession(session, { text: "case-stable transcript" });
|
||||
const artifacts = await store.materializeSessionArtifacts(session, "transcript");
|
||||
const renamedDir = path.join(stateDir, "transcripts", "2026-07-02", "capital");
|
||||
await fs.rename(artifacts.sessionDir, renamedDir);
|
||||
await fs.rename(path.join(renamedDir, "metadata.json"), path.join(renamedDir, "METADATA.JSON"));
|
||||
|
||||
expect(
|
||||
detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
env: databaseEnv(stateDir),
|
||||
doctorOnlyStateMigrations: true,
|
||||
}),
|
||||
).toMatchObject({ hasLegacy: false });
|
||||
await fs.rm(path.join(renamedDir, "METADATA.JSON"));
|
||||
const metadataLessDetected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
env: databaseEnv(stateDir),
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
expect(metadataLessDetected.hasLegacy).toBe(!fsSync.existsSync(artifacts.sessionDir));
|
||||
await fs.writeFile(
|
||||
path.join(renamedDir, "METADATA.JSON"),
|
||||
`${JSON.stringify(session, null, 2)}\n `,
|
||||
);
|
||||
const detected = detectLegacyMeetingTranscripts({
|
||||
stateDir,
|
||||
env: databaseEnv(stateDir),
|
||||
doctorOnlyStateMigrations: true,
|
||||
});
|
||||
expect(detected.hasLegacy).toBe(true);
|
||||
|
||||
const recovered = await migrateLegacyMeetingTranscripts({
|
||||
detected,
|
||||
env: databaseEnv(stateDir),
|
||||
stateDir,
|
||||
});
|
||||
expect(recovered.warnings).toEqual([]);
|
||||
expect(recovered.changes.join("\n")).toContain("modified meeting transcript export");
|
||||
await expect(store.readSession("2026-07-02/Capital")).resolves.toMatchObject({
|
||||
sessionId: "Capital",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,631 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
// Doctor-only import for the retired meeting-capture JSON/JSONL store.
|
||||
import fsSync from "node:fs";
|
||||
import path from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import {
|
||||
openOpenClawStateDatabase,
|
||||
runOpenClawStateWriteTransaction,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import { ensureMeetingTranscriptsSchema } from "../transcripts/sqlite-schema.js";
|
||||
import { TranscriptsStore } from "../transcripts/store.js";
|
||||
import { acquireGatewayLock } from "./gateway-lock.js";
|
||||
import { executeSqliteQuerySync, executeSqliteQueryTakeFirstSync } from "./kysely-sync.js";
|
||||
import { migrationDb } from "./state-migrations.meeting-transcripts-database.js";
|
||||
import {
|
||||
readMeetingTranscriptMigrationDetectionState,
|
||||
resolveMeetingTranscriptExportOwnership,
|
||||
} from "./state-migrations.meeting-transcripts-detection.js";
|
||||
import {
|
||||
archiveLegacyMeetingTranscriptSnapshots,
|
||||
archiveDivergentMeetingTranscriptExport,
|
||||
archivePartialMeetingTranscriptArtifacts,
|
||||
isRecordedCanonicalTranscriptExport,
|
||||
LegacyMeetingTranscriptArchiveMovedError,
|
||||
listLegacyMeetingTranscriptArtifactDirs,
|
||||
listLegacyMeetingTranscriptSessionDirs,
|
||||
openLegacyMeetingTranscriptStage,
|
||||
rehashLegacyMeetingTranscriptSnapshots,
|
||||
restoreCanonicalMeetingTranscriptExports,
|
||||
snapshotLegacyMeetingTranscriptSession,
|
||||
validateMeetingTranscriptRoot,
|
||||
type LegacyMeetingTranscriptSnapshot,
|
||||
} from "./state-migrations.meeting-transcripts-files.js";
|
||||
import { insertMeetingTranscriptSnapshots } from "./state-migrations.meeting-transcripts-insert.js";
|
||||
import { verifyImportedMeetingTranscriptSnapshots } from "./state-migrations.meeting-transcripts-verify.js";
|
||||
import type { LegacyMeetingTranscriptsDetection } from "./state-migrations.meeting-transcripts.types.js";
|
||||
import type { MigrationMessages } from "./state-migrations.types.js";
|
||||
|
||||
export { detectLegacyMeetingTranscripts } from "./state-migrations.meeting-transcripts-detection.js";
|
||||
|
||||
function resolveArchiveRoot(sourceRoot: string, now: number): string {
|
||||
const base = `${sourceRoot}.migrated-${new Date(now).toISOString().replace(/[:.]/g, "-")}`;
|
||||
return fsSync.existsSync(base) ? `${base}-${randomUUID()}` : base;
|
||||
}
|
||||
|
||||
function rollbackImportedSnapshots(params: {
|
||||
snapshots: LegacyMeetingTranscriptSnapshot[];
|
||||
runId: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
stateDir: string;
|
||||
}): void {
|
||||
runOpenClawStateWriteTransaction(
|
||||
({ db: database }) => {
|
||||
const db = migrationDb(database);
|
||||
for (const snapshot of params.snapshots) {
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db
|
||||
.deleteFrom("meeting_transcript_sessions")
|
||||
.where("session_id", "=", snapshot.session.sessionId)
|
||||
.where("started_at", "=", snapshot.session.startedAt),
|
||||
);
|
||||
}
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db.deleteFrom("migration_sources").where("last_run_id", "=", params.runId),
|
||||
);
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db.deleteFrom("migration_runs").where("id", "=", params.runId),
|
||||
);
|
||||
},
|
||||
{ env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir } },
|
||||
{ operationLabel: "meeting-transcripts.legacy-import.rollback" },
|
||||
);
|
||||
}
|
||||
|
||||
function finishPendingMigration(params: {
|
||||
runId: string;
|
||||
archiveRoot: string;
|
||||
now: number;
|
||||
env: NodeJS.ProcessEnv;
|
||||
stateDir: string;
|
||||
}): void {
|
||||
runOpenClawStateWriteTransaction(
|
||||
({ db: database }) => {
|
||||
const db = migrationDb(database);
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db
|
||||
.updateTable("migration_sources")
|
||||
.set({ status: "archived", removed_source: 1 })
|
||||
.where("last_run_id", "=", params.runId)
|
||||
.where("migration_kind", "=", "meeting-transcripts-files-v1"),
|
||||
);
|
||||
const run = executeSqliteQueryTakeFirstSync(
|
||||
database,
|
||||
db.selectFrom("migration_runs").select("report_json").where("id", "=", params.runId),
|
||||
);
|
||||
const report = run ? (JSON.parse(run.report_json) as Record<string, unknown>) : {};
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db
|
||||
.updateTable("migration_runs")
|
||||
.set({
|
||||
finished_at: params.now,
|
||||
status: "completed",
|
||||
report_json: JSON.stringify({ ...report, archiveRoot: params.archiveRoot }),
|
||||
})
|
||||
.where("id", "=", params.runId),
|
||||
);
|
||||
},
|
||||
{ env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir } },
|
||||
{ operationLabel: "meeting-transcripts.legacy-import.finish" },
|
||||
);
|
||||
}
|
||||
|
||||
type PendingImportRun = {
|
||||
runId: string;
|
||||
archiveRoot: string;
|
||||
canonicalRelativeDirs: string[];
|
||||
sources: Array<{ sourcePath: string; sourceHash: string }>;
|
||||
};
|
||||
|
||||
function isStrictRelativePathWithinRoot(root: string, relativePath: string): boolean {
|
||||
if (!relativePath || relativePath === "." || path.isAbsolute(relativePath)) {
|
||||
return false;
|
||||
}
|
||||
const resolvedRoot = path.resolve(root);
|
||||
const resolvedPath = path.resolve(resolvedRoot, relativePath);
|
||||
const relative = path.relative(resolvedRoot, resolvedPath);
|
||||
return Boolean(relative && !relative.startsWith("..") && !path.isAbsolute(relative));
|
||||
}
|
||||
|
||||
async function snapshotPendingImportRun(params: {
|
||||
run: PendingImportRun;
|
||||
snapshotRoot: string;
|
||||
sourceRoot: string;
|
||||
stageDatabase: DatabaseSync;
|
||||
}): Promise<{ snapshots: LegacyMeetingTranscriptSnapshot[]; hashesMatch: boolean }> {
|
||||
const snapshots: LegacyMeetingTranscriptSnapshot[] = [];
|
||||
let hashesMatch = true;
|
||||
for (const source of params.run.sources) {
|
||||
const relativeDir = path.relative(params.sourceRoot, source.sourcePath) || ".";
|
||||
if (relativeDir.startsWith("..") || path.isAbsolute(relativeDir)) {
|
||||
throw new Error(`pending meeting transcript source escaped its root: ${source.sourcePath}`);
|
||||
}
|
||||
const snapshot = await snapshotLegacyMeetingTranscriptSession({
|
||||
rootDir: params.snapshotRoot,
|
||||
relativeDir,
|
||||
stageDatabase: params.stageDatabase,
|
||||
});
|
||||
snapshots.push(snapshot);
|
||||
hashesMatch &&= snapshot.sourceHash === source.sourceHash;
|
||||
}
|
||||
return { snapshots, hashesMatch };
|
||||
}
|
||||
|
||||
function readPendingImportRuns(params: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
stateDir: string;
|
||||
sourceRoot: string;
|
||||
}): PendingImportRun[] {
|
||||
const database = openOpenClawStateDatabase({
|
||||
env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir },
|
||||
});
|
||||
const db = migrationDb(database.db);
|
||||
const rows = executeSqliteQuerySync(
|
||||
database.db,
|
||||
db
|
||||
.selectFrom("migration_sources as source")
|
||||
.innerJoin("migration_runs as run", "run.id", "source.last_run_id")
|
||||
.select([
|
||||
"source.last_run_id as run_id",
|
||||
"source.source_path",
|
||||
"source.source_sha256",
|
||||
"run.report_json as run_report_json",
|
||||
])
|
||||
.where("source.migration_kind", "=", "meeting-transcripts-files-v1")
|
||||
.where("source.status", "=", "imported")
|
||||
.where("source.removed_source", "=", 0)
|
||||
.orderBy("source.source_path", "asc"),
|
||||
).rows;
|
||||
const runs = new Map<string, PendingImportRun>();
|
||||
for (const row of rows) {
|
||||
const report = JSON.parse(row.run_report_json) as Record<string, unknown>;
|
||||
const format = report.format;
|
||||
const archiveRoot = report.archiveRoot;
|
||||
const canonicalRelativeDirs = report.canonicalRelativeDirs;
|
||||
if (
|
||||
typeof archiveRoot !== "string" ||
|
||||
format !== "meeting-transcripts-files-v1" ||
|
||||
!archiveRoot.startsWith(`${params.sourceRoot}.migrated-`) ||
|
||||
!Array.isArray(canonicalRelativeDirs) ||
|
||||
!canonicalRelativeDirs.every(
|
||||
(relativeDir) =>
|
||||
typeof relativeDir === "string" &&
|
||||
isStrictRelativePathWithinRoot(params.sourceRoot, relativeDir),
|
||||
) ||
|
||||
typeof row.source_sha256 !== "string"
|
||||
) {
|
||||
throw new Error(`invalid pending meeting transcript migration receipt: ${row.run_id}`);
|
||||
}
|
||||
const run: PendingImportRun = runs.get(row.run_id) ?? {
|
||||
runId: row.run_id,
|
||||
archiveRoot,
|
||||
canonicalRelativeDirs,
|
||||
sources: [],
|
||||
};
|
||||
if (
|
||||
run.archiveRoot !== archiveRoot ||
|
||||
JSON.stringify(run.canonicalRelativeDirs) !== JSON.stringify(canonicalRelativeDirs)
|
||||
) {
|
||||
throw new Error(`conflicting meeting transcript archive receipts: ${row.run_id}`);
|
||||
}
|
||||
run.sources.push({ sourcePath: row.source_path, sourceHash: row.source_sha256 });
|
||||
runs.set(row.run_id, run);
|
||||
}
|
||||
return [...runs.values()];
|
||||
}
|
||||
|
||||
async function listCanonicalMeetingTranscriptExportDirs(params: {
|
||||
rootDir: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
}): Promise<string[]> {
|
||||
const state = readMeetingTranscriptMigrationDetectionState({ env: params.env });
|
||||
const relativeDirs = await listLegacyMeetingTranscriptArtifactDirs(params.rootDir);
|
||||
return relativeDirs.filter((relativeDir) => {
|
||||
const selector = relativeDir.split(path.sep).join("/");
|
||||
const sessionDir = path.join(params.rootDir, relativeDir);
|
||||
const ownership = resolveMeetingTranscriptExportOwnership({
|
||||
state,
|
||||
selector,
|
||||
sessionDir,
|
||||
sourceRoot: params.rootDir,
|
||||
});
|
||||
return Boolean(
|
||||
ownership &&
|
||||
isRecordedCanonicalTranscriptExport({
|
||||
sessionDir,
|
||||
manifest: ownership.manifest,
|
||||
pending: ownership.pending,
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function resumePendingImports(params: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
stateDir: string;
|
||||
sourceRoot: string;
|
||||
store: TranscriptsStore;
|
||||
stageDatabase: DatabaseSync;
|
||||
}): Promise<MigrationMessages | undefined> {
|
||||
const runs = readPendingImportRuns(params);
|
||||
if (runs.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const changes: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
for (const run of runs) {
|
||||
if (fsSync.existsSync(run.archiveRoot)) {
|
||||
try {
|
||||
const archived = await snapshotPendingImportRun({
|
||||
run,
|
||||
snapshotRoot: run.archiveRoot,
|
||||
sourceRoot: params.sourceRoot,
|
||||
stageDatabase: params.stageDatabase,
|
||||
});
|
||||
if (!archived.hashesMatch) {
|
||||
throw new Error("archived source hashes do not match migration receipts");
|
||||
}
|
||||
const database = openOpenClawStateDatabase({
|
||||
env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir },
|
||||
});
|
||||
await verifyImportedMeetingTranscriptSnapshots({
|
||||
store: params.store,
|
||||
snapshots: archived.snapshots,
|
||||
stageDatabase: params.stageDatabase,
|
||||
database: database.db,
|
||||
});
|
||||
await restoreCanonicalMeetingTranscriptExports({
|
||||
sourceRoot: params.sourceRoot,
|
||||
archiveRoot: run.archiveRoot,
|
||||
migratedSourcePaths: run.sources.map((source) => source.sourcePath),
|
||||
canonicalRelativeDirs: run.canonicalRelativeDirs,
|
||||
});
|
||||
finishPendingMigration({
|
||||
runId: run.runId,
|
||||
archiveRoot: run.archiveRoot,
|
||||
now: Date.now(),
|
||||
env: params.env,
|
||||
stateDir: params.stateDir,
|
||||
});
|
||||
changes.push(`Finalized interrupted meeting transcript archive → ${run.archiveRoot}`);
|
||||
} catch (error) {
|
||||
warnings.push(
|
||||
`Pending meeting transcript migration ${run.runId} archive could not be verified or restored; left its rows and files for recovery: ${String(error)}`,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!fsSync.existsSync(params.sourceRoot)) {
|
||||
warnings.push(
|
||||
`Pending meeting transcript migration ${run.runId} has neither source tree nor archive`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const canonicalRelativeDirs = [
|
||||
...new Set([
|
||||
...run.canonicalRelativeDirs,
|
||||
...(await listCanonicalMeetingTranscriptExportDirs({
|
||||
rootDir: params.sourceRoot,
|
||||
env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir },
|
||||
})),
|
||||
]),
|
||||
];
|
||||
const expectedRelativeDirs = [
|
||||
...new Set([
|
||||
...run.sources.map((source) => path.relative(params.sourceRoot, source.sourcePath) || "."),
|
||||
...canonicalRelativeDirs,
|
||||
]),
|
||||
].toSorted((a, b) => a.localeCompare(b));
|
||||
const pending = await snapshotPendingImportRun({
|
||||
run,
|
||||
snapshotRoot: params.sourceRoot,
|
||||
sourceRoot: params.sourceRoot,
|
||||
stageDatabase: params.stageDatabase,
|
||||
});
|
||||
if (!pending.hashesMatch) {
|
||||
warnings.push(
|
||||
`Pending meeting transcript migration ${run.runId} source tree changed; left its imported rows and files for manual recovery`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const database = openOpenClawStateDatabase({
|
||||
env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir },
|
||||
});
|
||||
await verifyImportedMeetingTranscriptSnapshots({
|
||||
store: params.store,
|
||||
snapshots: pending.snapshots,
|
||||
stageDatabase: params.stageDatabase,
|
||||
database: database.db,
|
||||
});
|
||||
await archiveLegacyMeetingTranscriptSnapshots({
|
||||
sourceRoot: params.sourceRoot,
|
||||
snapshots: pending.snapshots,
|
||||
expectedRelativeDirs,
|
||||
canonicalRelativeDirs,
|
||||
archiveRoot: run.archiveRoot,
|
||||
});
|
||||
finishPendingMigration({
|
||||
runId: run.runId,
|
||||
archiveRoot: run.archiveRoot,
|
||||
now: Date.now(),
|
||||
env: params.env,
|
||||
stateDir: params.stateDir,
|
||||
});
|
||||
changes.push(`Resumed and archived meeting transcript migration → ${run.archiveRoot}`);
|
||||
}
|
||||
return { changes, warnings };
|
||||
}
|
||||
|
||||
export async function migrateLegacyMeetingTranscripts(params: {
|
||||
detected?: LegacyMeetingTranscriptsDetection;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
stateDir: string;
|
||||
now?: () => number;
|
||||
testHooks?: {
|
||||
afterImport?: () => void;
|
||||
afterArchive?: () => void;
|
||||
};
|
||||
}): Promise<MigrationMessages> {
|
||||
const detected = params.detected;
|
||||
if (!detected?.hasLegacy) {
|
||||
return { changes: [], warnings: [] };
|
||||
}
|
||||
const env = params.env ?? process.env;
|
||||
let lock: Awaited<ReturnType<typeof acquireGatewayLock>>;
|
||||
try {
|
||||
lock = await acquireGatewayLock({
|
||||
allowInTests: true,
|
||||
env: { ...env, OPENCLAW_STATE_DIR: params.stateDir },
|
||||
role: "sqlite-maintenance",
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
} catch (error) {
|
||||
return {
|
||||
changes: [],
|
||||
warnings: [
|
||||
`Skipped meeting transcript migration because exclusive state ownership is unavailable: ${String(error)}`,
|
||||
],
|
||||
};
|
||||
}
|
||||
if (!lock) {
|
||||
return {
|
||||
changes: [],
|
||||
warnings: [
|
||||
"Skipped meeting transcript migration because exclusive state ownership is unavailable",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
let stageDatabase: DatabaseSync | undefined;
|
||||
let stagePath: string | undefined;
|
||||
const recoveryChanges: string[] = [];
|
||||
try {
|
||||
fsSync.mkdirSync(params.stateDir, { recursive: true });
|
||||
stagePath = path.join(params.stateDir, `.meeting-transcripts-migration-${randomUUID()}.sqlite`);
|
||||
const stage = openLegacyMeetingTranscriptStage(stagePath);
|
||||
stageDatabase = stage;
|
||||
await validateMeetingTranscriptRoot(detected.sourceDir, { allowMissing: true });
|
||||
const databaseOptions = { env: { ...env, OPENCLAW_STATE_DIR: params.stateDir } };
|
||||
ensureMeetingTranscriptsSchema(databaseOptions);
|
||||
const store = new TranscriptsStore(detected.sourceDir, databaseOptions);
|
||||
const resumed = await resumePendingImports({
|
||||
env,
|
||||
stateDir: params.stateDir,
|
||||
sourceRoot: detected.sourceDir,
|
||||
store,
|
||||
stageDatabase: stage,
|
||||
});
|
||||
if (resumed) {
|
||||
return resumed;
|
||||
}
|
||||
const now = params.now?.() ?? Date.now();
|
||||
const relativeDirs = await listLegacyMeetingTranscriptArtifactDirs(detected.sourceDir);
|
||||
const sessionRelativeDirs = await listLegacyMeetingTranscriptSessionDirs(detected.sourceDir);
|
||||
const sessionRelativeDirSet = new Set(sessionRelativeDirs);
|
||||
const detectionState = readMeetingTranscriptMigrationDetectionState({
|
||||
env: { ...env, OPENCLAW_STATE_DIR: params.stateDir },
|
||||
});
|
||||
const legacyRelativeDirs: string[] = [];
|
||||
const partialRelativeDirs: string[] = [];
|
||||
const divergentExportDirs: Array<{ relativeDir: string; ownerSelector: string }> = [];
|
||||
for (const relativeDir of relativeDirs) {
|
||||
const selector = relativeDir.split(path.sep).join("/");
|
||||
const ownership = resolveMeetingTranscriptExportOwnership({
|
||||
state: detectionState,
|
||||
selector,
|
||||
sessionDir: path.join(detected.sourceDir, relativeDir),
|
||||
sourceRoot: detected.sourceDir,
|
||||
});
|
||||
if (
|
||||
ownership &&
|
||||
isRecordedCanonicalTranscriptExport({
|
||||
sessionDir: path.join(detected.sourceDir, relativeDir),
|
||||
manifest: ownership.manifest,
|
||||
pending: ownership.pending,
|
||||
})
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (ownership) {
|
||||
divergentExportDirs.push({ relativeDir, ownerSelector: ownership.selector });
|
||||
} else if (!sessionRelativeDirSet.has(relativeDir)) {
|
||||
partialRelativeDirs.push(relativeDir);
|
||||
} else {
|
||||
legacyRelativeDirs.push(relativeDir);
|
||||
}
|
||||
}
|
||||
const snapshots: LegacyMeetingTranscriptSnapshot[] = [];
|
||||
for (const relativeDir of legacyRelativeDirs) {
|
||||
snapshots.push(
|
||||
await snapshotLegacyMeetingTranscriptSession({
|
||||
rootDir: detected.sourceDir,
|
||||
relativeDir,
|
||||
stageDatabase: stage,
|
||||
}),
|
||||
);
|
||||
}
|
||||
const plans: LegacyMeetingTranscriptSnapshot[] = [];
|
||||
for (const snapshot of snapshots) {
|
||||
const database = openOpenClawStateDatabase(databaseOptions);
|
||||
const existing = executeSqliteQueryTakeFirstSync(
|
||||
database.db,
|
||||
migrationDb(database.db)
|
||||
.selectFrom("meeting_transcript_sessions")
|
||||
.select("session_id")
|
||||
.where("session_id", "=", snapshot.session.sessionId)
|
||||
.where("started_at", "=", snapshot.session.startedAt),
|
||||
);
|
||||
if (existing) {
|
||||
throw new Error(
|
||||
`legacy transcript conflicts with canonical SQLite state: ${snapshot.relativeDir}`,
|
||||
);
|
||||
}
|
||||
plans.push(snapshot);
|
||||
}
|
||||
if (divergentExportDirs.length > 0) {
|
||||
const recoveryRoot = `${detected.sourceDir}.exports-recovered-${new Date(now)
|
||||
.toISOString()
|
||||
.replace(/[:.]/g, "-")}`;
|
||||
for (const { relativeDir, ownerSelector } of divergentExportDirs) {
|
||||
const session = await store.readSession(ownerSelector);
|
||||
if (!session) {
|
||||
throw new Error(`divergent transcript export has no SQLite owner: ${relativeDir}`);
|
||||
}
|
||||
await archiveDivergentMeetingTranscriptExport({
|
||||
sourceRoot: detected.sourceDir,
|
||||
relativeDir,
|
||||
recoveryRoot,
|
||||
});
|
||||
recoveryChanges.push(
|
||||
`Archived modified meeting transcript export ${relativeDir} → ${recoveryRoot}`,
|
||||
);
|
||||
await store.materializeSessionArtifacts(session, "all");
|
||||
}
|
||||
}
|
||||
if (plans.length === 0 && partialRelativeDirs.length > 0) {
|
||||
const recoveryRoot = `${detected.sourceDir}.partials-recovered-${new Date(now)
|
||||
.toISOString()
|
||||
.replace(/[:.]/g, "-")}`;
|
||||
await archivePartialMeetingTranscriptArtifacts({
|
||||
sourceRoot: detected.sourceDir,
|
||||
relativeDirs: partialRelativeDirs,
|
||||
recoveryRoot,
|
||||
});
|
||||
recoveryChanges.push(
|
||||
`Archived ${partialRelativeDirs.length} incomplete meeting transcript director${partialRelativeDirs.length === 1 ? "y" : "ies"} → ${recoveryRoot}`,
|
||||
);
|
||||
}
|
||||
const expectedArchiveRelativeDirs = await listLegacyMeetingTranscriptSessionDirs(
|
||||
detected.sourceDir,
|
||||
);
|
||||
if (plans.length === 0) {
|
||||
return { changes: recoveryChanges, warnings: [] };
|
||||
}
|
||||
|
||||
const runId = randomUUID();
|
||||
const archiveRoot = resolveArchiveRoot(detected.sourceDir, now);
|
||||
const canonicalRelativeDirs = await listCanonicalMeetingTranscriptExportDirs({
|
||||
rootDir: detected.sourceDir,
|
||||
env: { ...env, OPENCLAW_STATE_DIR: params.stateDir },
|
||||
});
|
||||
insertMeetingTranscriptSnapshots({
|
||||
snapshots: plans,
|
||||
runId,
|
||||
now,
|
||||
archiveRoot,
|
||||
canonicalRelativeDirs,
|
||||
stageDatabase: stage,
|
||||
env,
|
||||
stateDir: params.stateDir,
|
||||
});
|
||||
try {
|
||||
const database = openOpenClawStateDatabase(databaseOptions);
|
||||
await verifyImportedMeetingTranscriptSnapshots({
|
||||
store,
|
||||
snapshots: plans,
|
||||
stageDatabase: stage,
|
||||
database: database.db,
|
||||
});
|
||||
if (!(await rehashLegacyMeetingTranscriptSnapshots(plans))) {
|
||||
rollbackImportedSnapshots({ snapshots: plans, runId, env, stateDir: params.stateDir });
|
||||
return {
|
||||
changes: recoveryChanges,
|
||||
warnings: [
|
||||
"Legacy meeting transcript files changed after import; rolled back SQLite rows and left every source in place for a Doctor retry",
|
||||
],
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
rollbackImportedSnapshots({ snapshots: plans, runId, env, stateDir: params.stateDir });
|
||||
throw error;
|
||||
}
|
||||
params.testHooks?.afterImport?.();
|
||||
let archiveRootAfterMove: string;
|
||||
try {
|
||||
archiveRootAfterMove = await archiveLegacyMeetingTranscriptSnapshots({
|
||||
sourceRoot: detected.sourceDir,
|
||||
snapshots: plans,
|
||||
expectedRelativeDirs: expectedArchiveRelativeDirs,
|
||||
canonicalRelativeDirs,
|
||||
archiveRoot,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof LegacyMeetingTranscriptArchiveMovedError) {
|
||||
return {
|
||||
changes: [
|
||||
...recoveryChanges,
|
||||
`Imported ${plans.length} meeting transcript session${plans.length === 1 ? "" : "s"} into shared SQLite state`,
|
||||
],
|
||||
warnings: [
|
||||
`Meeting transcript archive needs Doctor resume after moving the source tree: ${String(error)}`,
|
||||
],
|
||||
};
|
||||
}
|
||||
rollbackImportedSnapshots({ snapshots: plans, runId, env, stateDir: params.stateDir });
|
||||
return {
|
||||
changes: recoveryChanges,
|
||||
warnings: [
|
||||
`Failed archiving verified legacy meeting transcripts; rolled back SQLite rows and left every source in place for Doctor retry: ${String(error)}`,
|
||||
],
|
||||
};
|
||||
}
|
||||
params.testHooks?.afterArchive?.();
|
||||
finishPendingMigration({
|
||||
runId,
|
||||
archiveRoot: archiveRootAfterMove,
|
||||
now,
|
||||
env,
|
||||
stateDir: params.stateDir,
|
||||
});
|
||||
const utteranceCount = plans.reduce((total, snapshot) => total + snapshot.utteranceCount, 0);
|
||||
return {
|
||||
changes: [
|
||||
...recoveryChanges,
|
||||
`Migrated ${plans.length} meeting transcript session${plans.length === 1 ? "" : "s"} and ${utteranceCount} utterance${utteranceCount === 1 ? "" : "s"} to shared SQLite state`,
|
||||
`Archived legacy meeting transcript files → ${archiveRootAfterMove}`,
|
||||
],
|
||||
warnings: [],
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
changes: recoveryChanges,
|
||||
warnings: [`Failed migrating meeting transcripts: ${String(error)}`],
|
||||
};
|
||||
} finally {
|
||||
try {
|
||||
stageDatabase?.close();
|
||||
if (stagePath) {
|
||||
fsSync.rmSync(stagePath, { force: true });
|
||||
fsSync.rmSync(`${stagePath}-shm`, { force: true });
|
||||
fsSync.rmSync(`${stagePath}-wal`, { force: true });
|
||||
}
|
||||
} finally {
|
||||
await lock.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export type LegacyMeetingTranscriptsDetection = {
|
||||
sourceDir: string;
|
||||
hasLegacy: boolean;
|
||||
pendingImportCount: number;
|
||||
};
|
||||
@@ -5,6 +5,7 @@ import type { LegacyAuditLogsDetection } from "./state-migrations.audit-logs.typ
|
||||
import type { LegacyChannelPairingStateDetection } from "./state-migrations.channel-pairing.js";
|
||||
import type { LegacyDeviceIdentityDetection } from "./state-migrations.device-identity.types.js";
|
||||
import type { LegacyMcpOAuthDetection } from "./state-migrations.mcp-oauth.types.js";
|
||||
import type { LegacyMeetingTranscriptsDetection } from "./state-migrations.meeting-transcripts.types.js";
|
||||
import type { LegacyRestartSentinelDetection } from "./state-migrations.restart-sentinel.types.js";
|
||||
import type { LegacyWorkspaceStateDetection } from "./state-migrations.workspace-setup.types.js";
|
||||
|
||||
@@ -126,6 +127,7 @@ export type LegacyStateDetection = {
|
||||
};
|
||||
deviceIdentity: LegacyDeviceIdentityDetection;
|
||||
mcpOauth: LegacyMcpOAuthDetection;
|
||||
meetingTranscripts?: LegacyMeetingTranscriptsDetection;
|
||||
restartSentinel?: LegacyRestartSentinelDetection;
|
||||
workspace: LegacyWorkspaceStateDetection;
|
||||
webPush: {
|
||||
|
||||
+44
@@ -783,6 +783,47 @@ export interface MediaBlobs {
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
export interface MeetingTranscriptSessions {
|
||||
created_at_ms: number;
|
||||
export_key: string;
|
||||
export_manifest_json: Generated<string>;
|
||||
export_pending_json: Generated<string>;
|
||||
metadata_json: string | null;
|
||||
next_utterance_seq: Generated<number>;
|
||||
provider_id: string;
|
||||
selector: string;
|
||||
session_id: string;
|
||||
session_slug: string;
|
||||
source_json: string;
|
||||
started_at: string;
|
||||
stopped_at: string | null;
|
||||
title: string | null;
|
||||
updated_at_ms: number;
|
||||
}
|
||||
|
||||
export interface MeetingTranscriptSummaries {
|
||||
generated_at: string | null;
|
||||
markdown: string | null;
|
||||
session_id: string;
|
||||
session_started_at: string;
|
||||
summary_json: string | null;
|
||||
utterance_count: number;
|
||||
}
|
||||
|
||||
export interface MeetingTranscriptUtterances {
|
||||
ended_at: string | null;
|
||||
final: number | null;
|
||||
metadata_json: string | null;
|
||||
sequence: number;
|
||||
session_id: string;
|
||||
session_started_at: string;
|
||||
speaker_id: string | null;
|
||||
speaker_label: string | null;
|
||||
started_at: string | null;
|
||||
text: string;
|
||||
utterance_id: string | null;
|
||||
}
|
||||
|
||||
export interface MigrationRuns {
|
||||
finished_at: number | null;
|
||||
id: string;
|
||||
@@ -1481,6 +1522,9 @@ export interface DB {
|
||||
managed_outgoing_image_records: ManagedOutgoingImageRecords;
|
||||
mcp_oauth_stores: McpOauthStores;
|
||||
media_blobs: MediaBlobs;
|
||||
meeting_transcript_sessions: MeetingTranscriptSessions;
|
||||
meeting_transcript_summaries: MeetingTranscriptSummaries;
|
||||
meeting_transcript_utterances: MeetingTranscriptUtterances;
|
||||
migration_runs: MigrationRuns;
|
||||
migration_sources: MigrationSources;
|
||||
model_capability_cache: ModelCapabilityCache;
|
||||
|
||||
@@ -1579,6 +1579,71 @@ CREATE INDEX IF NOT EXISTS idx_flow_runs_status ON flow_runs(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_flow_runs_owner_key ON flow_runs(owner_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_flow_runs_updated_at ON flow_runs(updated_at);
|
||||
|
||||
-- Durable meeting-capture sessions are gateway-global rather than agent-session
|
||||
-- transcripts. JSON/JSONL files are doctor import inputs or explicit CLI exports.
|
||||
CREATE TABLE IF NOT EXISTS meeting_transcript_sessions (
|
||||
session_id TEXT NOT NULL,
|
||||
started_at TEXT NOT NULL,
|
||||
selector TEXT NOT NULL UNIQUE,
|
||||
export_key TEXT NOT NULL,
|
||||
session_slug TEXT NOT NULL,
|
||||
provider_id TEXT NOT NULL,
|
||||
title TEXT,
|
||||
source_json TEXT NOT NULL,
|
||||
stopped_at TEXT,
|
||||
metadata_json TEXT,
|
||||
export_manifest_json TEXT NOT NULL DEFAULT '{}',
|
||||
export_pending_json TEXT NOT NULL DEFAULT '[]',
|
||||
next_utterance_seq INTEGER NOT NULL DEFAULT 0 CHECK (next_utterance_seq >= 0),
|
||||
created_at_ms INTEGER NOT NULL CHECK (created_at_ms >= 0),
|
||||
updated_at_ms INTEGER NOT NULL CHECK (updated_at_ms >= 0),
|
||||
PRIMARY KEY (session_id, started_at)
|
||||
) STRICT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_started
|
||||
ON meeting_transcript_sessions(started_at DESC, session_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_id
|
||||
ON meeting_transcript_sessions(session_id, started_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_slug
|
||||
ON meeting_transcript_sessions(session_slug, started_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_export_key
|
||||
ON meeting_transcript_sessions(export_key);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meeting_transcript_utterances (
|
||||
session_id TEXT NOT NULL,
|
||||
session_started_at TEXT NOT NULL,
|
||||
sequence INTEGER NOT NULL CHECK (sequence >= 0),
|
||||
utterance_id TEXT,
|
||||
started_at TEXT,
|
||||
ended_at TEXT,
|
||||
speaker_id TEXT,
|
||||
speaker_label TEXT,
|
||||
text TEXT NOT NULL,
|
||||
final INTEGER CHECK (final IN (0, 1)),
|
||||
metadata_json TEXT,
|
||||
PRIMARY KEY (session_id, session_started_at, sequence),
|
||||
FOREIGN KEY (session_id, session_started_at)
|
||||
REFERENCES meeting_transcript_sessions(session_id, started_at)
|
||||
ON DELETE CASCADE
|
||||
) STRICT;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meeting_transcript_summaries (
|
||||
session_id TEXT NOT NULL,
|
||||
session_started_at TEXT NOT NULL,
|
||||
generated_at TEXT,
|
||||
summary_json TEXT,
|
||||
markdown TEXT,
|
||||
utterance_count INTEGER NOT NULL CHECK (utterance_count >= 0),
|
||||
PRIMARY KEY (session_id, session_started_at),
|
||||
FOREIGN KEY (session_id, session_started_at)
|
||||
REFERENCES meeting_transcript_sessions(session_id, started_at)
|
||||
ON DELETE CASCADE,
|
||||
CHECK (summary_json IS NOT NULL OR markdown IS NOT NULL)
|
||||
) STRICT;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS migration_runs (
|
||||
id TEXT NOT NULL PRIMARY KEY,
|
||||
started_at INTEGER NOT NULL,
|
||||
|
||||
@@ -1574,6 +1574,71 @@ CREATE INDEX IF NOT EXISTS idx_flow_runs_status ON flow_runs(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_flow_runs_owner_key ON flow_runs(owner_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_flow_runs_updated_at ON flow_runs(updated_at);
|
||||
|
||||
-- Durable meeting-capture sessions are gateway-global rather than agent-session
|
||||
-- transcripts. JSON/JSONL files are doctor import inputs or explicit CLI exports.
|
||||
CREATE TABLE IF NOT EXISTS meeting_transcript_sessions (
|
||||
session_id TEXT NOT NULL,
|
||||
started_at TEXT NOT NULL,
|
||||
selector TEXT NOT NULL UNIQUE,
|
||||
export_key TEXT NOT NULL,
|
||||
session_slug TEXT NOT NULL,
|
||||
provider_id TEXT NOT NULL,
|
||||
title TEXT,
|
||||
source_json TEXT NOT NULL,
|
||||
stopped_at TEXT,
|
||||
metadata_json TEXT,
|
||||
export_manifest_json TEXT NOT NULL DEFAULT '{}',
|
||||
export_pending_json TEXT NOT NULL DEFAULT '[]',
|
||||
next_utterance_seq INTEGER NOT NULL DEFAULT 0 CHECK (next_utterance_seq >= 0),
|
||||
created_at_ms INTEGER NOT NULL CHECK (created_at_ms >= 0),
|
||||
updated_at_ms INTEGER NOT NULL CHECK (updated_at_ms >= 0),
|
||||
PRIMARY KEY (session_id, started_at)
|
||||
) STRICT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_started
|
||||
ON meeting_transcript_sessions(started_at DESC, session_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_id
|
||||
ON meeting_transcript_sessions(session_id, started_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_slug
|
||||
ON meeting_transcript_sessions(session_slug, started_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_export_key
|
||||
ON meeting_transcript_sessions(export_key);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meeting_transcript_utterances (
|
||||
session_id TEXT NOT NULL,
|
||||
session_started_at TEXT NOT NULL,
|
||||
sequence INTEGER NOT NULL CHECK (sequence >= 0),
|
||||
utterance_id TEXT,
|
||||
started_at TEXT,
|
||||
ended_at TEXT,
|
||||
speaker_id TEXT,
|
||||
speaker_label TEXT,
|
||||
text TEXT NOT NULL,
|
||||
final INTEGER CHECK (final IN (0, 1)),
|
||||
metadata_json TEXT,
|
||||
PRIMARY KEY (session_id, session_started_at, sequence),
|
||||
FOREIGN KEY (session_id, session_started_at)
|
||||
REFERENCES meeting_transcript_sessions(session_id, started_at)
|
||||
ON DELETE CASCADE
|
||||
) STRICT;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meeting_transcript_summaries (
|
||||
session_id TEXT NOT NULL,
|
||||
session_started_at TEXT NOT NULL,
|
||||
generated_at TEXT,
|
||||
summary_json TEXT,
|
||||
markdown TEXT,
|
||||
utterance_count INTEGER NOT NULL CHECK (utterance_count >= 0),
|
||||
PRIMARY KEY (session_id, session_started_at),
|
||||
FOREIGN KEY (session_id, session_started_at)
|
||||
REFERENCES meeting_transcript_sessions(session_id, started_at)
|
||||
ON DELETE CASCADE,
|
||||
CHECK (summary_json IS NOT NULL OR markdown IS NOT NULL)
|
||||
) STRICT;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS migration_runs (
|
||||
id TEXT NOT NULL PRIMARY KEY,
|
||||
started_at INTEGER NOT NULL,
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// Additive meeting-transcript schema used by the feature's one-time lazy ensure.
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import {
|
||||
openOpenClawStateDatabase,
|
||||
runOpenClawStateWriteTransaction,
|
||||
type OpenClawStateDatabaseOptions,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
|
||||
const ensuredDatabases = new WeakSet<DatabaseSync>();
|
||||
const MEETING_TRANSCRIPTS_SCHEMA_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS meeting_transcript_sessions (
|
||||
session_id TEXT NOT NULL,
|
||||
started_at TEXT NOT NULL,
|
||||
selector TEXT NOT NULL UNIQUE,
|
||||
export_key TEXT NOT NULL,
|
||||
session_slug TEXT NOT NULL,
|
||||
provider_id TEXT NOT NULL,
|
||||
title TEXT,
|
||||
source_json TEXT NOT NULL,
|
||||
stopped_at TEXT,
|
||||
metadata_json TEXT,
|
||||
export_manifest_json TEXT NOT NULL DEFAULT '{}',
|
||||
export_pending_json TEXT NOT NULL DEFAULT '[]',
|
||||
next_utterance_seq INTEGER NOT NULL DEFAULT 0 CHECK (next_utterance_seq >= 0),
|
||||
created_at_ms INTEGER NOT NULL CHECK (created_at_ms >= 0),
|
||||
updated_at_ms INTEGER NOT NULL CHECK (updated_at_ms >= 0),
|
||||
PRIMARY KEY (session_id, started_at)
|
||||
) STRICT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_started
|
||||
ON meeting_transcript_sessions(started_at DESC, session_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_id
|
||||
ON meeting_transcript_sessions(session_id, started_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_slug
|
||||
ON meeting_transcript_sessions(session_slug, started_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_meeting_transcript_sessions_export_key
|
||||
ON meeting_transcript_sessions(export_key);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meeting_transcript_utterances (
|
||||
session_id TEXT NOT NULL,
|
||||
session_started_at TEXT NOT NULL,
|
||||
sequence INTEGER NOT NULL CHECK (sequence >= 0),
|
||||
utterance_id TEXT,
|
||||
started_at TEXT,
|
||||
ended_at TEXT,
|
||||
speaker_id TEXT,
|
||||
speaker_label TEXT,
|
||||
text TEXT NOT NULL,
|
||||
final INTEGER CHECK (final IN (0, 1)),
|
||||
metadata_json TEXT,
|
||||
PRIMARY KEY (session_id, session_started_at, sequence),
|
||||
FOREIGN KEY (session_id, session_started_at)
|
||||
REFERENCES meeting_transcript_sessions(session_id, started_at)
|
||||
ON DELETE CASCADE
|
||||
) STRICT;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meeting_transcript_summaries (
|
||||
session_id TEXT NOT NULL,
|
||||
session_started_at TEXT NOT NULL,
|
||||
generated_at TEXT,
|
||||
summary_json TEXT,
|
||||
markdown TEXT,
|
||||
utterance_count INTEGER NOT NULL CHECK (utterance_count >= 0),
|
||||
PRIMARY KEY (session_id, session_started_at),
|
||||
FOREIGN KEY (session_id, session_started_at)
|
||||
REFERENCES meeting_transcript_sessions(session_id, started_at)
|
||||
ON DELETE CASCADE,
|
||||
CHECK (summary_json IS NOT NULL OR markdown IS NOT NULL)
|
||||
) STRICT;
|
||||
`;
|
||||
|
||||
export function ensureMeetingTranscriptsSchema(options: OpenClawStateDatabaseOptions = {}): void {
|
||||
const database = openOpenClawStateDatabase(options);
|
||||
if (ensuredDatabases.has(database.db)) {
|
||||
return;
|
||||
}
|
||||
runOpenClawStateWriteTransaction(({ db }) => db.exec(MEETING_TRANSCRIPTS_SCHEMA_SQL), options, {
|
||||
operationLabel: "meeting-transcripts.schema.ensure",
|
||||
});
|
||||
ensuredDatabases.add(database.db);
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { sha256Hex } from "../infra/crypto-digest.js";
|
||||
import { writeExternalFileWithinRoot } from "../infra/fs-safe.js";
|
||||
import type { TranscriptSessionDescriptor } from "./provider-types.js";
|
||||
|
||||
export const TRANSCRIPT_EXPORT_FILE_NAMES = new Set([
|
||||
"metadata.json",
|
||||
"summary.json",
|
||||
"summary.md",
|
||||
"transcript.jsonl",
|
||||
]);
|
||||
|
||||
export function safeTranscriptPathSegment(value: string): string {
|
||||
const segment = value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
||||
if (segment === ".") {
|
||||
return "%2E";
|
||||
}
|
||||
if (segment === "..") {
|
||||
return "%2E%2E";
|
||||
}
|
||||
if (!segment) {
|
||||
return "session";
|
||||
}
|
||||
if (segment.endsWith(".") || /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?$/iu.test(segment)) {
|
||||
return Buffer.from(segment, "utf8")
|
||||
.toString("hex")
|
||||
.match(/.{2}/gu)!
|
||||
.map((byte) => `%${byte.toUpperCase()}`)
|
||||
.join("");
|
||||
}
|
||||
return segment;
|
||||
}
|
||||
|
||||
function legacyTranscriptPathSegment(value: string): string {
|
||||
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "session";
|
||||
}
|
||||
|
||||
function dateSegment(value: string | undefined): string {
|
||||
const isoDate = value?.match(/^(\d{4}-\d{2}-\d{2})T/)?.[1];
|
||||
return isoDate ?? new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
export function transcriptSessionSelector(session: TranscriptSessionDescriptor): string {
|
||||
return `${dateSegment(session.startedAt)}/${safeTranscriptPathSegment(session.sessionId)}`;
|
||||
}
|
||||
|
||||
export function legacyTranscriptSessionSelector(session: TranscriptSessionDescriptor): string {
|
||||
const date = dateSegment(session.startedAt);
|
||||
const segment = legacyTranscriptPathSegment(session.sessionId);
|
||||
// The shipped sanitizer allowed dot components: `.` collapsed to the date
|
||||
// directory and `..` collapsed to the transcript root.
|
||||
if (segment === ".") {
|
||||
return date;
|
||||
}
|
||||
if (segment === "..") {
|
||||
return ".";
|
||||
}
|
||||
return `${date}/${segment}`;
|
||||
}
|
||||
|
||||
export function transcriptSessionExportKey(session: TranscriptSessionDescriptor): string {
|
||||
return transcriptSessionSelector(session).toLowerCase();
|
||||
}
|
||||
|
||||
export function normalizeExportText(value: string): string {
|
||||
return value.endsWith("\n") ? value : `${value}\n`;
|
||||
}
|
||||
|
||||
export async function writeTranscriptArtifact(
|
||||
rootDir: string,
|
||||
fileName: string,
|
||||
content: string,
|
||||
): Promise<string> {
|
||||
await writeExternalFileWithinRoot({
|
||||
rootDir,
|
||||
path: fileName,
|
||||
write: async (tempPath) => await fs.writeFile(tempPath, content, { mode: 0o600 }),
|
||||
});
|
||||
return sha256Hex(content);
|
||||
}
|
||||
|
||||
export async function removeTranscriptArtifact(rootDir: string, fileName: string): Promise<void> {
|
||||
await fs.rm(path.join(rootDir, fileName), { force: true });
|
||||
}
|
||||
|
||||
export async function isCaseSensitiveDirectory(directory: string): Promise<boolean> {
|
||||
const probeName = `.openclaw-case-probe-${randomUUID().toLowerCase()}`;
|
||||
const probePath = path.join(directory, probeName);
|
||||
const alternatePath = path.join(directory, probeName.toUpperCase());
|
||||
const handle = await fs.open(probePath, "wx", 0o600);
|
||||
await handle.close();
|
||||
try {
|
||||
try {
|
||||
await fs.access(alternatePath);
|
||||
return false;
|
||||
} catch (error) {
|
||||
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
|
||||
return true;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
await fs.rm(probePath, { force: true });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import { writeExternalFileWithinRoot } from "../infra/fs-safe.js";
|
||||
import { executeSqliteQuerySync, executeSqliteQueryTakeFirstSync } from "../infra/kysely-sync.js";
|
||||
import {
|
||||
openOpenClawStateDatabase,
|
||||
type OpenClawStateDatabaseOptions,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import type { TranscriptSessionDescriptor } from "./provider-types.js";
|
||||
import { ensureMeetingTranscriptsSchema } from "./sqlite-schema.js";
|
||||
import { meetingTranscriptDb, utteranceFromRow } from "./store-sqlite.js";
|
||||
|
||||
const TRANSCRIPT_EXPORT_ROW_BATCH_SIZE = 64;
|
||||
|
||||
export async function writeTranscriptJsonlArtifact(params: {
|
||||
sessionDir: string;
|
||||
session: TranscriptSessionDescriptor;
|
||||
databaseOptions: OpenClawStateDatabaseOptions;
|
||||
}): Promise<string> {
|
||||
ensureMeetingTranscriptsSchema(params.databaseOptions);
|
||||
const database = openOpenClawStateDatabase(params.databaseOptions);
|
||||
const sequenceHead = executeSqliteQueryTakeFirstSync(
|
||||
database.db,
|
||||
meetingTranscriptDb(database.db)
|
||||
.selectFrom("meeting_transcript_sessions")
|
||||
.select("next_utterance_seq")
|
||||
.where("session_id", "=", params.session.sessionId)
|
||||
.where("started_at", "=", params.session.startedAt),
|
||||
)?.next_utterance_seq;
|
||||
if (sequenceHead === undefined) {
|
||||
throw new Error(`transcripts session not found: ${params.session.sessionId}`);
|
||||
}
|
||||
const digest = createHash("sha256");
|
||||
await writeExternalFileWithinRoot({
|
||||
rootDir: params.sessionDir,
|
||||
path: "transcript.jsonl",
|
||||
write: async (tempPath) => {
|
||||
const handle = await fs.open(tempPath, "w", 0o600);
|
||||
try {
|
||||
let nextSequence = 0;
|
||||
while (nextSequence < sequenceHead) {
|
||||
const rows = executeSqliteQuerySync(
|
||||
database.db,
|
||||
meetingTranscriptDb(database.db)
|
||||
.selectFrom("meeting_transcript_utterances")
|
||||
.selectAll()
|
||||
.where("session_id", "=", params.session.sessionId)
|
||||
.where("session_started_at", "=", params.session.startedAt)
|
||||
.where("sequence", ">=", nextSequence)
|
||||
.where("sequence", "<", sequenceHead)
|
||||
.orderBy("sequence", "asc")
|
||||
.limit(TRANSCRIPT_EXPORT_ROW_BATCH_SIZE),
|
||||
).rows;
|
||||
if (rows.length === 0) {
|
||||
break;
|
||||
}
|
||||
nextSequence = rows.at(-1)!.sequence + 1;
|
||||
const lines = rows.map((row) => `${JSON.stringify(utteranceFromRow(row))}\n`);
|
||||
for (const line of lines) {
|
||||
await handle.writeFile(line);
|
||||
digest.update(line);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
},
|
||||
});
|
||||
return digest.digest("hex");
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
import { constants as fsConstants } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { sha256File } from "../infra/crypto-digest.js";
|
||||
import { ensureAbsoluteDirectory } from "../infra/fs-safe.js";
|
||||
import { executeSqliteQuerySync } from "../infra/kysely-sync.js";
|
||||
import {
|
||||
openOpenClawStateDatabase,
|
||||
type OpenClawStateDatabaseOptions,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import type { TranscriptSessionDescriptor } from "./provider-types.js";
|
||||
import { ensureMeetingTranscriptsSchema } from "./sqlite-schema.js";
|
||||
import {
|
||||
isCaseSensitiveDirectory,
|
||||
transcriptSessionExportKey,
|
||||
transcriptSessionSelector,
|
||||
} from "./store-artifacts.js";
|
||||
import { meetingTranscriptDb } from "./store-sqlite.js";
|
||||
|
||||
type ExportOwnershipParams = {
|
||||
session: TranscriptSessionDescriptor;
|
||||
exportRootDir: string;
|
||||
databaseOptions: OpenClawStateDatabaseOptions;
|
||||
};
|
||||
|
||||
const TRANSCRIPT_EXPORT_FILE_NAMES = new Set([
|
||||
"metadata.json",
|
||||
"summary.json",
|
||||
"summary.md",
|
||||
"transcript.jsonl",
|
||||
]);
|
||||
|
||||
function database(options: OpenClawStateDatabaseOptions) {
|
||||
ensureMeetingTranscriptsSchema(options);
|
||||
return openOpenClawStateDatabase(options);
|
||||
}
|
||||
|
||||
export async function assertTranscriptExportPathAvailable(
|
||||
params: ExportOwnershipParams,
|
||||
): Promise<void> {
|
||||
const stateDatabase = database(params.databaseOptions);
|
||||
const collisions = executeSqliteQuerySync(
|
||||
stateDatabase.db,
|
||||
meetingTranscriptDb(stateDatabase.db)
|
||||
.selectFrom("meeting_transcript_sessions")
|
||||
.select(["session_id", "started_at", "selector", "export_pending_json"])
|
||||
.where("export_key", "=", transcriptSessionExportKey(params.session))
|
||||
.orderBy("selector", "asc"),
|
||||
).rows;
|
||||
if (collisions.length <= 1) {
|
||||
return;
|
||||
}
|
||||
const ensured = await ensureAbsoluteDirectory(params.exportRootDir, {
|
||||
mode: 0o700,
|
||||
scopeLabel: "transcript export root",
|
||||
});
|
||||
if (!ensured.ok) {
|
||||
throw ensured.error;
|
||||
}
|
||||
if (await isCaseSensitiveDirectory(params.exportRootDir)) {
|
||||
return;
|
||||
}
|
||||
let ownerSelector: string | undefined;
|
||||
try {
|
||||
const metadata = JSON.parse(
|
||||
await fs.readFile(
|
||||
path.join(params.exportRootDir, collisions[0]!.selector, "metadata.json"),
|
||||
"utf8",
|
||||
),
|
||||
) as { sessionId?: unknown; startedAt?: unknown };
|
||||
ownerSelector = collisions.find(
|
||||
(row) => row.session_id === metadata.sessionId && row.started_at === metadata.startedAt,
|
||||
)?.selector;
|
||||
} catch (error) {
|
||||
if (!(error && typeof error === "object" && "code" in error && error.code === "ENOENT")) {
|
||||
if (!(error instanceof SyntaxError)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!ownerSelector) {
|
||||
const pendingOwners = collisions.filter((row) =>
|
||||
(JSON.parse(row.export_pending_json) as string[]).includes("metadata.json"),
|
||||
);
|
||||
if (pendingOwners.length === 1) {
|
||||
ownerSelector = pendingOwners[0]?.selector;
|
||||
}
|
||||
}
|
||||
ownerSelector ??= transcriptSessionSelector(params.session);
|
||||
if (ownerSelector !== transcriptSessionSelector(params.session)) {
|
||||
throw new Error(
|
||||
`transcript export path collides case-insensitively with another session: ${path.join(params.exportRootDir, transcriptSessionSelector(params.session))}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function hasAliasedCanonicalTranscriptExportPathOwner(
|
||||
params: ExportOwnershipParams,
|
||||
): Promise<boolean> {
|
||||
const stateDatabase = database(params.databaseOptions);
|
||||
const owners = executeSqliteQuerySync(
|
||||
stateDatabase.db,
|
||||
meetingTranscriptDb(stateDatabase.db)
|
||||
.selectFrom("meeting_transcript_sessions")
|
||||
.select(["session_id", "started_at", "export_manifest_json", "export_pending_json"])
|
||||
.where("export_key", "=", transcriptSessionExportKey(params.session))
|
||||
.orderBy("selector", "asc"),
|
||||
).rows;
|
||||
if (owners.length === 0) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
await fs.access(params.exportRootDir);
|
||||
} catch (error) {
|
||||
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (await isCaseSensitiveDirectory(params.exportRootDir)) {
|
||||
return false;
|
||||
}
|
||||
const sessionDir = path.join(params.exportRootDir, transcriptSessionSelector(params.session));
|
||||
let entries;
|
||||
try {
|
||||
entries = await fs.readdir(sessionDir, { withFileTypes: true });
|
||||
} catch (error) {
|
||||
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
|
||||
return true;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const artifactCaseSensitive = await isCaseSensitiveDirectory(sessionDir);
|
||||
const artifacts = entries.flatMap((entry) => {
|
||||
const canonicalName = artifactCaseSensitive ? entry.name : entry.name.toLowerCase();
|
||||
return TRANSCRIPT_EXPORT_FILE_NAMES.has(canonicalName) ? [{ entry, canonicalName }] : [];
|
||||
});
|
||||
if (artifacts.length === 0) {
|
||||
return true;
|
||||
}
|
||||
let owner;
|
||||
let identityVerified = false;
|
||||
const metadataArtifact = artifacts.find(({ canonicalName }) => canonicalName === "metadata.json");
|
||||
if (metadataArtifact) {
|
||||
const metadataPath = path.join(sessionDir, metadataArtifact.entry.name);
|
||||
const metadataStat = await fs.lstat(metadataPath);
|
||||
if (metadataStat.isSymbolicLink() || !metadataStat.isFile()) {
|
||||
return false;
|
||||
}
|
||||
let handle;
|
||||
try {
|
||||
handle = await fs.open(metadataPath, fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0));
|
||||
const metadata = JSON.parse(await handle.readFile("utf8")) as {
|
||||
sessionId?: unknown;
|
||||
startedAt?: unknown;
|
||||
};
|
||||
owner = owners.find(
|
||||
(row) => row.session_id === metadata.sessionId && row.started_at === metadata.startedAt,
|
||||
);
|
||||
identityVerified = owner !== undefined;
|
||||
} catch {
|
||||
return false;
|
||||
} finally {
|
||||
await handle?.close();
|
||||
}
|
||||
}
|
||||
if (!owner && !metadataArtifact) {
|
||||
const manifestMatches = [];
|
||||
for (const candidate of owners) {
|
||||
const candidateManifest = JSON.parse(candidate.export_manifest_json) as Record<
|
||||
string,
|
||||
string
|
||||
>;
|
||||
const candidatePending = new Set(JSON.parse(candidate.export_pending_json) as string[]);
|
||||
let verified = 0;
|
||||
let matches = true;
|
||||
for (const { entry, canonicalName } of artifacts) {
|
||||
const artifactPath = path.join(sessionDir, entry.name);
|
||||
const stat = await fs.lstat(artifactPath);
|
||||
const expectedHash = candidateManifest[canonicalName];
|
||||
if (
|
||||
stat.isSymbolicLink() ||
|
||||
!stat.isFile() ||
|
||||
candidatePending.has(canonicalName) ||
|
||||
!expectedHash ||
|
||||
(await sha256File(artifactPath)) !== expectedHash
|
||||
) {
|
||||
matches = false;
|
||||
break;
|
||||
}
|
||||
verified += 1;
|
||||
}
|
||||
if (matches && verified > 0) {
|
||||
manifestMatches.push(candidate);
|
||||
}
|
||||
}
|
||||
owner = manifestMatches.length === 1 ? manifestMatches[0] : undefined;
|
||||
}
|
||||
if (!owner) {
|
||||
return false;
|
||||
}
|
||||
const manifest = JSON.parse(owner.export_manifest_json) as Record<string, string>;
|
||||
const pending = new Set(JSON.parse(owner.export_pending_json) as string[]);
|
||||
let verifiedArtifactCount = 0;
|
||||
for (const { entry, canonicalName } of artifacts) {
|
||||
const artifactPath = path.join(sessionDir, entry.name);
|
||||
const stat = await fs.lstat(artifactPath);
|
||||
if (stat.isSymbolicLink() || !stat.isFile()) {
|
||||
return false;
|
||||
}
|
||||
if (pending.has(canonicalName)) {
|
||||
return false;
|
||||
}
|
||||
const expectedHash = manifest[canonicalName];
|
||||
if (!expectedHash || (await sha256File(artifactPath)) !== expectedHash) {
|
||||
return false;
|
||||
}
|
||||
verifiedArtifactCount += 1;
|
||||
}
|
||||
return identityVerified || verifiedArtifactCount > 0;
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { Selectable } from "kysely";
|
||||
import { getNodeSqliteKysely } from "../infra/kysely-sync.js";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
|
||||
import type { TranscriptSessionDescriptor, TranscriptUtterance } from "./provider-types.js";
|
||||
import type { TranscriptsSummary } from "./summary.js";
|
||||
|
||||
type MeetingTranscriptsDatabase = Pick<
|
||||
OpenClawStateKyselyDatabase,
|
||||
"meeting_transcript_sessions" | "meeting_transcript_summaries" | "meeting_transcript_utterances"
|
||||
>;
|
||||
|
||||
export type MeetingTranscriptSessionRow = Selectable<
|
||||
OpenClawStateKyselyDatabase["meeting_transcript_sessions"]
|
||||
>;
|
||||
type MeetingTranscriptSummaryRow = Selectable<
|
||||
OpenClawStateKyselyDatabase["meeting_transcript_summaries"]
|
||||
>;
|
||||
type MeetingTranscriptUtteranceRow = Selectable<
|
||||
OpenClawStateKyselyDatabase["meeting_transcript_utterances"]
|
||||
>;
|
||||
|
||||
export function meetingTranscriptDb(db: DatabaseSync) {
|
||||
return getNodeSqliteKysely<MeetingTranscriptsDatabase>(db);
|
||||
}
|
||||
|
||||
function parseOptionalJsonRecord(value: string | null): Record<string, unknown> | undefined {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
const parsed = JSON.parse(value) as unknown;
|
||||
return parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
export function sessionFromRow(row: MeetingTranscriptSessionRow): TranscriptSessionDescriptor {
|
||||
const source = parseOptionalJsonRecord(row.source_json);
|
||||
const metadata = parseOptionalJsonRecord(row.metadata_json);
|
||||
if (!source || typeof source.providerId !== "string") {
|
||||
throw new Error(`invalid meeting transcript source for ${row.session_id}`);
|
||||
}
|
||||
return {
|
||||
sessionId: row.session_id,
|
||||
source: source as TranscriptSessionDescriptor["source"],
|
||||
startedAt: row.started_at,
|
||||
...(row.title !== null ? { title: row.title } : {}),
|
||||
...(row.stopped_at !== null ? { stoppedAt: row.stopped_at } : {}),
|
||||
...(metadata ? { metadata } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function utteranceFromRow(row: MeetingTranscriptUtteranceRow): TranscriptUtterance {
|
||||
const speaker =
|
||||
row.speaker_label !== null
|
||||
? {
|
||||
label: row.speaker_label,
|
||||
...(row.speaker_id !== null ? { id: row.speaker_id } : {}),
|
||||
}
|
||||
: undefined;
|
||||
const metadata = parseOptionalJsonRecord(row.metadata_json);
|
||||
return {
|
||||
sessionId: row.session_id,
|
||||
text: row.text,
|
||||
...(row.utterance_id !== null ? { id: row.utterance_id } : {}),
|
||||
...(row.started_at !== null ? { startedAt: row.started_at } : {}),
|
||||
...(row.ended_at !== null ? { endedAt: row.ended_at } : {}),
|
||||
...(speaker ? { speaker } : {}),
|
||||
...(row.final === null ? {} : { final: row.final === 1 }),
|
||||
...(metadata ? { metadata } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function summaryFromRow(row: MeetingTranscriptSummaryRow): TranscriptsSummary | undefined {
|
||||
return row.summary_json ? (JSON.parse(row.summary_json) as TranscriptsSummary) : undefined;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { TranscriptSessionDescriptor } from "./provider-types.js";
|
||||
|
||||
export type TranscriptsSessionEntry = {
|
||||
session: TranscriptSessionDescriptor;
|
||||
sessionDir: string;
|
||||
selector: string;
|
||||
summaryPath: string;
|
||||
hasSummary: boolean;
|
||||
};
|
||||
|
||||
export type TranscriptArtifactKind = "all" | "metadata" | "summary" | "transcript";
|
||||
|
||||
export type MaterializedTranscriptArtifacts = {
|
||||
sessionDir: string;
|
||||
metadataPath: string;
|
||||
transcriptPath: string;
|
||||
summaryJsonPath: string;
|
||||
summaryPath: string;
|
||||
hasSummary: boolean;
|
||||
};
|
||||
+337
-142
@@ -1,172 +1,367 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
// Tests TranscriptsStore stream cleanup and transcript reading behavior.
|
||||
import { PassThrough } from "node:stream";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const createReadStreamMock = vi.hoisted(() => vi.fn());
|
||||
vi.mock("node:fs", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("node:fs")>();
|
||||
return {
|
||||
...actual,
|
||||
createReadStream: (...args: Parameters<typeof actual.createReadStream>) =>
|
||||
createReadStreamMock(...args) ?? actual.createReadStream(...args),
|
||||
};
|
||||
});
|
||||
import { listOpenFileDescriptorsForPath } from "../../src/infra/open-file-descriptors.test-support.js";
|
||||
import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js";
|
||||
import { TranscriptsStore } from "./store.js";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import type { TranscriptSessionDescriptor } from "./provider-types.js";
|
||||
import { safeTranscriptPathSegment, TranscriptsStore } from "./store.js";
|
||||
import { summarizeTranscripts } from "./summary.js";
|
||||
|
||||
const tempRoots: string[] = [];
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
describe("TranscriptsStore.readUtterancesFromSessionDir", () => {
|
||||
afterEach(() => {
|
||||
cleanupTempDirs(tempRoots);
|
||||
afterEach(() => closeOpenClawStateDatabaseForTest());
|
||||
|
||||
function createStore(): { stateDir: string; store: TranscriptsStore } {
|
||||
const stateDir = tempDirs.make("openclaw-transcript-test-");
|
||||
return {
|
||||
stateDir,
|
||||
store: new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function session(
|
||||
sessionId = "session-1",
|
||||
startedAt = "2026-07-01T10:00:00.000Z",
|
||||
): TranscriptSessionDescriptor {
|
||||
return {
|
||||
sessionId,
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt,
|
||||
};
|
||||
}
|
||||
|
||||
describe("TranscriptsStore", () => {
|
||||
it("encodes portable slugs for Windows-reserved and trailing-dot IDs", () => {
|
||||
expect(safeTranscriptPathSegment("CON")).toBe("%43%4F%4E");
|
||||
expect(safeTranscriptPathSegment("foo.")).toBe("%66%6F%6F%2E");
|
||||
expect(safeTranscriptPathSegment("foo")).toBe("foo");
|
||||
});
|
||||
|
||||
it("returns an empty array when transcript.jsonl is missing", () => {
|
||||
const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-");
|
||||
const store = new TranscriptsStore(tmpDir);
|
||||
const sessionDir = path.join(tmpDir, "2026-07-01", "missing");
|
||||
fs.mkdirSync(sessionDir, { recursive: true });
|
||||
it("persists sessions and utterances only in SQLite until export", async () => {
|
||||
const { stateDir, store } = createStore();
|
||||
const target = session();
|
||||
|
||||
const result = store.readUtterancesFromSessionDir(sessionDir, { maxUtterances: 10 });
|
||||
await store.writeSession(target);
|
||||
await store.appendUtteranceForSession(target, { text: "hello", final: true });
|
||||
await store.appendUtteranceForSession(target, { text: "world", final: true });
|
||||
|
||||
return expect(result).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it("reads utterances from transcript.jsonl", () => {
|
||||
const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-");
|
||||
const store = new TranscriptsStore(tmpDir);
|
||||
const sessionDir = path.join(tmpDir, "2026-07-01", "session-1");
|
||||
fs.mkdirSync(sessionDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(sessionDir, "transcript.jsonl"),
|
||||
[
|
||||
JSON.stringify({ text: "hello", sessionId: "session-1" }),
|
||||
JSON.stringify({ text: "world", sessionId: "session-1" }),
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
|
||||
const result = store.readUtterancesFromSessionDir(sessionDir, { maxUtterances: 10 });
|
||||
|
||||
return expect(result).resolves.toEqual([
|
||||
expect.objectContaining({ text: "hello" }),
|
||||
expect.objectContaining({ text: "world" }),
|
||||
expect(await store.readSession(target.sessionId)).toEqual(target);
|
||||
expect(await store.readUtterancesForSession(target)).toEqual([
|
||||
{ sessionId: target.sessionId, text: "hello", final: true },
|
||||
{ sessionId: target.sessionId, text: "world", final: true },
|
||||
]);
|
||||
expect(fs.existsSync(path.join(stateDir, "transcripts"))).toBe(false);
|
||||
expect(fs.existsSync(path.join(stateDir, "state", "openclaw.sqlite"))).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps only the tail when utterances exceed maxUtterances", () => {
|
||||
const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-");
|
||||
const store = new TranscriptsStore(tmpDir);
|
||||
const sessionDir = path.join(tmpDir, "2026-07-01", "session-1");
|
||||
fs.mkdirSync(sessionDir, { recursive: true });
|
||||
const lines = Array.from({ length: 5 }, (_, i) =>
|
||||
JSON.stringify({ text: `line-${i}`, sessionId: "session-1" }),
|
||||
);
|
||||
fs.writeFileSync(path.join(sessionDir, "transcript.jsonl"), lines.join("\n") + "\n");
|
||||
it("returns the requested ordered utterance tail", async () => {
|
||||
const { store } = createStore();
|
||||
const target = session();
|
||||
await store.writeSession(target);
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
await store.appendUtteranceForSession(target, { text: `line-${index}` });
|
||||
}
|
||||
|
||||
const result = store.readUtterancesFromSessionDir(sessionDir, { maxUtterances: 2 });
|
||||
|
||||
return expect(result).resolves.toEqual([
|
||||
await expect(store.readUtterancesForSession(target, { maxUtterances: 2 })).resolves.toEqual([
|
||||
expect.objectContaining({ text: "line-3" }),
|
||||
expect.objectContaining({ text: "line-4" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it.runIf(process.platform === "linux")(
|
||||
"does not leak file descriptors when JSON.parse throws",
|
||||
async () => {
|
||||
const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-");
|
||||
const store = new TranscriptsStore(tmpDir);
|
||||
const sessionDir = path.join(tmpDir, "2026-07-01", "session-1");
|
||||
fs.mkdirSync(sessionDir, { recursive: true });
|
||||
const transcriptPath = path.join(sessionDir, "transcript.jsonl");
|
||||
fs.writeFileSync(transcriptPath, "not valid json\n");
|
||||
it("requires date-qualified selectors for repeated ids", async () => {
|
||||
const { store } = createStore();
|
||||
await store.writeSession(session("standup", "2026-07-01T10:00:00.000Z"));
|
||||
await store.writeSession(session("standup", "2026-07-02T10:00:00.000Z"));
|
||||
|
||||
const fdsBefore = listOpenFileDescriptorsForPath(sessionDir);
|
||||
await expect(
|
||||
store.readUtterancesFromSessionDir(sessionDir, { maxUtterances: 10 }),
|
||||
).rejects.toThrow();
|
||||
const fdsAfter = listOpenFileDescriptorsForPath(sessionDir);
|
||||
|
||||
const leaked = fdsAfter.filter((p) => !fdsBefore.includes(p));
|
||||
expect(leaked).toHaveLength(0);
|
||||
},
|
||||
);
|
||||
|
||||
it.runIf(process.platform === "linux")(
|
||||
"does not leak file descriptors in the happy path",
|
||||
async () => {
|
||||
const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-");
|
||||
const store = new TranscriptsStore(tmpDir);
|
||||
const sessionDir = path.join(tmpDir, "2026-07-01", "session-1");
|
||||
fs.mkdirSync(sessionDir, { recursive: true });
|
||||
const transcriptPath = path.join(sessionDir, "transcript.jsonl");
|
||||
fs.writeFileSync(
|
||||
transcriptPath,
|
||||
JSON.stringify({ text: "hello", sessionId: "session-1" }) + "\n",
|
||||
);
|
||||
|
||||
const fdsBefore = listOpenFileDescriptorsForPath(sessionDir);
|
||||
await store.readUtterancesFromSessionDir(sessionDir, { maxUtterances: 10 });
|
||||
const fdsAfter = listOpenFileDescriptorsForPath(sessionDir);
|
||||
|
||||
const leaked = fdsAfter.filter((p) => !fdsBefore.includes(p));
|
||||
expect(leaked).toHaveLength(0);
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects non-ENOENT read stream errors", async () => {
|
||||
const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-");
|
||||
const store = new TranscriptsStore(tmpDir);
|
||||
const sessionDir = path.join(tmpDir, "2026-07-01", "session-1");
|
||||
fs.mkdirSync(sessionDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(sessionDir, "transcript.jsonl"), "");
|
||||
|
||||
createReadStreamMock.mockImplementation(() => {
|
||||
const stream = new PassThrough();
|
||||
setTimeout(() => {
|
||||
stream.write(JSON.stringify({ text: "hello", sessionId: "session-1" }) + "\n");
|
||||
stream.destroy(new Error("read failed"));
|
||||
}, 10);
|
||||
return stream;
|
||||
await expect(store.readSession("standup")).rejects.toThrow(
|
||||
"multiple transcripts sessions match standup",
|
||||
);
|
||||
await expect(store.readSession("2026-07-01/standup")).resolves.toMatchObject({
|
||||
startedAt: "2026-07-01T10:00:00.000Z",
|
||||
});
|
||||
});
|
||||
|
||||
it("matches bare selector slugs literally and case-sensitively", async () => {
|
||||
const { store } = createStore();
|
||||
await store.writeSession(session("fooXbar"));
|
||||
await store.writeSession(session("Capital", "2026-07-02T10:00:00.000Z"));
|
||||
await store.writeSession(session("foo@bar", "2026-07-03T10:00:00.000Z"));
|
||||
await store.writeSession(session("foo-bar", "2026-07-04T10:00:00.000Z"));
|
||||
|
||||
await expect(store.readSession("foo_bar")).resolves.toBeUndefined();
|
||||
await expect(store.readSession("capital")).resolves.toBeUndefined();
|
||||
await expect(store.readSession("foo#bar")).resolves.toBeUndefined();
|
||||
await expect(store.readSession("foo@bar")).resolves.toMatchObject({ sessionId: "foo@bar" });
|
||||
await expect(store.readSession("foo-bar")).rejects.toThrow(
|
||||
"multiple transcripts sessions match foo-bar",
|
||||
);
|
||||
});
|
||||
|
||||
it("round-trips empty nullable text values", async () => {
|
||||
const { store } = createStore();
|
||||
const target = { ...session("empty-values"), title: "" };
|
||||
await store.writeSession(target);
|
||||
await store.appendUtteranceForSession(target, {
|
||||
id: "",
|
||||
speaker: { id: "", label: "" },
|
||||
text: "",
|
||||
});
|
||||
|
||||
await expect(store.readSession("empty-values")).resolves.toEqual(target);
|
||||
await expect(store.readUtterancesForSession(target)).resolves.toEqual([
|
||||
{ id: "", sessionId: "empty-values", speaker: { id: "", label: "" }, text: "" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects two session identities that map to one shipped selector", async () => {
|
||||
const { store } = createStore();
|
||||
await store.writeSession(session("standup", "2026-07-01T10:00:00.000Z"));
|
||||
|
||||
await expect(
|
||||
store.readUtterancesFromSessionDir(sessionDir, { maxUtterances: 10 }),
|
||||
).rejects.toThrow("read failed");
|
||||
});
|
||||
});
|
||||
|
||||
describe("TranscriptsStore.writeSummary", () => {
|
||||
afterEach(() => {
|
||||
cleanupTempDirs(tempRoots);
|
||||
store.writeSession(session("standup", "2026-07-01T11:00:00.000Z")),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("stores the summary in the existing session directory without a session descriptor", async () => {
|
||||
const tmpDir = makeTempDir(tempRoots, "openclaw-transcript-test-");
|
||||
const store = new TranscriptsStore(tmpDir);
|
||||
const session = {
|
||||
sessionId: "ansi-\u001b[31mprovider\u001b[0m",
|
||||
title: "ANSI import",
|
||||
source: { providerId: "manual-transcript" },
|
||||
startedAt: "2026-05-22T10:00:00.000Z",
|
||||
};
|
||||
await store.writeSession(session);
|
||||
const summary = summarizeTranscripts({
|
||||
session,
|
||||
utterances: [{ text: "We decided to ship the CLI.", speaker: { label: "Sam" } }],
|
||||
it("stores case-distinct sessions and rejects only unsafe export collisions", async () => {
|
||||
const { store } = createStore();
|
||||
const upper = session("Capital", "2026-07-01T10:00:00.000Z");
|
||||
const lower = session("capital", "2026-07-01T11:00:00.000Z");
|
||||
await store.writeSession(lower);
|
||||
await store.materializeSessionArtifacts(lower, "metadata");
|
||||
await expect(store.writeSession(upper)).resolves.toBeUndefined();
|
||||
|
||||
if (fs.existsSync(store.sessionDir(upper))) {
|
||||
await expect(store.materializeSessionArtifacts(lower, "metadata")).resolves.toMatchObject({
|
||||
metadataPath: path.join(store.sessionDir(lower), "metadata.json"),
|
||||
});
|
||||
await expect(store.materializeSessionArtifacts(upper, "metadata")).rejects.toThrow(
|
||||
"collides case-insensitively",
|
||||
);
|
||||
} else {
|
||||
await expect(store.materializeSessionArtifacts(upper, "metadata")).resolves.toMatchObject({
|
||||
metadataPath: path.join(store.sessionDir(upper), "metadata.json"),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("uses remaining manifest artifacts when aliased export metadata is absent", async () => {
|
||||
const { store } = createStore();
|
||||
const upper = session("Capital", "2026-07-01T10:00:00.000Z");
|
||||
const lower = session("capital", "2026-07-01T11:00:00.000Z");
|
||||
await store.writeSession(upper);
|
||||
await store.appendUtteranceForSession(upper, { text: "owned transcript" });
|
||||
const artifacts = await store.materializeSessionArtifacts(upper, "transcript");
|
||||
fs.rmSync(artifacts.metadataPath);
|
||||
|
||||
await expect(store.writeSession(lower)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not let a case-distinct SQLite owner mask a legacy directory", async () => {
|
||||
const { stateDir, store } = createStore();
|
||||
const upper = session("Capital", "2026-07-01T10:00:00.000Z");
|
||||
const lower = session("capital", "2026-07-01T11:00:00.000Z");
|
||||
await store.writeSession(upper);
|
||||
openOpenClawStateDatabase({ env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } })
|
||||
.db.prepare(
|
||||
"UPDATE meeting_transcript_sessions SET export_pending_json = ? WHERE session_id = ?",
|
||||
)
|
||||
.run('["metadata.json","transcript.jsonl"]', upper.sessionId);
|
||||
fs.mkdirSync(store.sessionDir(lower), { recursive: true });
|
||||
fs.writeFileSync(path.join(store.sessionDir(lower), "transcript.jsonl"), "legacy\n");
|
||||
|
||||
await expect(store.writeSession(lower)).rejects.toThrow("run openclaw doctor --fix");
|
||||
await expect(store.readSession(lower.sessionId)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("recognizes case-variant artifact names only when the filesystem aliases them", async () => {
|
||||
const { store } = createStore();
|
||||
const upper = session("Capital", "2026-07-01T10:00:00.000Z");
|
||||
const lower = session("capital", "2026-07-01T11:00:00.000Z");
|
||||
await store.writeSession(upper);
|
||||
expect(fs.existsSync(store.sessionDir(upper))).toBe(false);
|
||||
fs.mkdirSync(store.sessionDir(lower), { recursive: true });
|
||||
fs.rmSync(path.join(store.sessionDir(lower), "transcript.jsonl"), { force: true });
|
||||
fs.writeFileSync(path.join(store.sessionDir(lower), "TRANSCRIPT.JSONL"), "legacy\n");
|
||||
expect(fs.readdirSync(store.sessionDir(lower))).toContain("TRANSCRIPT.JSONL");
|
||||
|
||||
if (fs.existsSync(path.join(store.sessionDir(lower), "transcript.jsonl"))) {
|
||||
await expect(store.writeSession(lower)).rejects.toThrow("run openclaw doctor --fix");
|
||||
} else {
|
||||
await expect(store.writeSession(lower)).resolves.toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("refuses to overwrite an unclaimed legacy export directory", async () => {
|
||||
const { store } = createStore();
|
||||
const target = session("legacy-collision");
|
||||
const sessionDir = store.sessionDir(target);
|
||||
fs.mkdirSync(sessionDir, { recursive: true });
|
||||
const transcriptPath = path.join(sessionDir, "transcript.jsonl");
|
||||
fs.writeFileSync(transcriptPath, '{"text":"legacy line"}\n');
|
||||
|
||||
await expect(store.writeSession(target)).rejects.toThrow("run openclaw doctor --fix");
|
||||
expect(fs.readFileSync(transcriptPath, "utf8")).toContain("legacy line");
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"checks the shipped slug path before inserting a portable encoded session",
|
||||
async () => {
|
||||
const { stateDir, store } = createStore();
|
||||
const target = session("trailing-dot.");
|
||||
const legacyDir = path.join(stateDir, "transcripts", "2026-07-01", "trailing-dot.");
|
||||
fs.mkdirSync(legacyDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(legacyDir, "transcript.jsonl"), "legacy\n");
|
||||
|
||||
await expect(store.writeSession(target)).rejects.toThrow("run openclaw doctor --fix");
|
||||
await expect(store.readSession(target.sessionId)).resolves.toBeUndefined();
|
||||
},
|
||||
);
|
||||
|
||||
it("does not let a dot session mask the shipped dot-dot root layout", async () => {
|
||||
const { stateDir, store } = createStore();
|
||||
await store.writeSession(session("."));
|
||||
const transcriptRoot = path.join(stateDir, "transcripts");
|
||||
fs.mkdirSync(transcriptRoot, { recursive: true });
|
||||
fs.writeFileSync(path.join(transcriptRoot, "transcript.jsonl"), "legacy root transcript\n");
|
||||
|
||||
await expect(store.writeSession(session(".."))).rejects.toThrow("run openclaw doctor --fix");
|
||||
await expect(store.readSession("..")).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not let a modified export block canonical session updates", async () => {
|
||||
const { store } = createStore();
|
||||
const target = session("mutable-export");
|
||||
await store.writeSession(target);
|
||||
await store.appendUtteranceForSession(target, { text: "canonical" });
|
||||
const artifacts = await store.materializeSessionArtifacts(target, "transcript");
|
||||
fs.appendFileSync(artifacts.transcriptPath, '{"text":"external edit"}\n');
|
||||
|
||||
await expect(store.updateStopped(target.sessionId, "2026-07-01T11:00:00.000Z")).resolves.toBe(
|
||||
undefined,
|
||||
);
|
||||
await expect(store.readSession(target.sessionId)).resolves.toMatchObject({
|
||||
stoppedAt: "2026-07-01T11:00:00.000Z",
|
||||
});
|
||||
const summary = summarizeTranscripts({ session: target, utterances: [{ text: "canonical" }] });
|
||||
await expect(store.writeSummary(summary, target)).resolves.toBe(
|
||||
path.join(store.sessionDir(target), "summary.md"),
|
||||
);
|
||||
await expect(store.readSummary(target)).resolves.toMatchObject({
|
||||
summary: { sessionId: target.sessionId },
|
||||
});
|
||||
await expect(store.materializeSessionArtifacts(target, "transcript")).rejects.toThrow(
|
||||
"run openclaw doctor --fix",
|
||||
);
|
||||
});
|
||||
|
||||
const markdownPath = await store.writeSummary(summary);
|
||||
it("resolves descriptor exports through canonical SQLite identity", async () => {
|
||||
const { store } = createStore();
|
||||
const target = { ...session("canonical-export"), title: "Canonical title" };
|
||||
await store.writeSession(target);
|
||||
|
||||
const sessionDir = store.sessionDir(session);
|
||||
expect(markdownPath).toBe(path.join(sessionDir, "summary.md"));
|
||||
const stored = JSON.parse(fs.readFileSync(path.join(sessionDir, "summary.json"), "utf8")) as {
|
||||
sessionId: string;
|
||||
const artifacts = await store.materializeSessionArtifacts(
|
||||
{ ...target, title: "Stale title" },
|
||||
"metadata",
|
||||
);
|
||||
|
||||
expect(fs.readFileSync(artifacts.metadataPath, "utf8")).toContain("Canonical title");
|
||||
await expect(
|
||||
store.materializeSessionArtifacts(session("phantom-export"), "metadata"),
|
||||
).rejects.toThrow("transcripts session not found");
|
||||
expect(fs.existsSync(store.sessionDir(session("phantom-export")))).toBe(false);
|
||||
});
|
||||
|
||||
it("stores summaries in SQLite and materializes explicit artifacts", async () => {
|
||||
const { stateDir, store } = createStore();
|
||||
const target = {
|
||||
...session("ansi-\u001b[31mprovider\u001b[0m", "2026-05-22T10:00:00.000Z"),
|
||||
title: "ANSI import",
|
||||
};
|
||||
expect(stored.sessionId).toBe(session.sessionId);
|
||||
await store.writeSession(target);
|
||||
const utterance = {
|
||||
text: "We decided to ship the CLI.",
|
||||
speaker: { label: "Sam" },
|
||||
};
|
||||
const utterances = [utterance];
|
||||
await store.appendUtteranceForSession(target, utterance);
|
||||
const summary = summarizeTranscripts({ session: target, utterances });
|
||||
|
||||
const markdownPath = await store.writeSummary(summary, target);
|
||||
const artifacts = await store.materializeSessionArtifacts(target, "all");
|
||||
|
||||
expect(markdownPath).toBe(path.join(store.sessionDir(target), "summary.md"));
|
||||
expect(JSON.parse(fs.readFileSync(artifacts.summaryJsonPath, "utf8"))).toMatchObject({
|
||||
sessionId: target.sessionId,
|
||||
});
|
||||
expect(fs.readFileSync(artifacts.transcriptPath, "utf8")).toContain(
|
||||
'"text":"We decided to ship the CLI."',
|
||||
);
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
const reopened = new TranscriptsStore(path.join(stateDir, "transcripts"), {
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
|
||||
});
|
||||
await expect(reopened.readSummary(target)).resolves.toMatchObject({
|
||||
summary: { sessionId: target.sessionId },
|
||||
});
|
||||
});
|
||||
|
||||
it("removes stale summary exports when canonical state has no summary", async () => {
|
||||
const { stateDir, store } = createStore();
|
||||
const target = session("no-summary");
|
||||
await store.writeSession(target);
|
||||
await store.writeSummary(
|
||||
summarizeTranscripts({ session: target, utterances: [{ text: "stale" }] }),
|
||||
target,
|
||||
);
|
||||
openOpenClawStateDatabase({ env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } })
|
||||
.db.prepare("DELETE FROM meeting_transcript_summaries WHERE session_id = ?")
|
||||
.run(target.sessionId);
|
||||
|
||||
const artifacts = await store.materializeSessionArtifacts(target, "summary");
|
||||
|
||||
expect(artifacts.hasSummary).toBe(false);
|
||||
expect(fs.existsSync(artifacts.summaryJsonPath)).toBe(false);
|
||||
expect(fs.existsSync(artifacts.summaryPath)).toBe(false);
|
||||
});
|
||||
|
||||
it("repairs an interrupted manifest update and serializes concurrent exports", async () => {
|
||||
const { stateDir, store } = createStore();
|
||||
const target = session("recover-export");
|
||||
await store.writeSession(target);
|
||||
await store.appendUtteranceForSession(target, { text: "recover me" });
|
||||
await store.writeSummary(
|
||||
summarizeTranscripts({ session: target, utterances: [{ text: "recover me" }] }),
|
||||
target,
|
||||
);
|
||||
openOpenClawStateDatabase({ env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } })
|
||||
.db.prepare(
|
||||
"UPDATE meeting_transcript_sessions SET export_manifest_json = '{}' WHERE session_id = ?",
|
||||
)
|
||||
.run(target.sessionId);
|
||||
|
||||
await expect(
|
||||
Promise.all([
|
||||
store.materializeSessionArtifacts(target, "summary"),
|
||||
store.materializeSessionArtifacts(target, "transcript"),
|
||||
]),
|
||||
).resolves.toHaveLength(2);
|
||||
|
||||
const manifest = openOpenClawStateDatabase({
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
|
||||
})
|
||||
.db.prepare(
|
||||
"SELECT export_manifest_json FROM meeting_transcript_sessions WHERE session_id = ?",
|
||||
)
|
||||
.get(target.sessionId) as { export_manifest_json: string };
|
||||
expect(JSON.parse(manifest.export_manifest_json)).toMatchObject({
|
||||
"metadata.json": expect.any(String),
|
||||
"summary.md": expect.any(String),
|
||||
"transcript.jsonl": expect.any(String),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
+655
-249
@@ -1,313 +1,719 @@
|
||||
// Stores and streams transcript files for later summary and replay.
|
||||
import { createReadStream } from "node:fs";
|
||||
import type { Dirent } from "node:fs";
|
||||
// Stores meeting-capture transcripts in the shared SQLite state database.
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { resolveOptionalIntegerOption } from "@openclaw/normalization-core/number-coercion";
|
||||
import { sha256File, sha256Hex } from "../infra/crypto-digest.js";
|
||||
import { ensureAbsoluteDirectory } from "../infra/fs-safe.js";
|
||||
import {
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
iterateSqliteQuerySync,
|
||||
} from "../infra/kysely-sync.js";
|
||||
import {
|
||||
openOpenClawStateDatabase,
|
||||
runOpenClawStateWriteTransaction,
|
||||
type OpenClawStateDatabase,
|
||||
type OpenClawStateDatabaseOptions,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import { withOpenClawStateLease } from "../state/openclaw-state-lease.js";
|
||||
import type { TranscriptSessionDescriptor, TranscriptUtterance } from "./provider-types.js";
|
||||
import { ensureMeetingTranscriptsSchema } from "./sqlite-schema.js";
|
||||
import {
|
||||
isCaseSensitiveDirectory,
|
||||
legacyTranscriptSessionSelector,
|
||||
normalizeExportText,
|
||||
removeTranscriptArtifact,
|
||||
safeTranscriptPathSegment,
|
||||
TRANSCRIPT_EXPORT_FILE_NAMES,
|
||||
transcriptSessionExportKey,
|
||||
transcriptSessionSelector,
|
||||
writeTranscriptArtifact,
|
||||
} from "./store-artifacts.js";
|
||||
import { writeTranscriptJsonlArtifact } from "./store-export-jsonl.js";
|
||||
import {
|
||||
assertTranscriptExportPathAvailable,
|
||||
hasAliasedCanonicalTranscriptExportPathOwner,
|
||||
} from "./store-export-ownership.js";
|
||||
import {
|
||||
meetingTranscriptDb,
|
||||
type MeetingTranscriptSessionRow,
|
||||
sessionFromRow,
|
||||
summaryFromRow,
|
||||
utteranceFromRow,
|
||||
} from "./store-sqlite.js";
|
||||
import type * as StoreTypes from "./store-types.js";
|
||||
import type { TranscriptsSummary } from "./summary.js";
|
||||
import { renderTranscriptsMarkdown } from "./summary.js";
|
||||
|
||||
/**
|
||||
* File-backed transcript session store.
|
||||
*
|
||||
* Sessions are stored by date/session id with metadata JSON, append-only
|
||||
* utterance JSONL, and rendered summary artifacts.
|
||||
*/
|
||||
/** Stored session metadata plus the resolved session directory. */
|
||||
export type TranscriptsSessionEntry = {
|
||||
session: TranscriptSessionDescriptor;
|
||||
sessionDir: string;
|
||||
};
|
||||
export type * from "./store-types.js";
|
||||
export { safeTranscriptPathSegment, transcriptSessionExportKey, transcriptSessionSelector };
|
||||
|
||||
function safeSegment(value: string): string {
|
||||
// Session ids can come from external providers; path segments stay conservative.
|
||||
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "session";
|
||||
}
|
||||
|
||||
function dateSegment(value: string | undefined): string {
|
||||
const isoDate = value?.match(/^(\d{4}-\d{2}-\d{2})T/)?.[1];
|
||||
return isoDate ?? new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
async function readJsonFile<T>(filePath: string): Promise<T | undefined> {
|
||||
try {
|
||||
return JSON.parse(await fs.readFile(filePath, "utf8")) as T;
|
||||
} catch (err) {
|
||||
if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") {
|
||||
return undefined;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function sameSessionIdentity(
|
||||
left: TranscriptSessionDescriptor,
|
||||
right: TranscriptSessionDescriptor,
|
||||
): boolean {
|
||||
return left.sessionId === right.sessionId && left.startedAt === right.startedAt;
|
||||
}
|
||||
|
||||
/** Durable transcript store rooted at a caller-provided directory. */
|
||||
/** Canonical meeting-capture transcript store. Files are explicit exports only. */
|
||||
export class TranscriptsStore {
|
||||
constructor(private readonly rootDir: string) {}
|
||||
constructor(
|
||||
private readonly exportRootDir: string,
|
||||
private readonly databaseOptions: OpenClawStateDatabaseOptions = {},
|
||||
) {}
|
||||
|
||||
private database() {
|
||||
ensureMeetingTranscriptsSchema(this.databaseOptions);
|
||||
return openOpenClawStateDatabase(this.databaseOptions);
|
||||
}
|
||||
|
||||
/** Resolve the dated directory for a transcript session. */
|
||||
sessionDir(session: TranscriptSessionDescriptor): string {
|
||||
return path.join(this.rootDir, dateSegment(session.startedAt), safeSegment(session.sessionId));
|
||||
return path.join(this.exportRootDir, transcriptSessionSelector(session));
|
||||
}
|
||||
|
||||
private async hasSessionMetadata(dir: string): Promise<boolean> {
|
||||
return (await readJsonFile<unknown>(path.join(dir, "metadata.json"))) !== undefined;
|
||||
private entryFromRow(
|
||||
row: MeetingTranscriptSessionRow,
|
||||
summaryKeys: ReadonlySet<string>,
|
||||
): StoreTypes.TranscriptsSessionEntry {
|
||||
const session = sessionFromRow(row);
|
||||
const sessionDir = this.sessionDir(session);
|
||||
const key = `${session.sessionId}\0${session.startedAt}`;
|
||||
return {
|
||||
session,
|
||||
sessionDir,
|
||||
selector: row.selector,
|
||||
summaryPath: path.join(sessionDir, "summary.md"),
|
||||
hasSummary: summaryKeys.has(key),
|
||||
};
|
||||
}
|
||||
|
||||
private async findSessionDirForSession(session: TranscriptSessionDescriptor): Promise<string> {
|
||||
const datedDir = this.sessionDir(session);
|
||||
const datedSession = await readJsonFile<TranscriptSessionDescriptor>(
|
||||
path.join(datedDir, "metadata.json"),
|
||||
private readSummaryKeys(database: OpenClawStateDatabase): Set<string> {
|
||||
const rows = executeSqliteQuerySync(
|
||||
database.db,
|
||||
meetingTranscriptDb(database.db)
|
||||
.selectFrom("meeting_transcript_summaries")
|
||||
.select(["session_id", "session_started_at"]),
|
||||
).rows;
|
||||
return new Set(rows.map((row) => `${row.session_id}\0${row.session_started_at}`));
|
||||
}
|
||||
|
||||
private hasSummary(database: OpenClawStateDatabase, row: MeetingTranscriptSessionRow): boolean {
|
||||
return Boolean(
|
||||
executeSqliteQueryTakeFirstSync(
|
||||
database.db,
|
||||
meetingTranscriptDb(database.db)
|
||||
.selectFrom("meeting_transcript_summaries")
|
||||
.select("session_id")
|
||||
.where("session_id", "=", row.session_id)
|
||||
.where("session_started_at", "=", row.started_at)
|
||||
.limit(1),
|
||||
),
|
||||
);
|
||||
if (datedSession && sameSessionIdentity(datedSession, session)) {
|
||||
return datedDir;
|
||||
}
|
||||
return datedDir;
|
||||
}
|
||||
|
||||
private async findSessionDir(selector: string): Promise<string | undefined> {
|
||||
const qualified = selector.match(/^(\d{4}-\d{2}-\d{2})\/(.+)$/);
|
||||
if (qualified?.[1] && qualified[2]) {
|
||||
const directDir = path.join(this.rootDir, qualified[1], safeSegment(qualified[2]));
|
||||
return (await this.hasSessionMetadata(directDir)) ? directDir : undefined;
|
||||
}
|
||||
private readExportOwnership(session: TranscriptSessionDescriptor): {
|
||||
manifest: Record<string, string>;
|
||||
pending: Set<string>;
|
||||
} {
|
||||
const database = this.database();
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
database.db,
|
||||
meetingTranscriptDb(database.db)
|
||||
.selectFrom("meeting_transcript_sessions")
|
||||
.select(["export_manifest_json", "export_pending_json"])
|
||||
.where("session_id", "=", session.sessionId)
|
||||
.where("started_at", "=", session.startedAt),
|
||||
);
|
||||
return row
|
||||
? {
|
||||
manifest: JSON.parse(row.export_manifest_json) as Record<string, string>,
|
||||
pending: new Set(JSON.parse(row.export_pending_json) as string[]),
|
||||
}
|
||||
: { manifest: {}, pending: new Set() };
|
||||
}
|
||||
|
||||
const safeSessionId = safeSegment(selector);
|
||||
const idDate = selector
|
||||
.match(/^meeting-(\d{4})-(\d{2})-(\d{2})T/)
|
||||
?.slice(1, 4)
|
||||
.join("-");
|
||||
if (idDate) {
|
||||
const directDir = path.join(this.rootDir, idDate, safeSessionId);
|
||||
return (await this.hasSessionMetadata(directDir)) ? directDir : undefined;
|
||||
private readSessionByIdentity(
|
||||
session: TranscriptSessionDescriptor,
|
||||
): TranscriptSessionDescriptor | undefined {
|
||||
const database = this.database();
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
database.db,
|
||||
meetingTranscriptDb(database.db)
|
||||
.selectFrom("meeting_transcript_sessions")
|
||||
.selectAll()
|
||||
.where("session_id", "=", session.sessionId)
|
||||
.where("started_at", "=", session.startedAt),
|
||||
);
|
||||
return row ? sessionFromRow(row) : undefined;
|
||||
}
|
||||
|
||||
private transcriptRows(session: TranscriptSessionDescriptor) {
|
||||
const database = this.database();
|
||||
return {
|
||||
database,
|
||||
query: meetingTranscriptDb(database.db)
|
||||
.selectFrom("meeting_transcript_utterances")
|
||||
.selectAll()
|
||||
.where("session_id", "=", session.sessionId)
|
||||
.where("session_started_at", "=", session.startedAt)
|
||||
.orderBy("sequence", "asc"),
|
||||
};
|
||||
}
|
||||
|
||||
private transcriptJsonlDigest(session: TranscriptSessionDescriptor): string {
|
||||
const { database, query } = this.transcriptRows(session);
|
||||
const digest = createHash("sha256");
|
||||
for (const row of iterateSqliteQuerySync(database.db, query)) {
|
||||
digest.update(`${JSON.stringify(utteranceFromRow(row))}\n`);
|
||||
}
|
||||
let entries: Dirent[];
|
||||
return digest.digest("hex");
|
||||
}
|
||||
|
||||
private async expectedExportHashes(
|
||||
session: TranscriptSessionDescriptor,
|
||||
): Promise<Record<string, string>> {
|
||||
const storedSession = this.readSessionByIdentity(session);
|
||||
if (!storedSession) {
|
||||
return {};
|
||||
}
|
||||
const hashes: Record<string, string> = {
|
||||
"metadata.json": sha256Hex(`${JSON.stringify(storedSession, null, 2)}\n`),
|
||||
};
|
||||
hashes["transcript.jsonl"] = this.transcriptJsonlDigest(storedSession);
|
||||
const summary = await this.readSummary(storedSession);
|
||||
if (summary.summary) {
|
||||
hashes["summary.json"] = sha256Hex(`${JSON.stringify(summary.summary, null, 2)}\n`);
|
||||
}
|
||||
if (summary.markdown !== undefined) {
|
||||
hashes["summary.md"] = sha256Hex(normalizeExportText(summary.markdown));
|
||||
}
|
||||
return hashes;
|
||||
}
|
||||
|
||||
private updateExportManifest(
|
||||
session: TranscriptSessionDescriptor,
|
||||
exportedHashes: Readonly<Record<string, string>>,
|
||||
removedExports: ReadonlySet<string> = new Set(),
|
||||
): void {
|
||||
runOpenClawStateWriteTransaction(
|
||||
({ db: database }) => {
|
||||
const db = meetingTranscriptDb(database);
|
||||
const stored = executeSqliteQueryTakeFirstSync(
|
||||
database,
|
||||
db
|
||||
.selectFrom("meeting_transcript_sessions")
|
||||
.select(["export_manifest_json", "export_pending_json"])
|
||||
.where("session_id", "=", session.sessionId)
|
||||
.where("started_at", "=", session.startedAt),
|
||||
);
|
||||
const manifest = stored
|
||||
? (JSON.parse(stored.export_manifest_json) as Record<string, string>)
|
||||
: {};
|
||||
const pending = new Set(stored ? (JSON.parse(stored.export_pending_json) as string[]) : []);
|
||||
for (const fileName of removedExports) {
|
||||
delete manifest[fileName];
|
||||
}
|
||||
for (const fileName of [...Object.keys(exportedHashes), ...removedExports]) {
|
||||
pending.delete(fileName);
|
||||
}
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db
|
||||
.updateTable("meeting_transcript_sessions")
|
||||
.set({
|
||||
export_manifest_json: JSON.stringify({ ...manifest, ...exportedHashes }),
|
||||
export_pending_json: JSON.stringify([...pending].toSorted()),
|
||||
})
|
||||
.where("session_id", "=", session.sessionId)
|
||||
.where("started_at", "=", session.startedAt),
|
||||
);
|
||||
},
|
||||
this.databaseOptions,
|
||||
{ operationLabel: "meeting-transcripts.export.record" },
|
||||
);
|
||||
}
|
||||
|
||||
private markPendingExports(session: TranscriptSessionDescriptor, fileNames: string[]): void {
|
||||
runOpenClawStateWriteTransaction(
|
||||
({ db: database }) => {
|
||||
const db = meetingTranscriptDb(database);
|
||||
const stored = executeSqliteQueryTakeFirstSync(
|
||||
database,
|
||||
db
|
||||
.selectFrom("meeting_transcript_sessions")
|
||||
.select("export_pending_json")
|
||||
.where("session_id", "=", session.sessionId)
|
||||
.where("started_at", "=", session.startedAt),
|
||||
);
|
||||
if (!stored) {
|
||||
throw new Error(`transcripts session not found: ${session.sessionId}`);
|
||||
}
|
||||
const pending = new Set(JSON.parse(stored.export_pending_json) as string[]);
|
||||
for (const fileName of fileNames) {
|
||||
pending.add(fileName);
|
||||
}
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db
|
||||
.updateTable("meeting_transcript_sessions")
|
||||
.set({ export_pending_json: JSON.stringify([...pending].toSorted()) })
|
||||
.where("session_id", "=", session.sessionId)
|
||||
.where("started_at", "=", session.startedAt),
|
||||
);
|
||||
},
|
||||
this.databaseOptions,
|
||||
{ operationLabel: "meeting-transcripts.export.pending" },
|
||||
);
|
||||
}
|
||||
|
||||
private async assertExportDestinationOwned(
|
||||
session: TranscriptSessionDescriptor,
|
||||
sessionDir = this.sessionDir(session),
|
||||
): Promise<void> {
|
||||
let entries;
|
||||
try {
|
||||
entries = await fs.readdir(this.rootDir, { withFileTypes: true });
|
||||
} catch (err) {
|
||||
if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") {
|
||||
return undefined;
|
||||
entries = await fs.readdir(sessionDir, { withFileTypes: true });
|
||||
} catch (error) {
|
||||
if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") {
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
throw error;
|
||||
}
|
||||
const datedEntries = entries
|
||||
.filter((entry) => entry.isDirectory() && /^\d{4}-\d{2}-\d{2}$/.test(entry.name))
|
||||
.toSorted((left, right) => right.name.localeCompare(left.name));
|
||||
const matches: string[] = [];
|
||||
for (const entry of datedEntries) {
|
||||
const candidate = path.join(this.rootDir, entry.name, safeSessionId);
|
||||
const session = await readJsonFile<TranscriptSessionDescriptor>(
|
||||
path.join(candidate, "metadata.json"),
|
||||
);
|
||||
if (session?.sessionId === selector) {
|
||||
matches.push(candidate);
|
||||
const ownership = this.readExportOwnership(session);
|
||||
const caseSensitive = await isCaseSensitiveDirectory(sessionDir);
|
||||
let expectedHashes: Record<string, string> | undefined;
|
||||
const repairedHashes: Record<string, string> = {};
|
||||
for (const entry of entries) {
|
||||
const canonicalName = caseSensitive ? entry.name : entry.name.toLowerCase();
|
||||
if (!TRANSCRIPT_EXPORT_FILE_NAMES.has(canonicalName)) {
|
||||
continue;
|
||||
}
|
||||
const filePath = path.join(sessionDir, entry.name);
|
||||
const stat = await fs.lstat(filePath);
|
||||
if (stat.isSymbolicLink() || !stat.isFile()) {
|
||||
throw new Error(
|
||||
`legacy transcript artifacts require migration before writing ${sessionDir}; run openclaw doctor --fix`,
|
||||
);
|
||||
}
|
||||
const actualHash = await sha256File(filePath);
|
||||
if (
|
||||
ownership.manifest[canonicalName] === actualHash ||
|
||||
ownership.pending.has(canonicalName)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
expectedHashes ??= await this.expectedExportHashes(session);
|
||||
if (expectedHashes[canonicalName] !== actualHash) {
|
||||
throw new Error(
|
||||
`legacy transcript artifacts require migration before writing ${sessionDir}; run openclaw doctor --fix`,
|
||||
);
|
||||
}
|
||||
repairedHashes[canonicalName] = actualHash;
|
||||
}
|
||||
if (matches.length > 1) {
|
||||
// Ambiguous bare ids require an explicit date prefix to avoid reading the wrong session.
|
||||
throw new Error(
|
||||
`multiple transcripts sessions match ${selector}; use a YYYY-MM-DD/${selector} selector`,
|
||||
);
|
||||
if (Object.keys(repairedHashes).length > 0) {
|
||||
this.updateExportManifest(session, repairedHashes);
|
||||
}
|
||||
return matches[0];
|
||||
}
|
||||
|
||||
/** Persist transcript session metadata. */
|
||||
async listSessionEntries(): Promise<StoreTypes.TranscriptsSessionEntry[]> {
|
||||
const database = this.database();
|
||||
const rows = executeSqliteQuerySync(
|
||||
database.db,
|
||||
meetingTranscriptDb(database.db)
|
||||
.selectFrom("meeting_transcript_sessions")
|
||||
.selectAll()
|
||||
.orderBy("started_at", "desc")
|
||||
.orderBy("session_id", "asc"),
|
||||
).rows;
|
||||
const summaryKeys = this.readSummaryKeys(database);
|
||||
return rows.map((row) => this.entryFromRow(row, summaryKeys));
|
||||
}
|
||||
|
||||
async writeSession(session: TranscriptSessionDescriptor): Promise<void> {
|
||||
const dir = this.sessionDir(session);
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
await fs.writeFile(path.join(dir, "metadata.json"), `${JSON.stringify(session, null, 2)}\n`);
|
||||
ensureMeetingTranscriptsSchema(this.databaseOptions);
|
||||
if (
|
||||
!this.readSessionByIdentity(session) &&
|
||||
!(await hasAliasedCanonicalTranscriptExportPathOwner({
|
||||
session,
|
||||
exportRootDir: this.exportRootDir,
|
||||
databaseOptions: this.databaseOptions,
|
||||
}))
|
||||
) {
|
||||
await this.assertExportDestinationOwned(session);
|
||||
const legacySessionDir = path.join(
|
||||
this.exportRootDir,
|
||||
legacyTranscriptSessionSelector(session),
|
||||
);
|
||||
const legacyOwner = await this.readSession(legacyTranscriptSessionSelector(session));
|
||||
const legacyPathIsCanonical =
|
||||
legacyOwner !== undefined &&
|
||||
path.resolve(this.sessionDir(legacyOwner)) === path.resolve(legacySessionDir);
|
||||
if (
|
||||
path.resolve(legacySessionDir) !== path.resolve(this.sessionDir(session)) &&
|
||||
!legacyPathIsCanonical
|
||||
) {
|
||||
await this.assertExportDestinationOwned(session, legacySessionDir);
|
||||
}
|
||||
}
|
||||
const selector = transcriptSessionSelector(session);
|
||||
const sourceJson = JSON.stringify(session.source);
|
||||
const metadataJson = session.metadata ? JSON.stringify(session.metadata) : null;
|
||||
const now = Date.now();
|
||||
runOpenClawStateWriteTransaction(
|
||||
({ db: database }) => {
|
||||
const db = meetingTranscriptDb(database);
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db
|
||||
.insertInto("meeting_transcript_sessions")
|
||||
.values({
|
||||
session_id: session.sessionId,
|
||||
started_at: session.startedAt,
|
||||
selector,
|
||||
export_key: transcriptSessionExportKey(session),
|
||||
session_slug: safeTranscriptPathSegment(session.sessionId),
|
||||
provider_id: session.source.providerId,
|
||||
title: session.title ?? null,
|
||||
source_json: sourceJson,
|
||||
stopped_at: session.stoppedAt ?? null,
|
||||
metadata_json: metadataJson,
|
||||
export_manifest_json: "{}",
|
||||
export_pending_json: "[]",
|
||||
next_utterance_seq: 0,
|
||||
created_at_ms: now,
|
||||
updated_at_ms: now,
|
||||
})
|
||||
.onConflict((conflict) =>
|
||||
conflict.columns(["session_id", "started_at"]).doUpdateSet({
|
||||
selector,
|
||||
export_key: transcriptSessionExportKey(session),
|
||||
session_slug: safeTranscriptPathSegment(session.sessionId),
|
||||
provider_id: session.source.providerId,
|
||||
title: session.title ?? null,
|
||||
source_json: sourceJson,
|
||||
stopped_at: session.stoppedAt ?? null,
|
||||
metadata_json: metadataJson,
|
||||
updated_at_ms: now,
|
||||
}),
|
||||
),
|
||||
);
|
||||
},
|
||||
this.databaseOptions,
|
||||
{ operationLabel: "meeting-transcripts.session.write" },
|
||||
);
|
||||
}
|
||||
|
||||
/** Read one session descriptor by session id or qualified date/id selector. */
|
||||
async readSession(sessionId: string): Promise<TranscriptSessionDescriptor | undefined> {
|
||||
return (await this.readSessionEntry(sessionId))?.session;
|
||||
async readSession(sessionSelector: string): Promise<TranscriptSessionDescriptor | undefined> {
|
||||
return (await this.readSessionEntry(sessionSelector))?.session;
|
||||
}
|
||||
|
||||
/** Read one session descriptor plus its directory. */
|
||||
async readSessionEntry(sessionId: string): Promise<TranscriptsSessionEntry | undefined> {
|
||||
const dir = await this.findSessionDir(sessionId);
|
||||
if (!dir) {
|
||||
async readSessionEntry(
|
||||
sessionSelector: string,
|
||||
): Promise<StoreTypes.TranscriptsSessionEntry | undefined> {
|
||||
const database = this.database();
|
||||
const db = meetingTranscriptDb(database.db);
|
||||
const qualified = /^\d{4}-\d{2}-\d{2}\//u.test(sessionSelector);
|
||||
const exactRows = qualified
|
||||
? executeSqliteQuerySync(
|
||||
database.db,
|
||||
db
|
||||
.selectFrom("meeting_transcript_sessions")
|
||||
.selectAll()
|
||||
.where("selector", "=", sessionSelector),
|
||||
).rows
|
||||
: executeSqliteQuerySync(
|
||||
database.db,
|
||||
db
|
||||
.selectFrom("meeting_transcript_sessions")
|
||||
.selectAll()
|
||||
.where("session_id", "=", sessionSelector)
|
||||
.orderBy("started_at", "desc")
|
||||
.limit(2),
|
||||
).rows;
|
||||
const slugRows = qualified
|
||||
? []
|
||||
: executeSqliteQuerySync(
|
||||
database.db,
|
||||
db
|
||||
.selectFrom("meeting_transcript_sessions")
|
||||
.selectAll()
|
||||
.where("session_slug", "=", sessionSelector)
|
||||
.orderBy("started_at", "desc")
|
||||
.limit(2),
|
||||
).rows;
|
||||
const rows = [
|
||||
...new Map(
|
||||
[...exactRows, ...slugRows].map((row) => [`${row.session_id}\0${row.started_at}`, row]),
|
||||
).values(),
|
||||
];
|
||||
if (rows.length > 1) {
|
||||
throw new Error(
|
||||
`multiple transcripts sessions match ${sessionSelector}; use one of: ${rows
|
||||
.map((row) => row.selector)
|
||||
.join(", ")}`,
|
||||
);
|
||||
}
|
||||
const row = rows[0];
|
||||
if (!row) {
|
||||
return undefined;
|
||||
}
|
||||
const session = await readJsonFile<TranscriptSessionDescriptor>(
|
||||
path.join(dir, "metadata.json"),
|
||||
);
|
||||
return session ? { session, sessionDir: dir } : undefined;
|
||||
const summaryKeys = this.hasSummary(database, row)
|
||||
? new Set([`${row.session_id}\0${row.started_at}`])
|
||||
: new Set<string>();
|
||||
return this.entryFromRow(row, summaryKeys);
|
||||
}
|
||||
|
||||
/** Append an utterance for an exact session descriptor. */
|
||||
async appendUtteranceForSession(
|
||||
session: TranscriptSessionDescriptor,
|
||||
utterance: TranscriptUtterance,
|
||||
): Promise<void> {
|
||||
const dir = await this.findSessionDirForSession(session);
|
||||
await this.appendUtteranceToDir(dir, session.sessionId, utterance);
|
||||
}
|
||||
|
||||
private async appendUtteranceToDir(
|
||||
dir: string,
|
||||
sessionId: string,
|
||||
utterance: TranscriptUtterance,
|
||||
): Promise<void> {
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
await fs.appendFile(
|
||||
path.join(dir, "transcript.jsonl"),
|
||||
`${JSON.stringify({ ...utterance, sessionId })}\n`,
|
||||
const metadataJson = utterance.metadata ? JSON.stringify(utterance.metadata) : null;
|
||||
const now = Date.now();
|
||||
ensureMeetingTranscriptsSchema(this.databaseOptions);
|
||||
runOpenClawStateWriteTransaction(
|
||||
({ db: database }) => {
|
||||
const db = meetingTranscriptDb(database);
|
||||
const stored = executeSqliteQueryTakeFirstSync(
|
||||
database,
|
||||
db
|
||||
.selectFrom("meeting_transcript_sessions")
|
||||
.select("next_utterance_seq")
|
||||
.where("session_id", "=", session.sessionId)
|
||||
.where("started_at", "=", session.startedAt),
|
||||
);
|
||||
if (!stored) {
|
||||
throw new Error(`transcripts session not found: ${session.sessionId}`);
|
||||
}
|
||||
const sequence = stored.next_utterance_seq;
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db.insertInto("meeting_transcript_utterances").values({
|
||||
session_id: session.sessionId,
|
||||
session_started_at: session.startedAt,
|
||||
sequence,
|
||||
utterance_id: utterance.id ?? null,
|
||||
started_at: utterance.startedAt ?? null,
|
||||
ended_at: utterance.endedAt ?? null,
|
||||
speaker_id: utterance.speaker?.id ?? null,
|
||||
speaker_label: utterance.speaker?.label ?? null,
|
||||
text: utterance.text,
|
||||
final: utterance.final === undefined ? null : utterance.final ? 1 : 0,
|
||||
metadata_json: metadataJson,
|
||||
}),
|
||||
);
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db
|
||||
.updateTable("meeting_transcript_sessions")
|
||||
.set({ next_utterance_seq: sequence + 1, updated_at_ms: now })
|
||||
.where("session_id", "=", session.sessionId)
|
||||
.where("started_at", "=", session.startedAt),
|
||||
);
|
||||
},
|
||||
this.databaseOptions,
|
||||
{ operationLabel: "meeting-transcripts.utterance.append" },
|
||||
);
|
||||
}
|
||||
|
||||
/** Read utterances for an exact session descriptor. */
|
||||
async readUtterancesForSession(
|
||||
session: TranscriptSessionDescriptor,
|
||||
options: { maxUtterances?: number } = {},
|
||||
): Promise<TranscriptUtterance[]> {
|
||||
return await this.readUtterancesFromDir(await this.findSessionDirForSession(session), options);
|
||||
}
|
||||
|
||||
/** Read utterances directly from a known session directory. */
|
||||
async readUtterancesFromSessionDir(
|
||||
sessionDir: string,
|
||||
options: { maxUtterances?: number } = {},
|
||||
): Promise<TranscriptUtterance[]> {
|
||||
return await this.readUtterancesFromDir(sessionDir, options);
|
||||
}
|
||||
|
||||
private async readUtterancesFromDir(
|
||||
dir: string,
|
||||
options: { maxUtterances?: number } = {},
|
||||
): Promise<TranscriptUtterance[]> {
|
||||
const transcriptPath = path.join(dir, "transcript.jsonl");
|
||||
const database = this.database();
|
||||
const maxUtterances = resolveOptionalIntegerOption(options.maxUtterances, { min: 1 });
|
||||
if (maxUtterances !== undefined) {
|
||||
return await new Promise<TranscriptUtterance[]>((resolve, reject) => {
|
||||
const utterances: TranscriptUtterance[] = [];
|
||||
const stream = createReadStream(transcriptPath, { encoding: "utf8" });
|
||||
const lines = createInterface({
|
||||
input: stream,
|
||||
crlfDelay: Infinity,
|
||||
});
|
||||
let settled = false;
|
||||
let emptyForENOENT = false;
|
||||
let pendingError: Error | undefined;
|
||||
|
||||
const settle = () => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
lines.close();
|
||||
stream.destroy();
|
||||
if (pendingError) {
|
||||
reject(pendingError);
|
||||
} else if (emptyForENOENT) {
|
||||
resolve([]);
|
||||
} else {
|
||||
resolve(utterances);
|
||||
}
|
||||
};
|
||||
const setError = (err: unknown) => {
|
||||
if (!pendingError) {
|
||||
pendingError = err instanceof Error ? err : new Error(String(err));
|
||||
}
|
||||
};
|
||||
|
||||
stream.on("close", settle);
|
||||
stream.on("error", (err) => {
|
||||
if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") {
|
||||
emptyForENOENT = true;
|
||||
return;
|
||||
}
|
||||
setError(err);
|
||||
stream.destroy();
|
||||
});
|
||||
lines.on("error", (err) => {
|
||||
if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") {
|
||||
emptyForENOENT = true;
|
||||
return;
|
||||
}
|
||||
setError(err);
|
||||
stream.destroy();
|
||||
});
|
||||
lines.on("line", (line) => {
|
||||
if (!line) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
utterances.push(JSON.parse(line) as TranscriptUtterance);
|
||||
} catch (err) {
|
||||
setError(err);
|
||||
stream.destroy();
|
||||
return;
|
||||
}
|
||||
if (utterances.length > maxUtterances) {
|
||||
// Stream and keep only the tail so large transcripts do not require full-file memory.
|
||||
utterances.shift();
|
||||
}
|
||||
});
|
||||
});
|
||||
const query = meetingTranscriptDb(database.db)
|
||||
.selectFrom("meeting_transcript_utterances")
|
||||
.selectAll()
|
||||
.where("session_id", "=", session.sessionId)
|
||||
.where("session_started_at", "=", session.startedAt);
|
||||
if (maxUtterances === undefined) {
|
||||
return executeSqliteQuerySync(database.db, query.orderBy("sequence", "asc")).rows.map(
|
||||
utteranceFromRow,
|
||||
);
|
||||
}
|
||||
let raw: string;
|
||||
try {
|
||||
raw = await fs.readFile(transcriptPath, "utf8");
|
||||
} catch (err) {
|
||||
if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") {
|
||||
return [];
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
return raw
|
||||
.split(/\r?\n/)
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line) as TranscriptUtterance);
|
||||
return executeSqliteQuerySync(
|
||||
database.db,
|
||||
query.orderBy("sequence", "desc").limit(maxUtterances),
|
||||
)
|
||||
.rows.toReversed()
|
||||
.map(utteranceFromRow);
|
||||
}
|
||||
|
||||
/** Mark a transcript session as stopped when metadata exists. */
|
||||
async updateStopped(sessionId: string, stoppedAt: string): Promise<void> {
|
||||
const dir = await this.findSessionDir(sessionId);
|
||||
if (!dir) {
|
||||
async updateStopped(sessionSelector: string, stoppedAt: string): Promise<void> {
|
||||
const entry = await this.readSessionEntry(sessionSelector);
|
||||
if (!entry) {
|
||||
return;
|
||||
}
|
||||
const session = await readJsonFile<TranscriptSessionDescriptor>(
|
||||
path.join(dir, "metadata.json"),
|
||||
);
|
||||
if (!session) {
|
||||
return;
|
||||
}
|
||||
await fs.writeFile(
|
||||
path.join(dir, "metadata.json"),
|
||||
`${JSON.stringify({ ...session, stoppedAt }, null, 2)}\n`,
|
||||
);
|
||||
await this.writeSession({ ...entry.session, stoppedAt });
|
||||
}
|
||||
|
||||
/** Write summary artifacts for a session and return the markdown path. */
|
||||
async writeSummary(
|
||||
summary: TranscriptsSummary,
|
||||
session?: TranscriptSessionDescriptor,
|
||||
): Promise<string> {
|
||||
const dir =
|
||||
session !== undefined
|
||||
? await this.findSessionDirForSession(session)
|
||||
: ((await this.findSessionDir(summary.sessionId)) ??
|
||||
path.join(this.rootDir, dateSegment(summary.sessionId), safeSegment(summary.sessionId)));
|
||||
return await this.writeSummaryToDir(summary, dir);
|
||||
const resolved = session ?? (await this.readSession(summary.sessionId));
|
||||
if (!resolved) {
|
||||
throw new Error(`transcripts session not found: ${summary.sessionId}`);
|
||||
}
|
||||
const summaryJson = JSON.stringify(summary);
|
||||
const markdown = renderTranscriptsMarkdown(summary);
|
||||
ensureMeetingTranscriptsSchema(this.databaseOptions);
|
||||
runOpenClawStateWriteTransaction(
|
||||
({ db: database }) => {
|
||||
const db = meetingTranscriptDb(database);
|
||||
executeSqliteQuerySync(
|
||||
database,
|
||||
db
|
||||
.insertInto("meeting_transcript_summaries")
|
||||
.values({
|
||||
session_id: resolved.sessionId,
|
||||
session_started_at: resolved.startedAt,
|
||||
generated_at: summary.generatedAt,
|
||||
summary_json: summaryJson,
|
||||
markdown,
|
||||
utterance_count: summary.utteranceCount,
|
||||
})
|
||||
.onConflict((conflict) =>
|
||||
conflict.columns(["session_id", "session_started_at"]).doUpdateSet({
|
||||
generated_at: summary.generatedAt,
|
||||
summary_json: summaryJson,
|
||||
markdown,
|
||||
utterance_count: summary.utteranceCount,
|
||||
}),
|
||||
),
|
||||
);
|
||||
},
|
||||
this.databaseOptions,
|
||||
{ operationLabel: "meeting-transcripts.summary.write" },
|
||||
);
|
||||
return path.join(this.sessionDir(resolved), "summary.md");
|
||||
}
|
||||
|
||||
/** Write summary JSON and markdown to a known directory. */
|
||||
async writeSummaryToDir(summary: TranscriptsSummary, dir: string): Promise<string> {
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
await fs.writeFile(path.join(dir, "summary.json"), `${JSON.stringify(summary, null, 2)}\n`);
|
||||
const markdown = renderTranscriptsMarkdown(summary);
|
||||
const markdownPath = path.join(dir, "summary.md");
|
||||
await fs.writeFile(markdownPath, `${markdown}\n`);
|
||||
return markdownPath;
|
||||
async readSummary(
|
||||
session: TranscriptSessionDescriptor,
|
||||
): Promise<{ summary?: TranscriptsSummary; markdown?: string }> {
|
||||
const database = this.database();
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
database.db,
|
||||
meetingTranscriptDb(database.db)
|
||||
.selectFrom("meeting_transcript_summaries")
|
||||
.selectAll()
|
||||
.where("session_id", "=", session.sessionId)
|
||||
.where("session_started_at", "=", session.startedAt),
|
||||
);
|
||||
if (!row) {
|
||||
return {};
|
||||
}
|
||||
const summary = summaryFromRow(row);
|
||||
return {
|
||||
...(summary ? { summary } : {}),
|
||||
...(row.markdown !== null ? { markdown: row.markdown } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async materializeSessionArtifacts(
|
||||
sessionOrSelector: TranscriptSessionDescriptor | string,
|
||||
kind: StoreTypes.TranscriptArtifactKind,
|
||||
): Promise<StoreTypes.MaterializedTranscriptArtifacts> {
|
||||
const session =
|
||||
typeof sessionOrSelector === "string"
|
||||
? await this.readSession(sessionOrSelector)
|
||||
: this.readSessionByIdentity(sessionOrSelector);
|
||||
if (!session) {
|
||||
const selector =
|
||||
typeof sessionOrSelector === "string" ? sessionOrSelector : sessionOrSelector.sessionId;
|
||||
throw new Error(`transcripts session not found: ${selector}`);
|
||||
}
|
||||
return await withOpenClawStateLease(
|
||||
{
|
||||
scope: "meeting-transcript.export",
|
||||
key: transcriptSessionExportKey(session),
|
||||
database: { scope: "shared", options: this.databaseOptions },
|
||||
leaseMs: 60_000,
|
||||
waitMs: 10_000,
|
||||
leaseLabel: "meeting transcript export lease",
|
||||
operationLabel: "meeting-transcripts.export.lease",
|
||||
},
|
||||
async () => await this.materializeSessionArtifactsOwned(session, kind),
|
||||
);
|
||||
}
|
||||
|
||||
private async materializeSessionArtifactsOwned(
|
||||
session: TranscriptSessionDescriptor,
|
||||
kind: StoreTypes.TranscriptArtifactKind,
|
||||
): Promise<StoreTypes.MaterializedTranscriptArtifacts> {
|
||||
const sessionDir = this.sessionDir(session);
|
||||
const metadataPath = path.join(sessionDir, "metadata.json");
|
||||
const transcriptPath = path.join(sessionDir, "transcript.jsonl");
|
||||
const summaryJsonPath = path.join(sessionDir, "summary.json");
|
||||
const summaryPath = path.join(sessionDir, "summary.md");
|
||||
// Every export starts with identity metadata, so even an interrupted partial
|
||||
// materialization remains inspectable by Doctor without guessing its owner.
|
||||
const includeMetadata = true;
|
||||
const includeTranscript = kind === "all" || kind === "transcript";
|
||||
const includeSummary = kind === "all" || kind === "summary";
|
||||
const storedSummary = includeSummary ? await this.readSummary(session) : {};
|
||||
const exportedHashes: Record<string, string> = {};
|
||||
const removedExports = new Set<string>();
|
||||
await assertTranscriptExportPathAvailable({
|
||||
session,
|
||||
exportRootDir: this.exportRootDir,
|
||||
databaseOptions: this.databaseOptions,
|
||||
});
|
||||
await this.assertExportDestinationOwned(session);
|
||||
const pendingFiles = [
|
||||
"metadata.json",
|
||||
...(includeTranscript ? ["transcript.jsonl"] : []),
|
||||
...(includeSummary ? ["summary.json", "summary.md"] : []),
|
||||
];
|
||||
this.markPendingExports(session, pendingFiles);
|
||||
const ensured = await ensureAbsoluteDirectory(sessionDir, {
|
||||
mode: 0o700,
|
||||
scopeLabel: "transcript export directory",
|
||||
});
|
||||
if (!ensured.ok) {
|
||||
throw ensured.error;
|
||||
}
|
||||
if (includeMetadata) {
|
||||
exportedHashes["metadata.json"] = await writeTranscriptArtifact(
|
||||
sessionDir,
|
||||
"metadata.json",
|
||||
`${JSON.stringify(session, null, 2)}\n`,
|
||||
);
|
||||
}
|
||||
if (includeTranscript) {
|
||||
exportedHashes["transcript.jsonl"] = await writeTranscriptJsonlArtifact({
|
||||
sessionDir,
|
||||
session,
|
||||
databaseOptions: this.databaseOptions,
|
||||
});
|
||||
}
|
||||
if (includeSummary) {
|
||||
if (storedSummary.summary) {
|
||||
exportedHashes["summary.json"] = await writeTranscriptArtifact(
|
||||
sessionDir,
|
||||
"summary.json",
|
||||
`${JSON.stringify(storedSummary.summary, null, 2)}\n`,
|
||||
);
|
||||
} else {
|
||||
await removeTranscriptArtifact(sessionDir, "summary.json");
|
||||
removedExports.add("summary.json");
|
||||
}
|
||||
if (storedSummary.markdown !== undefined) {
|
||||
exportedHashes["summary.md"] = await writeTranscriptArtifact(
|
||||
sessionDir,
|
||||
"summary.md",
|
||||
normalizeExportText(storedSummary.markdown),
|
||||
);
|
||||
} else {
|
||||
await removeTranscriptArtifact(sessionDir, "summary.md");
|
||||
removedExports.add("summary.md");
|
||||
}
|
||||
}
|
||||
if (Object.keys(exportedHashes).length > 0 || removedExports.size > 0) {
|
||||
this.updateExportManifest(session, exportedHashes, removedExports);
|
||||
}
|
||||
return {
|
||||
sessionDir,
|
||||
metadataPath,
|
||||
transcriptPath,
|
||||
summaryJsonPath,
|
||||
summaryPath,
|
||||
hasSummary: storedSummary.summary !== undefined || storedSummary.markdown !== undefined,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -403,7 +403,7 @@ describe("scripts/test-docker-all scheduler", () => {
|
||||
const summary = JSON.parse(readFileSync(path.join(logDir, "summary.json"), "utf8"));
|
||||
expect(summary.status).toBe("passed");
|
||||
expect(summary.lanes).toEqual([]);
|
||||
expect(summary.omittedUnsupportedLanes).toHaveLength(10);
|
||||
expect(summary.omittedUnsupportedLanes).toHaveLength(11);
|
||||
expect(summary.omittedUnsupportedLanes).toContain("published-upgrade-survivor");
|
||||
expect(summary.omittedUnsupportedLanes).toContain(
|
||||
"published-upgrade-survivor-versioned-runtime-deps",
|
||||
|
||||
@@ -793,6 +793,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
|
||||
"published-upgrade-survivor-2026.4.29-configured-plugin-installs",
|
||||
"published-upgrade-survivor-2026.4.29-stale-source-plugin-shadow",
|
||||
"published-upgrade-survivor-2026.4.29-tilde-log-path",
|
||||
"published-upgrade-survivor-2026.4.29-meeting-transcripts-sqlite",
|
||||
"published-upgrade-survivor-2026.4.29-versioned-runtime-deps",
|
||||
]);
|
||||
});
|
||||
@@ -830,6 +831,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
|
||||
]);
|
||||
expect(plan.omittedUnsupportedLanes).toEqual([
|
||||
"published-upgrade-survivor-2026.6.11-acpx-openclaw-tools-bridge",
|
||||
"published-upgrade-survivor-2026.6.11-meeting-transcripts-sqlite",
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -867,6 +869,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
|
||||
|
||||
expect(plan.omittedUnsupportedLanes).toEqual([
|
||||
"published-upgrade-survivor-2026.6.11-acpx-openclaw-tools-bridge",
|
||||
"published-upgrade-survivor-2026.6.11-meeting-transcripts-sqlite",
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -882,7 +885,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
|
||||
});
|
||||
|
||||
expect(plan.lanes).toEqual([]);
|
||||
expect(plan.omittedUnsupportedLanes).toHaveLength(10);
|
||||
expect(plan.omittedUnsupportedLanes).toHaveLength(11);
|
||||
expect(plan.omittedUnsupportedLanes).toContain("published-upgrade-survivor-2026.6.11");
|
||||
expect(plan.omittedUnsupportedLanes).toContain(
|
||||
"published-upgrade-survivor-2026.6.11-versioned-runtime-deps",
|
||||
@@ -927,7 +930,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
|
||||
});
|
||||
|
||||
expect(plan.lanes.map((lane) => lane.name)).toEqual(["plugin-binding-command-escape"]);
|
||||
expect(plan.omittedUnsupportedLanes).toHaveLength(10);
|
||||
expect(plan.omittedUnsupportedLanes).toHaveLength(11);
|
||||
expect(plan.omittedUnsupportedLanes).toContain("published-upgrade-survivor");
|
||||
expect(plan.omittedUnsupportedLanes).toContain(
|
||||
"published-upgrade-survivor-versioned-runtime-deps",
|
||||
@@ -1061,6 +1064,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
|
||||
"published-upgrade-survivor-2026.4.29-configured-plugin-installs",
|
||||
"published-upgrade-survivor-2026.4.29-stale-source-plugin-shadow",
|
||||
"published-upgrade-survivor-2026.4.29-tilde-log-path",
|
||||
"published-upgrade-survivor-2026.4.29-meeting-transcripts-sqlite",
|
||||
"published-upgrade-survivor-2026.4.29-versioned-runtime-deps",
|
||||
"published-upgrade-survivor-2026.4.22",
|
||||
"published-upgrade-survivor-2026.4.22-acpx-openclaw-tools-bridge",
|
||||
@@ -1070,6 +1074,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
|
||||
"published-upgrade-survivor-2026.4.22-configured-plugin-installs",
|
||||
"published-upgrade-survivor-2026.4.22-stale-source-plugin-shadow",
|
||||
"published-upgrade-survivor-2026.4.22-tilde-log-path",
|
||||
"published-upgrade-survivor-2026.4.22-meeting-transcripts-sqlite",
|
||||
"published-upgrade-survivor-2026.4.22-versioned-runtime-deps",
|
||||
"published-upgrade-survivor-2026.4.21",
|
||||
"published-upgrade-survivor-2026.4.21-feishu-channel",
|
||||
@@ -1078,6 +1083,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
|
||||
"published-upgrade-survivor-2026.4.21-configured-plugin-installs",
|
||||
"published-upgrade-survivor-2026.4.21-stale-source-plugin-shadow",
|
||||
"published-upgrade-survivor-2026.4.21-tilde-log-path",
|
||||
"published-upgrade-survivor-2026.4.21-meeting-transcripts-sqlite",
|
||||
"published-upgrade-survivor-2026.4.21-versioned-runtime-deps",
|
||||
"published-upgrade-survivor-2026.3.13",
|
||||
"published-upgrade-survivor-2026.3.13-feishu-channel",
|
||||
@@ -1086,6 +1092,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
|
||||
"published-upgrade-survivor-2026.3.13-configured-plugin-installs",
|
||||
"published-upgrade-survivor-2026.3.13-stale-source-plugin-shadow",
|
||||
"published-upgrade-survivor-2026.3.13-tilde-log-path",
|
||||
"published-upgrade-survivor-2026.3.13-meeting-transcripts-sqlite",
|
||||
"published-upgrade-survivor-2026.3.13-versioned-runtime-deps",
|
||||
]);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user