From 2b003f186dd8723310fdcb8a69bc0c714d86ba4f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 4 Aug 2026 20:55:58 -0700 Subject: [PATCH] refactor(memory-wiki): reuse canonical plugin normalization (#119449) --- extensions/memory-wiki/src/chatgpt-import.ts | 56 ++++-------- extensions/memory-wiki/src/import-insights.ts | 19 ++-- .../memory-wiki/src/import-runs-state.ts | 84 +++++------------- extensions/memory-wiki/src/markdown.ts | 88 +++++++------------ extensions/memory-wiki/src/wiki-overview.ts | 17 ++-- 5 files changed, 85 insertions(+), 179 deletions(-) diff --git a/extensions/memory-wiki/src/chatgpt-import.ts b/extensions/memory-wiki/src/chatgpt-import.ts index e8615fbd5b8e..206ec3856119 100644 --- a/extensions/memory-wiki/src/chatgpt-import.ts +++ b/extensions/memory-wiki/src/chatgpt-import.ts @@ -8,7 +8,11 @@ import { } from "openclaw/plugin-sdk/memory-host-markdown"; import { timestampMsToIsoString } from "openclaw/plugin-sdk/number-runtime"; import { FsSafeError, root as fsRoot } from "openclaw/plugin-sdk/security-runtime"; -import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { + asNullableRecord, + isRecord, + uniqueStrings, +} from "openclaw/plugin-sdk/string-coerce-runtime"; import { compileMemoryWikiVault } from "./compile.js"; import type { ResolvedMemoryWikiConfig } from "./config.js"; import { @@ -134,19 +138,12 @@ export type ChatGptRollbackResult = { alreadyRolledBack: boolean; }; -function asRecord(value: unknown): Record | null { - if (!value || typeof value !== "object" || Array.isArray(value)) { - return null; - } - return value as Record; -} - function normalizeWhitespace(value: string): string { return value.trim().replace(/\s+/g, " "); } function isMissingConversationPageError(error: unknown): boolean { - return asRecord(error)?.code === "ENOENT"; + return asNullableRecord(error)?.code === "ENOENT"; } async function readExistingConversationPage(absolutePath: string): Promise { @@ -186,30 +183,13 @@ async function loadConversations(exportInputPath: string): Promise<{ const { exportPath, conversationsPath } = resolveConversationSourcePath(exportInputPath); const raw = await fs.readFile(conversationsPath, "utf8"); const parsed = JSON.parse(raw) as unknown; - if (Array.isArray(parsed)) { - return { - exportPath, - conversationsPath, - conversations: parsed.filter( - (entry): entry is Record => asRecord(entry) !== null, - ), - }; + const conversations = Array.isArray(parsed) + ? parsed + : Object.values(asNullableRecord(parsed) ?? {}).find(Array.isArray); + if (!conversations) { + throw new Error(`Unrecognized ChatGPT conversations export format: ${conversationsPath}`); } - const record = asRecord(parsed); - if (record) { - for (const value of Object.values(record)) { - if (Array.isArray(value)) { - return { - exportPath, - conversationsPath, - conversations: value.filter( - (entry): entry is Record => asRecord(entry) !== null, - ), - }; - } - } - } - throw new Error(`Unrecognized ChatGPT conversations export format: ${conversationsPath}`); + return { exportPath, conversationsPath, conversations: conversations.filter(isRecord) }; } function isoFromUnix(raw: unknown): string | undefined { @@ -249,7 +229,7 @@ function cleanMessageText(value: string): string { } function extractMessageText(message: Record): string { - const content = asRecord(message.content); + const content = asNullableRecord(message.content); if (content) { const parts = content.parts; if (Array.isArray(parts)) { @@ -262,7 +242,7 @@ function extractMessageText(message: Record): string { } continue; } - const partRecord = asRecord(part); + const partRecord = asNullableRecord(part); if (partRecord && typeof partRecord.text === "string" && partRecord.text.trim()) { collected.push(partRecord.text.trim()); } @@ -277,7 +257,7 @@ function extractMessageText(message: Record): string { } function activeBranchMessages(conversation: Record): ChatGptMessage[] { - const mapping = asRecord(conversation.mapping); + const mapping = asNullableRecord(conversation.mapping); if (!mapping) { return []; } @@ -287,13 +267,13 @@ function activeBranchMessages(conversation: Record): ChatGptMes const chain: ChatGptMessage[] = []; while (currentNode && !seen.has(currentNode)) { seen.add(currentNode); - const node = asRecord(mapping[currentNode]); + const node = asNullableRecord(mapping[currentNode]); if (!node) { break; } - const message = asRecord(node.message); + const message = asNullableRecord(node.message); if (message) { - const author = asRecord(message.author); + const author = asNullableRecord(message.author); const role = typeof author?.role === "string" ? author.role : "unknown"; const text = extractMessageText(message); if (text) { diff --git a/extensions/memory-wiki/src/import-insights.ts b/extensions/memory-wiki/src/import-insights.ts index 6ff8a89526bc..4a5679cfee5f 100644 --- a/extensions/memory-wiki/src/import-insights.ts +++ b/extensions/memory-wiki/src/import-insights.ts @@ -1,3 +1,4 @@ +import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; // Memory Wiki plugin module implements import insights behavior. import type { ResolvedMemoryWikiConfig } from "./config.js"; @@ -61,14 +62,6 @@ function normalizeFiniteInt(value: unknown): number { return Math.max(0, Math.floor(value)); } -function normalizeTimestamp(value: unknown): string | undefined { - if (typeof value !== "string") { - return undefined; - } - const trimmed = value.trim(); - return trimmed.length > 0 ? trimmed : undefined; -} - function humanizeLabelSuffix(label: string): string { const suffix = label.includes("/") ? label.split("/").slice(1).join("/") : label; return suffix @@ -348,6 +341,8 @@ export async function listMemoryWikiImportInsights( const lastUserLine = exposeImportContent ? extractDigestField(digestLines, "Last user line") : undefined; + const createdAt = normalizeOptionalString(parsed.frontmatter.createdAt); + const updatedAt = normalizeOptionalString(parsed.frontmatter.updatedAt); return [ { pagePath: page.relativePath, @@ -381,12 +376,8 @@ export async function listMemoryWikiImportInsights( candidateSignals, correctionSignals, preferenceSignals, - ...(normalizeTimestamp(parsed.frontmatter.createdAt) - ? { createdAt: normalizeTimestamp(parsed.frontmatter.createdAt) } - : {}), - ...(normalizeTimestamp(parsed.frontmatter.updatedAt) - ? { updatedAt: normalizeTimestamp(parsed.frontmatter.updatedAt) } - : {}), + ...(createdAt ? { createdAt } : {}), + ...(updatedAt ? { updatedAt } : {}), } satisfies MemoryWikiImportInsightItem, ]; }) diff --git a/extensions/memory-wiki/src/import-runs-state.ts b/extensions/memory-wiki/src/import-runs-state.ts index 5b420b915c97..27459231718d 100644 --- a/extensions/memory-wiki/src/import-runs-state.ts +++ b/extensions/memory-wiki/src/import-runs-state.ts @@ -6,6 +6,11 @@ import type { OpenKeyedStoreOptions, PluginStateKeyedStore, } from "openclaw/plugin-sdk/plugin-state-runtime"; +import { + asNullableRecord, + normalizeOptionalString, + normalizeUniqueTrimmedStringList, +} from "openclaw/plugin-sdk/string-coerce-runtime"; import pMap, { pMapSkip } from "p-map"; import { walkMemoryWikiDirectory } from "./bounded-walk.js"; @@ -113,13 +118,6 @@ function cloneImportRunRecord(record: ChatGptImportRunRecord): ChatGptImportRunR }; } -function asRecord(value: unknown): Record | null { - if (!value || typeof value !== "object" || Array.isArray(value)) { - return null; - } - return value as Record; -} - function normalizeImportRunEntries(value: unknown): ChatGptImportRunEntry[] { if (!Array.isArray(value)) { return []; @@ -129,7 +127,7 @@ function normalizeImportRunEntries(value: unknown): ChatGptImportRunEntry[] { const entryPath = raw.trim(); return entryPath ? [{ path: entryPath }] : []; } - const entry = asRecord(raw); + const entry = asNullableRecord(raw); if (!entry) { return []; } @@ -137,15 +135,9 @@ function normalizeImportRunEntries(value: unknown): ChatGptImportRunEntry[] { if (!entryPath) { return []; } - const snapshotPath = - typeof entry.snapshotPath === "string" && entry.snapshotPath.trim() - ? entry.snapshotPath.trim() - : undefined; - const contentHash = - typeof entry.contentHash === "string" && entry.contentHash.trim() - ? entry.contentHash.trim() - : undefined; - const recoveryPaths = normalizeStringArray(entry.recoveryPaths); + const snapshotPath = normalizeOptionalString(entry.snapshotPath); + const contentHash = normalizeOptionalString(entry.contentHash); + const recoveryPaths = normalizeUniqueTrimmedStringList(entry.recoveryPaths); return [ { path: entryPath, @@ -157,33 +149,19 @@ function normalizeImportRunEntries(value: unknown): ChatGptImportRunEntry[] { }); } -function normalizeStringArray(value: unknown): string[] { - if (!Array.isArray(value)) { - return []; - } - return [ - ...new Set( - value - .filter((entry): entry is string => typeof entry === "string") - .map((entry) => entry.trim()) - .filter(Boolean), - ), - ]; -} - function asNonNegativeInteger(value: unknown): number { return typeof value === "number" && Number.isFinite(value) ? Math.max(0, Math.floor(value)) : 0; } function normalizeMemoryWikiImportRunRecord(raw: unknown): ChatGptImportRunRecord | null { - const record = asRecord(raw); + const record = asNullableRecord(raw); if (!record) { return null; } - const runId = typeof record.runId === "string" ? record.runId.trim() : ""; - const exportPath = typeof record.exportPath === "string" ? record.exportPath.trim() : ""; - const sourcePath = typeof record.sourcePath === "string" ? record.sourcePath.trim() : ""; - const appliedAt = typeof record.appliedAt === "string" ? record.appliedAt.trim() : ""; + const runId = normalizeOptionalString(record.runId) ?? ""; + const exportPath = normalizeOptionalString(record.exportPath) ?? ""; + const sourcePath = normalizeOptionalString(record.sourcePath) ?? ""; + const appliedAt = normalizeOptionalString(record.appliedAt) ?? ""; if ( record.version !== 1 || record.importType !== "chatgpt" || @@ -194,19 +172,9 @@ function normalizeMemoryWikiImportRunRecord(raw: unknown): ChatGptImportRunRecor ) { return null; } - const rolledBackAt = - typeof record.rolledBackAt === "string" && record.rolledBackAt.trim() - ? record.rolledBackAt.trim() - : undefined; - const rollbackStartedAt = - typeof record.rollbackStartedAt === "string" && record.rollbackStartedAt.trim() - ? record.rollbackStartedAt.trim() - : undefined; - const rollbackTargetsFinalizedAt = - typeof record.rollbackTargetsFinalizedAt === "string" && - record.rollbackTargetsFinalizedAt.trim() - ? record.rollbackTargetsFinalizedAt.trim() - : undefined; + const rolledBackAt = normalizeOptionalString(record.rolledBackAt); + const rollbackStartedAt = normalizeOptionalString(record.rollbackStartedAt); + const rollbackTargetsFinalizedAt = normalizeOptionalString(record.rollbackTargetsFinalizedAt); return { version: 1, runId, @@ -227,7 +195,7 @@ function normalizeMemoryWikiImportRunRecord(raw: unknown): ChatGptImportRunRecor } function normalizeMetaRecord(raw: unknown): MemoryWikiImportRunMetaStateRecord | null { - const record = asRecord(raw); + const record = asNullableRecord(raw); if (!record || record.kind !== "meta") { return null; } @@ -247,7 +215,7 @@ function normalizeMetaRecord(raw: unknown): MemoryWikiImportRunMetaStateRecord | } function normalizePathRecord(raw: unknown): MemoryWikiImportRunPathStateRecord | null { - const record = asRecord(raw); + const record = asNullableRecord(raw); if ( !record || (record.kind !== "created-path" && record.kind !== "updated-path") || @@ -259,15 +227,9 @@ function normalizePathRecord(raw: unknown): MemoryWikiImportRunPathStateRecord | ) { return null; } - const snapshotPath = - typeof record.snapshotPath === "string" && record.snapshotPath.trim() - ? record.snapshotPath.trim() - : undefined; - const contentHash = - typeof record.contentHash === "string" && record.contentHash.trim() - ? record.contentHash.trim() - : undefined; - const recoveryPaths = normalizeStringArray(record.recoveryPaths); + const snapshotPath = normalizeOptionalString(record.snapshotPath); + const contentHash = normalizeOptionalString(record.contentHash); + const recoveryPaths = normalizeUniqueTrimmedStringList(record.recoveryPaths); return { kind: record.kind, vaultRootKey: record.vaultRootKey, @@ -545,7 +507,7 @@ export async function readLegacyMemoryWikiImportRunRecords( ? "include" : "skip", }).catch((error: unknown) => { - const code = asRecord(error)?.code; + const code = asNullableRecord(error)?.code; if (code === "ENOENT") { return []; } diff --git a/extensions/memory-wiki/src/markdown.ts b/extensions/memory-wiki/src/markdown.ts index 5c7fedb048b2..6318a9c77495 100644 --- a/extensions/memory-wiki/src/markdown.ts +++ b/extensions/memory-wiki/src/markdown.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { fromMarkdown } from "mdast-util-from-markdown"; import { asFiniteNumber, + asNullableRecord, normalizeLowercaseStringOrEmpty, normalizeOptionalString, normalizeSingleOrTrimmedStringList, @@ -204,15 +205,15 @@ export function parseWikiMarkdown(content: string): ParsedWikiMarkdown { if (frontmatter === undefined) { return { hasFrontmatter: false, frontmatter: {}, body: content }; } - const parsed: unknown = YAML.parse(frontmatter); - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + const parsed = asNullableRecord(YAML.parse(frontmatter) as unknown); + if (!parsed) { // Every writer spreads this value back into YAML. Reject non-mapping roots // so an edit cannot silently replace scalar or sequence frontmatter. throw new TypeError("Wiki frontmatter must be a YAML mapping"); } return { hasFrontmatter: true, - frontmatter: parsed as Record, + frontmatter: parsed, body: content.slice(match[0].length), }; } @@ -235,10 +236,10 @@ export function normalizeSourceIds(value: unknown): string[] { } function normalizeWikiClaimEvidence(value: unknown): WikiClaimEvidence | null { - if (!value || typeof value !== "object" || Array.isArray(value)) { + const record = asNullableRecord(value); + if (!record) { return null; } - const record = value as Record; const kind = normalizeOptionalString(record.kind); const sourceId = normalizeOptionalString(record.sourceId); const evidencePath = normalizeOptionalString(record.path); @@ -246,9 +247,8 @@ function normalizeWikiClaimEvidence(value: unknown): WikiClaimEvidence | null { const note = normalizeOptionalString(record.note); const updatedAt = normalizeOptionalString(record.updatedAt); const privacyTier = normalizeOptionalString(record.privacyTier); - const weight = - typeof record.weight === "number" && Number.isFinite(record.weight) ? record.weight : undefined; - const confidence = normalizeOptionalNumber(record.confidence); + const weight = asFiniteNumber(record.weight); + const confidence = asFiniteNumber(record.confidence); if ( !kind && !sourceId && @@ -280,10 +280,10 @@ export function normalizeWikiClaims(value: unknown): WikiClaim[] { return []; } return value.flatMap((entry) => { - if (!entry || typeof entry !== "object" || Array.isArray(entry)) { + const record = asNullableRecord(entry); + if (!record) { return []; } - const record = entry as Record; const text = normalizeOptionalString(record.text); if (!text) { return []; @@ -294,60 +294,46 @@ export function normalizeWikiClaims(value: unknown): WikiClaim[] { return normalized ? [normalized] : []; }) : []; - const confidence = - typeof record.confidence === "number" && Number.isFinite(record.confidence) - ? record.confidence - : undefined; + const confidence = asFiniteNumber(record.confidence); + const status = normalizeOptionalString(record.status); + const updatedAt = normalizeOptionalString(record.updatedAt); return [ { ...(normalizeOptionalString(record.id) ? { id: normalizeOptionalString(record.id) } : {}), text, - ...(normalizeOptionalString(record.status) - ? { status: normalizeOptionalString(record.status) } - : {}), + ...(status ? { status } : {}), ...(confidence !== undefined ? { confidence } : {}), evidence, - ...(normalizeOptionalString(record.updatedAt) - ? { updatedAt: normalizeOptionalString(record.updatedAt) } - : {}), + ...(updatedAt ? { updatedAt } : {}), }, ]; }); } -function normalizeOptionalNumber(value: unknown): number | undefined { - return asFiniteNumber(value); -} - function normalizeWikiPersonCard(value: unknown): WikiPersonCard | undefined { - if (!value || typeof value !== "object" || Array.isArray(value)) { + const record = asNullableRecord(value); + if (!record) { return undefined; } - const record = value as Record; + const canonicalId = normalizeOptionalString(record.canonicalId); + const timezone = normalizeOptionalString(record.timezone); + const confidence = asFiniteNumber(record.confidence); + const privacyTier = normalizeOptionalString(record.privacyTier); + const lastRefreshedAt = normalizeOptionalString(record.lastRefreshedAt); const card: WikiPersonCard = { - ...(normalizeOptionalString(record.canonicalId) - ? { canonicalId: normalizeOptionalString(record.canonicalId) } - : {}), + ...(canonicalId ? { canonicalId } : {}), handles: normalizeSingleOrTrimmedStringList(record.handles), socials: normalizeSingleOrTrimmedStringList(record.socials), emails: normalizeSingleOrTrimmedStringList(record.emails ?? record.email), - ...(normalizeOptionalString(record.timezone) - ? { timezone: normalizeOptionalString(record.timezone) } - : {}), + ...(timezone ? { timezone } : {}), ...(normalizeOptionalString(record.lane) ? { lane: normalizeOptionalString(record.lane) } : {}), askFor: normalizeSingleOrTrimmedStringList(record.askFor), avoidAskingFor: normalizeSingleOrTrimmedStringList(record.avoidAskingFor), bestUsedFor: normalizeSingleOrTrimmedStringList(record.bestUsedFor), notEnoughFor: normalizeSingleOrTrimmedStringList(record.notEnoughFor), - ...(normalizeOptionalNumber(record.confidence) !== undefined - ? { confidence: normalizeOptionalNumber(record.confidence) } - : {}), - ...(normalizeOptionalString(record.privacyTier) - ? { privacyTier: normalizeOptionalString(record.privacyTier) } - : {}), - ...(normalizeOptionalString(record.lastRefreshedAt) - ? { lastRefreshedAt: normalizeOptionalString(record.lastRefreshedAt) } - : {}), + ...(confidence !== undefined ? { confidence } : {}), + ...(privacyTier ? { privacyTier } : {}), + ...(lastRefreshedAt ? { lastRefreshedAt } : {}), }; const hasAnyValue = Boolean( @@ -369,10 +355,12 @@ function normalizeWikiRelationships(value: unknown): WikiRelationship[] { return []; } return value.flatMap((entry) => { - if (!entry || typeof entry !== "object" || Array.isArray(entry)) { + const record = asNullableRecord(entry); + if (!record) { return []; } - const record = entry as Record; + const weight = asFiniteNumber(record.weight); + const confidence = asFiniteNumber(record.confidence); const relationship: WikiRelationship = { ...(normalizeOptionalString(record.targetId) ? { targetId: normalizeOptionalString(record.targetId) } @@ -386,12 +374,8 @@ function normalizeWikiRelationships(value: unknown): WikiRelationship[] { ...(normalizeOptionalString(record.kind) ? { kind: normalizeOptionalString(record.kind) } : {}), - ...(normalizeOptionalNumber(record.weight) !== undefined - ? { weight: normalizeOptionalNumber(record.weight) } - : {}), - ...(normalizeOptionalNumber(record.confidence) !== undefined - ? { confidence: normalizeOptionalNumber(record.confidence) } - : {}), + ...(weight !== undefined ? { weight } : {}), + ...(confidence !== undefined ? { confidence } : {}), ...(normalizeOptionalString(record.evidenceKind) ? { evidenceKind: normalizeOptionalString(record.evidenceKind) } : {}), @@ -744,11 +728,7 @@ export function scanWikiPageSummary(params: { claims: normalizeWikiClaims(parsed.frontmatter.claims), contradictions: normalizeSingleOrTrimmedStringList(parsed.frontmatter.contradictions), questions: normalizeSingleOrTrimmedStringList(parsed.frontmatter.questions), - confidence: - typeof parsed.frontmatter.confidence === "number" && - Number.isFinite(parsed.frontmatter.confidence) - ? parsed.frontmatter.confidence - : undefined, + confidence: asFiniteNumber(parsed.frontmatter.confidence), privacyTier: normalizeOptionalString(parsed.frontmatter.privacyTier), personCard: normalizeWikiPersonCard(parsed.frontmatter.personCard), relationships: normalizeWikiRelationships(parsed.frontmatter.relationships), diff --git a/extensions/memory-wiki/src/wiki-overview.ts b/extensions/memory-wiki/src/wiki-overview.ts index 4155ef87a1ab..46f3b9fad90b 100644 --- a/extensions/memory-wiki/src/wiki-overview.ts +++ b/extensions/memory-wiki/src/wiki-overview.ts @@ -1,4 +1,5 @@ // Memory Wiki plugin module implements the memory wiki overview. +import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import type { ResolvedMemoryWikiConfig } from "./config.js"; import { parseWikiMarkdown, type WikiPageKind } from "./markdown.js"; import { readQueryableWikiPages } from "./query.js"; @@ -62,14 +63,6 @@ function createEmptyOverviewPageCounts(): MemoryWikiOverviewPageCounts { }; } -function normalizeTimestamp(value: unknown): string | undefined { - if (typeof value !== "string") { - return undefined; - } - const trimmed = value.trim(); - return trimmed.length > 0 ? trimmed : undefined; -} - function extractSnippet(body: string): string | undefined { for (const rawLine of body.split(/\r?\n/)) { const line = rawLine.trim(); @@ -114,13 +107,13 @@ export async function listMemoryWikiOverview( const items = pages .map((page) => { const parsed = parseWikiMarkdown(page.raw); + const updatedAt = normalizeOptionalString(page.updatedAt); + const sourceType = normalizeOptionalString(page.sourceType); return Object.assign( { pagePath: page.relativePath, title: page.title, kind: page.kind }, page.id ? { id: page.id } : {}, - normalizeTimestamp(page.updatedAt) ? { updatedAt: normalizeTimestamp(page.updatedAt) } : {}, - typeof page.sourceType === `string` && page.sourceType.trim().length > 0 - ? { sourceType: page.sourceType.trim() } - : {}, + updatedAt ? { updatedAt } : {}, + sourceType ? { sourceType } : {}, { claimCount: page.claims.length, questionCount: page.questions.length,