From f59392fdf5e6b1e2f941bfb766beec685e221be7 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 18 Aug 2026 13:37:17 -0700 Subject: [PATCH] fix(channels): redact logs and reject unknown filters (#125939) * fix(channels): redact and validate log filters * fix(logging): preserve filtered redacted channel tails * fix(channels): match log filters on exact boundaries * fix(logging): honor redaction off mode --- src/cli/channels-cli.test.ts | 17 ++ src/cli/channels-cli.ts | 2 +- src/commands/channels.logs.test.ts | 272 +++++++++++++++++------------ src/commands/channels/logs.ts | 138 +++++++-------- src/logging/log-tail.test.ts | 37 +++- src/logging/log-tail.ts | 46 ++++- src/logging/parse-log-line.test.ts | 13 ++ src/logging/parse-log-line.ts | 20 ++- src/logging/redact.test.ts | 15 +- src/logging/redact.ts | 10 +- 10 files changed, 348 insertions(+), 222 deletions(-) diff --git a/src/cli/channels-cli.test.ts b/src/cli/channels-cli.test.ts index 6ec9bee7e0f2..f78e60990f62 100644 --- a/src/cli/channels-cli.test.ts +++ b/src/cli/channels-cli.test.ts @@ -17,6 +17,9 @@ const listRawChannelPluginCatalogEntriesMock = vi.hoisted(() => vi.fn<() => ChannelPluginCatalogEntry[]>(() => []), ); const channelsAddCommandMock = vi.hoisted(() => vi.fn(async () => undefined)); +const channelsLogsCommandMock = vi.hoisted(() => + vi.fn(async (_options: { channel?: string }, _runtime: unknown) => undefined), +); const channelsResolveCommandMock = vi.hoisted(() => vi.fn(async () => undefined)); const runtimeMock = vi.hoisted(() => ({ log: vi.fn(), @@ -34,6 +37,7 @@ vi.mock("../channels/plugins/catalog.js", () => ({ vi.mock("../commands/channels.js", () => ({ channelsAddCommand: channelsAddCommandMock, + channelsLogsCommand: channelsLogsCommandMock, channelsResolveCommand: channelsResolveCommandMock, })); @@ -91,6 +95,19 @@ describe("registerChannelsCli", () => { expect(getChannelSubcommandNames(program, "dead-letters")).toEqual(["list", "resubmit"]); }); + it.each([ + ["omitted", ["channels", "logs"], undefined], + ["explicit all", ["channels", "logs", "--channel", "all"], "all"], + ])("distinguishes an %s channels logs filter", async (_label, args, expectedChannel) => { + const program = new Command().name("openclaw").exitOverride(); + + await registerChannelsCli(program, ["node", "openclaw", ...args]); + await program.parseAsync(args, { from: "user" }); + + const [options] = channelsLogsCommandMock.mock.calls[0] ?? []; + expect(options?.channel).toBe(expectedChannel); + }); + it.each(["auto", "user", "group", "channel"])( "forwards the supported %s resolve target kind", async (kind) => { diff --git a/src/cli/channels-cli.ts b/src/cli/channels-cli.ts index d12532f3a9f3..dcf77b818077 100644 --- a/src/cli/channels-cli.ts +++ b/src/cli/channels-cli.ts @@ -257,7 +257,7 @@ export async function registerChannelsCli( channels .command("logs") .description("Show recent channel logs from the gateway log file") - .option("--channel ", `Channel (${formatCliChannelOptions(["all"])})`, "all") + .option("--channel ", `Channel (${formatCliChannelOptions(["all"])}; default: all)`) .option("--lines ", "Number of lines (default: 200)", "200") .option("--json", "Output JSON", false) .action(async (opts) => { diff --git a/src/commands/channels.logs.test.ts b/src/commands/channels.logs.test.ts index a8155f705b1a..8917f586c675 100644 --- a/src/commands/channels.logs.test.ts +++ b/src/commands/channels.logs.test.ts @@ -4,17 +4,20 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { setLoggerOverride } from "../logging.js"; +import { registerSecretValueForRedaction } from "../logging/secret-redaction-registry.js"; +import { resetSecretRedactionRegistryForTest } from "../logging/secret-redaction-registry.test-support.js"; import { createTestRuntime } from "./test-runtime-config-helpers.js"; -const pluginRegistryMocks = vi.hoisted(() => ({ - loadPluginRegistrySnapshot: vi.fn(() => ({ plugins: [] })), - listPluginContributionIds: vi.fn(() => ["external-chat"]), -})); +const pluginRegistryMocks = vi.hoisted(() => { + const plugins = [{ id: "vendor-external-chat", channels: ["external-chat"] }]; + return { + loadPluginManifestRegistryForPluginRegistry: vi.fn(() => ({ diagnostics: [], plugins })), + }; +}); vi.mock("../plugins/plugin-registry.js", () => ({ - loadPluginManifestRegistryForPluginRegistry: () => ({ diagnostics: [], plugins: [] }), - loadPluginRegistrySnapshot: pluginRegistryMocks.loadPluginRegistrySnapshot, - listPluginContributionIds: pluginRegistryMocks.listPluginContributionIds, + loadPluginManifestRegistryForPluginRegistry: + pluginRegistryMocks.loadPluginManifestRegistryForPluginRegistry, })); vi.mock("../channels/plugins/index.js", () => ({ @@ -26,20 +29,22 @@ vi.mock("../channels/plugins/index.js", () => ({ import { channelsLogsCommand } from "./channels/logs.js"; const runtime = createTestRuntime(); -type PositionalRead = ( - buffer: Buffer, - offset: number, - length: number, - position: number | null, -) => Promise<{ bytesRead: number; buffer: Buffer }>; - -function logLine(params: { module: string; message: string }) { +function logLine(params: { + subsystem?: string; + module?: string; + plugin?: string; + message: string; +}) { return JSON.stringify({ time: "2026-04-25T12:00:00.000Z", 0: params.message, _meta: { logLevelName: "INFO", - name: JSON.stringify({ module: params.module }), + name: JSON.stringify({ + ...(params.subsystem ? { subsystem: params.subsystem } : {}), + ...(params.module ? { module: params.module } : {}), + ...(params.plugin ? { plugin: params.plugin } : {}), + }), }, }); } @@ -48,7 +53,7 @@ function readJsonPayload() { return JSON.parse(String(runtime.log.mock.calls[0]?.[0])) as { file: string; channel: string; - lines: Array<{ message: string }>; + lines: Array<{ message: string; raw: string }>; }; } @@ -63,12 +68,12 @@ describe("channelsLogsCommand", () => { runtime.log.mockClear(); runtime.error.mockClear(); runtime.exit.mockClear(); - pluginRegistryMocks.loadPluginRegistrySnapshot.mockClear(); - pluginRegistryMocks.listPluginContributionIds.mockClear(); + pluginRegistryMocks.loadPluginManifestRegistryForPluginRegistry.mockClear(); }); afterEach(async () => { vi.restoreAllMocks(); + resetSecretRedactionRegistryForTest(); setLoggerOverride(null); await fs.rm(tempDir, { recursive: true, force: true }); }); @@ -77,24 +82,153 @@ describe("channelsLogsCommand", () => { await fs.writeFile( logPath, [ - logLine({ module: "gateway/channels/external-chat/send", message: "external sent" }), + logLine({ plugin: "vendor-external-chat", message: "external sent" }), + logLine({ plugin: "vendor-external-chat-shadow", message: "shadow sent" }), logLine({ module: "gateway/channels/slack/send", message: "slack sent" }), ].join("\n"), ); await channelsLogsCommand({ channel: "external-chat", json: true }, runtime); - expect(pluginRegistryMocks.loadPluginRegistrySnapshot).toHaveBeenCalledOnce(); - expect(pluginRegistryMocks.listPluginContributionIds).toHaveBeenCalledOnce(); - const [contributionOptions] = pluginRegistryMocks.listPluginContributionIds.mock - .calls[0] as unknown as [{ contribution?: string; includeDisabled?: boolean }]; - expect(contributionOptions?.contribution).toBe("channels"); - expect(contributionOptions?.includeDisabled).toBe(true); + expect(pluginRegistryMocks.loadPluginManifestRegistryForPluginRegistry).toHaveBeenCalledWith({ + includeDisabled: true, + env: process.env, + }); const payload = readJsonPayload(); expect(payload.channel).toBe("external-chat"); expect(payload.lines.map((line) => line.message)).toEqual(["external sent"]); }); + it.each([ + { + label: "subsystem", + channel: "slack", + shadow: { subsystem: "gateway/channels/slack-archive" }, + match: { subsystem: "gateway/channels/slack/send" }, + }, + { + label: "module", + channel: "external-chat", + shadow: { module: "external-chat-shadow" }, + match: { module: "external-chat" }, + }, + ])("excludes a shadow $label while preserving an exact channel match", async (fixture) => { + await fs.writeFile( + logPath, + [ + logLine({ ...fixture.shadow, message: "shadow" }), + logLine({ ...fixture.match, message: "match" }), + ].join("\n"), + ); + + await channelsLogsCommand({ channel: fixture.channel, json: true }, runtime); + + expect(readJsonPayload().lines.map((line) => line.message)).toEqual(["match"]); + }); + + it.each([false, true])( + "rejects an unknown explicit channel without widening output (json=%s)", + async (json) => { + await fs.writeFile( + logPath, + logLine({ module: "gateway/channels/slack/send", message: "unrelated message" }), + ); + + const error = await channelsLogsCommand({ channel: "slakc", json }, runtime).catch( + (cause: unknown) => cause, + ); + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toContain('Unknown channel "slakc". Valid channels: all,'); + expect((error as Error).message).toContain("external-chat"); + expect((error as Error).message).toContain("slack"); + expect(runtime.log).not.toHaveBeenCalled(); + }, + ); + + it("redacts credential-bearing channel lines in text output", async () => { + const fixtureCredential = "opaque-registry-value-1234567890"; + registerSecretValueForRedaction(fixtureCredential); + await fs.writeFile( + logPath, + logLine({ + module: "gateway/channels/slack/send", + message: `opaque=${fixtureCredential}`, + }), + ); + + await channelsLogsCommand({ channel: "slack" }, runtime); + + const output = runtime.log.mock.calls.flat().join("\n"); + expect(output).toContain("2026-04-25T12:00:00.000Z info"); + expect(output).toContain("opaque=opaque…7890"); + expect(output).not.toContain(fixtureCredential); + }); + + it("redacts credential-bearing channel lines in JSON output", async () => { + const fixtureCredential = "opaque-registry-value-1234567890"; + registerSecretValueForRedaction(fixtureCredential); + await fs.writeFile( + logPath, + logLine({ + module: "gateway/channels/slack/send", + message: `opaque=${fixtureCredential}`, + }), + ); + + await channelsLogsCommand({ channel: "slack", json: true }, runtime); + + const payload = readJsonPayload(); + expect(payload.lines[0]?.message).toBe("opaque=opaque…7890"); + expect(JSON.stringify(payload)).not.toContain(fixtureCredential); + }); + + it("preserves ordering and line limits for an explicit all filter", async () => { + await fs.writeFile( + logPath, + [ + logLine({ module: "gateway/channels/slack/send", message: "first" }), + logLine({ module: "gateway/channels/external-chat/send", message: "second" }), + logLine({ module: "gateway/channels/slack/send", message: "third" }), + ].join("\n"), + ); + + await channelsLogsCommand({ channel: "all", lines: 2, json: true }, runtime); + + const payload = readJsonPayload(); + expect(payload.channel).toBe("all"); + expect(payload.lines.map((line) => line.message)).toEqual(["second", "third"]); + }); + + it("finds sparse channel records beyond the shared 5000-line cap", async () => { + const filler = logLine({ module: "gateway/health", message: "ok" }); + const lines = [ + logLine({ module: "gateway/channels/slack/send", message: "first match" }), + ...Array.from({ length: 5000 }, () => filler), + logLine({ module: "gateway/channels/slack/send", message: "second match" }), + ]; + await fs.writeFile(logPath, lines.join("\n")); + + await channelsLogsCommand({ channel: "slack", lines: 2000, json: true }, runtime); + + expect(readJsonPayload().lines.map((line) => line.message)).toEqual([ + "first match", + "second match", + ]); + }); + + it("treats an omitted channel filter as all", async () => { + await fs.writeFile( + logPath, + logLine({ module: "gateway/channels/slack/send", message: "omitted filter" }), + ); + + await channelsLogsCommand({ json: true }, runtime); + + const payload = readJsonPayload(); + expect(payload.channel).toBe("all"); + expect(payload.lines.map((line) => line.message)).toEqual(["omitted filter"]); + }); + it("falls back to the latest rolling log when the configured rolling file is missing", async () => { const configuredFile = path.join(tempDir, "openclaw-2026-04-26.log"); const fallbackFile = path.join(tempDir, "openclaw-2026-04-25.log"); @@ -149,92 +283,6 @@ describe("channelsLogsCommand", () => { expect(payload.lines.map((line) => line.message)).toEqual(["current sent"]); }); - it("fills short positional reads before parsing channel log lines", async () => { - const realOpen = fs.open.bind(fs); - const readLengths: number[] = []; - vi.spyOn(fs, "open").mockImplementation(async (...args) => { - const handle = await realOpen(...args); - const realRead = handle.read.bind(handle) as PositionalRead; - const shortRead = vi.fn((buffer, offset, length, position) => { - readLengths.push(length); - return realRead(buffer, offset, Math.min(length, 4), position); - }); - Object.defineProperty(handle, "read", { configurable: true, value: shortRead }); - return handle; - }); - await fs.writeFile( - logPath, - [ - logLine({ module: "gateway/channels/slack/send", message: "first" }), - logLine({ module: "gateway/channels/slack/send", message: "second" }), - ].join("\n"), - ); - - await channelsLogsCommand({ channel: "slack", json: true }, runtime); - - expect(readJsonPayload().lines.map((line) => line.message)).toEqual(["first", "second"]); - expect(readLengths.length).toBeGreaterThan(1); - }); - - it("returns the first line of the tail window when start aligns with a line boundary", async () => { - // MAX_BYTES in readTailLines is 1_000_000. We build a file of 2_000_000 bytes - // made of 10_000 lines each exactly 200 bytes (199 payload + "\n"), so the - // read window starts at byte offset 1_000_000 which is exactly on a line - // boundary (byte 999_999 is the trailing "\n" of the previous line). - // Without checking the byte before the window, readTailLines drops line 5000 silently. - const LINE_SIZE = 200; - const TOTAL_LINES = 10_000; - const FIRST_INDEX = 5000; // first line of the tail window after alignment - - const buildLine = (message: string) => { - const base = logLine({ - module: "gateway/channels/slack/send", - message, - }); - const payloadLen = LINE_SIZE - 1; // reserve 1 byte for newline - // Re-emit with a padded message so total byte length is constant. - const padNeeded = payloadLen - Buffer.byteLength(base); - if (padNeeded < 0) { - throw new Error(`base log line too long: ${Buffer.byteLength(base)} > ${payloadLen}`); - } - const padded = logLine({ - module: "gateway/channels/slack/send", - message: message + " ".repeat(padNeeded), - }); - if (Buffer.byteLength(padded) !== payloadLen) { - throw new Error(`padded line wrong size: ${Buffer.byteLength(padded)} vs ${payloadLen}`); - } - return padded + "\n"; - }; - - const handle = await fs.open(logPath, "w"); - try { - for (let i = 0; i < TOTAL_LINES; i++) { - let message: string; - if (i === FIRST_INDEX) { - message = "first-line-in-window"; - } else if (i === TOTAL_LINES - 1) { - message = "last-line"; - } else { - message = "filler"; - } - await handle.write(buildLine(message)); - } - } finally { - await handle.close(); - } - - await channelsLogsCommand( - { channel: "slack", json: true, lines: String(TOTAL_LINES) }, - runtime, - ); - - const payload = readJsonPayload(); - const messages = payload.lines.map((line) => line.message.trimEnd()); - expect(messages[0]).toBe("first-line-in-window"); - expect(messages[messages.length - 1]).toBe("last-line"); - }); - it("does not fall back to rolling logs for a missing custom log file", async () => { const configuredFile = path.join(tempDir, "custom-channel.log"); const fallbackFile = path.join(tempDir, "openclaw-2026-04-25.log"); diff --git a/src/commands/channels/logs.ts b/src/commands/channels/logs.ts index 1175a94ae478..7658a7e98907 100644 --- a/src/commands/channels/logs.ts +++ b/src/commands/channels/logs.ts @@ -1,14 +1,14 @@ // Implements channel-scoped tailing of the OpenClaw log file. -import fs from "node:fs/promises"; import { parseStrictPositiveInteger } from "@openclaw/normalization-core/number-coercion"; import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; import { theme } from "../../../packages/terminal-core/src/theme.js"; -import { normalizeChatChannelId as normalizeBundledChannelId } from "../../channels/registry.js"; -import { readFileWindowFully } from "../../infra/file-read.js"; -import { getResolvedLoggerSettings } from "../../logging.js"; -import { resolveLogFile } from "../../logging/log-tail.js"; -import { parseLogLine } from "../../logging/parse-log-line.js"; -import { listManifestChannelContributionIds } from "../../plugins/manifest-contribution-ids.js"; +import { + CHAT_CHANNEL_ORDER, + normalizeChatChannelId as normalizeBundledChannelId, +} from "../../channels/registry.js"; +import { readConfiguredParsedLogTail } from "../../logging/log-tail.js"; +import type { ParsedLogLine } from "../../logging/parse-log-line.js"; +import { loadPluginManifestRegistryForPluginRegistry } from "../../plugins/plugin-registry.js"; import { defaultRuntime, type RuntimeEnv, writeRuntimeJson } from "../../runtime.js"; export type ChannelsLogsOptions = { @@ -17,44 +17,66 @@ export type ChannelsLogsOptions = { json?: boolean; }; -type LogLine = ReturnType; - const DEFAULT_LIMIT = 200; const MAX_BYTES = 1_000_000; -function listManifestChannelIds(): Set { - return new Set( - listManifestChannelContributionIds({ - includeDisabled: true, - env: process.env, +type ChannelLogFilter = { channel: string; pluginIds: ReadonlySet }; +type ManifestChannel = { id: string; pluginId: string }; + +function listManifestChannels(): ManifestChannel[] { + return loadPluginManifestRegistryForPluginRegistry({ + includeDisabled: true, + env: process.env, + }).plugins.flatMap((plugin) => + plugin.channels.flatMap((rawChannel) => { + const id = normalizeLowercaseStringOrEmpty(rawChannel); + return id ? [{ id, pluginId: plugin.id }] : []; }), ); } -function parseChannelFilter(raw?: string) { +function parseChannelFilter(raw?: string): ChannelLogFilter { + if (raw === undefined) { + return { channel: "all", pluginIds: new Set() }; + } const trimmed = normalizeLowercaseStringOrEmpty(raw); - if (!trimmed || trimmed === "all") { - return "all"; + if (trimmed === "all") { + return { channel: "all", pluginIds: new Set() }; } + const manifestChannels = listManifestChannels(); const bundled = normalizeBundledChannelId(trimmed); - if (bundled) { - return bundled; + const channel = bundled ?? trimmed; + const pluginIds = new Set( + manifestChannels.filter((entry) => entry.id === channel).map((entry) => entry.pluginId), + ); + if (bundled || pluginIds.size > 0) { + return { channel, pluginIds }; } - return listManifestChannelIds().has(trimmed) ? trimmed : "all"; + const manifestIds = [...new Set(manifestChannels.map((entry) => entry.id))].toSorted(); + const validChannels = ["all", ...new Set([...CHAT_CHANNEL_ORDER, ...manifestIds])]; + throw new Error( + `Unknown channel ${JSON.stringify(raw)}. Valid channels: ${validChannels.join(", ")}`, + ); } -function matchesChannel(line: NonNullable, channel: string) { +function matchesChannelContext(value: string | undefined, channel: string) { + const path = `gateway/channels/${channel}`; + return value === channel || value === path || value?.startsWith(`${path}/`) === true; +} + +function matchesChannel( + line: Pick, + filter: ChannelLogFilter, +) { + const { channel } = filter; if (channel === "all") { return true; } - const needle = `gateway/channels/${channel}`; - if (line.subsystem?.includes(needle)) { - return true; - } - if (line.module?.includes(channel)) { - return true; - } - return false; + return ( + matchesChannelContext(line.subsystem, channel) || + matchesChannelContext(line.module, channel) || + (line.plugin !== undefined && filter.pluginIds.has(line.plugin)) + ); } function parseLinesOption(value: unknown): number { @@ -68,66 +90,28 @@ function parseLinesOption(value: unknown): number { return parsed; } -async function readTailLines(file: string, limit: number): Promise { - const stat = await fs.stat(file).catch(() => null); - if (!stat) { - return []; - } - const size = stat.size; - const start = Math.max(0, size - MAX_BYTES); - const handle = await fs.open(file, "r"); - try { - let prefix = ""; - if (start > 0) { - const prefixBuf = Buffer.alloc(1); - const prefixRead = await handle.read(prefixBuf, 0, 1, start - 1); - prefix = prefixBuf.toString("utf8", 0, prefixRead.bytesRead); - } - const length = Math.max(0, size - start); - if (length === 0) { - return []; - } - const buffer = Buffer.alloc(length); - const bytesRead = await readFileWindowFully(handle, buffer, start); - const text = buffer.toString("utf8", 0, bytesRead); - let lines = text.split("\n"); - if (start > 0 && prefix !== "\n") { - lines = lines.slice(1); - } - if (lines.length && lines[lines.length - 1] === "") { - lines = lines.slice(0, -1); - } - if (lines.length > limit) { - lines = lines.slice(lines.length - limit); - } - return lines; - } finally { - await handle.close(); - } -} - /** Print or serialize recent log lines matching one channel subsystem/module. */ export async function channelsLogsCommand( opts: ChannelsLogsOptions, runtime: RuntimeEnv = defaultRuntime, ) { - const channel = parseChannelFilter(opts.channel); + const filter = parseChannelFilter(opts.channel); + const { channel } = filter; const limit = parseLinesOption(opts.lines); - const file = await resolveLogFile(getResolvedLoggerSettings().file); - const rawLines = await readTailLines(file, limit * 4); - const parsed = rawLines - .map(parseLogLine) - .filter((line): line is NonNullable => Boolean(line)); - const filtered = parsed.filter((line) => matchesChannel(line, channel)); - const lines = filtered.slice(Math.max(0, filtered.length - limit)); + const tail = await readConfiguredParsedLogTail({ + limit, + maxBytes: MAX_BYTES, + filter: (line) => matchesChannel(line, filter), + }); + const lines = tail.lines; if (opts.json) { - writeRuntimeJson(runtime, { file, channel, lines }); + writeRuntimeJson(runtime, { file: tail.file, channel, lines }); return; } - runtime.log(theme.info(`Log file: ${file}`)); + runtime.log(theme.info(`Log file: ${tail.file}`)); if (channel !== "all") { runtime.log(theme.info(`Channel: ${channel}`)); } diff --git a/src/logging/log-tail.test.ts b/src/logging/log-tail.test.ts index 4c88e64aadce..50ed3137b0e7 100644 --- a/src/logging/log-tail.test.ts +++ b/src/logging/log-tail.test.ts @@ -105,19 +105,40 @@ describe("readConfiguredLogTail", () => { }); }); + it("keeps the first line when the byte window starts exactly after a newline", async () => { + const { readConfiguredLogTail } = await import("./log-tail.js"); + const tempDir = tempDirs.make("openclaw-log-tail-"); + const file = path.join(tempDir, "openclaw-2026-01-22.log"); + const line = (message: string) => `${message}${" ".repeat(199 - message.length)}\n`; + const content = Array.from({ length: 10_000 }, (_, index) => + line(index === 5000 ? "first-line-in-window" : "filler"), + ).join(""); + + await fs.writeFile(file, content); + setLoggerOverride({ file }); + + const result = await readConfiguredLogTail({ limit: 5000, maxBytes: 1_000_000 }); + + expect(result.lines).toHaveLength(5000); + expect(result.lines[0]?.trimEnd()).toBe("first-line-in-window"); + }); + it("falls back only within the active profile's rolling log family", async () => { const tempDir = tempDirs.make("openclaw-log-tail-"); - const missing = path.join(tempDir, "openclaw-dev-2026-01-22.log"); - const devLog = path.join(tempDir, "openclaw-dev-2026-01-21.log"); + const missing = path.join(tempDir, "openclaw-2026-01-22.log"); const defaultLog = path.join(tempDir, "openclaw-2026-01-21.log"); - await fs.writeFile(devLog, "dev profile\n"); + const devLog = path.join(tempDir, "openclaw-dev-2026-01-21.log"); await fs.writeFile(defaultLog, "default profile\n"); - await fs.utimes(devLog, new Date(0), new Date(0)); - await fs.utimes(defaultLog, new Date(), new Date()); - const { resolveLogFile } = await import("./log-tail.js"); - const result = await resolveLogFile(missing, { rolling: true }); + await fs.writeFile(devLog, "dev profile\n"); + await fs.utimes(defaultLog, new Date(0), new Date(0)); + await fs.utimes(devLog, new Date(), new Date()); + setLoggerOverride({ file: missing }); - expect(result).toBe(devLog); + const { readConfiguredLogTail } = await import("./log-tail.js"); + const result = await readConfiguredLogTail(); + + expect(result.file).toBe(defaultLog); + expect(result.lines).toEqual(["default profile"]); }); it("does not reinterpret an explicit profile-shaped logging.file as rolling", async () => { diff --git a/src/logging/log-tail.ts b/src/logging/log-tail.ts index 73c574487b7a..2ba8ec025923 100644 --- a/src/logging/log-tail.ts +++ b/src/logging/log-tail.ts @@ -6,6 +6,7 @@ import { clamp } from "../utils.js"; import { isRollingLogFilePath, isSameRollingLogFileFamily } from "./log-file-path.js"; import "./logger.js"; import { getResolvedLoggerFileTarget } from "./logger-settings-internal.js"; +import { parseLogLine, type ParsedLogLine } from "./parse-log-line.js"; import { redactSensitiveLines, resolveRedactOptions } from "./redact.js"; // Tail reader for the active log file, with cursor reset and line redaction. @@ -24,11 +25,13 @@ export type LogTailPayload = { reset: boolean; }; +/** Redacted configured log tail with only parseable structured records. */ +type ParsedLogTailPayload = Omit & { + lines: ParsedLogLine[]; +}; + /** Resolves a rolling daily log path to the newest existing rolling log when needed. */ -export async function resolveLogFile( - file: string, - options?: { rolling?: boolean }, -): Promise { +async function resolveLogFile(file: string, options?: { rolling?: boolean }): Promise { const stat = await fs.stat(file).catch(() => null); if (stat) { return file; @@ -63,6 +66,7 @@ async function readLogSlice(params: { cursor?: number; limit: number; maxBytes: number; + filter?: (line: string) => boolean; }): Promise> { const stat = await fs.stat(params.file).catch(() => null); if (!stat) { @@ -137,6 +141,10 @@ async function readLogSlice(params: { if (lines.length > 0 && lines[lines.length - 1] === "") { lines = lines.slice(0, -1); } + if (params.filter) { + // Sparse consumers inspect the full byte-bounded window before the shared line cap. + lines = lines.filter(params.filter); + } if (lines.length > limit) { truncated = true; lines = lines.slice(lines.length - limit); @@ -157,11 +165,10 @@ async function readLogSlice(params: { } /** Reads and redacts the configured log tail with bounded bytes and line count. */ -export async function readConfiguredLogTail(params?: { - cursor?: number; - limit?: number; - maxBytes?: number; -}): Promise { +export async function readConfiguredLogTail( + params?: { cursor?: number; limit?: number; maxBytes?: number }, + filter?: (line: string) => boolean, +): Promise { const target = getResolvedLoggerFileTarget(); const file = await resolveLogFile(target.file, { rolling: target.rolling }); const result = await readLogSlice({ @@ -169,6 +176,7 @@ export async function readConfiguredLogTail(params?: { cursor: params?.cursor, limit: params?.limit ?? DEFAULT_LIMIT, maxBytes: params?.maxBytes ?? DEFAULT_MAX_BYTES, + filter, }); const redaction = resolveRedactOptions(); return { @@ -177,3 +185,23 @@ export async function readConfiguredLogTail(params?: { lines: redactSensitiveLines(result.lines, redaction), }; } + +/** Reads the canonical configured tail and parses its already-redacted lines. */ +export async function readConfiguredParsedLogTail(params?: { + cursor?: number; + limit?: number; + maxBytes?: number; + filter?: (line: Pick) => boolean; +}): Promise { + const tail = await readConfiguredLogTail(params, (raw) => { + const parsed = parseLogLine(raw); + return parsed !== null && (params?.filter?.(parsed) ?? true); + }); + return { + ...tail, + lines: tail.lines.flatMap((line) => { + const parsed = parseLogLine(line); + return parsed ? [parsed] : []; + }), + }; +} diff --git a/src/logging/parse-log-line.test.ts b/src/logging/parse-log-line.test.ts index 1462198aa559..6f6a5b08e536 100644 --- a/src/logging/parse-log-line.test.ts +++ b/src/logging/parse-log-line.test.ts @@ -56,6 +56,19 @@ describe("parseLogLine", () => { expect(parseLogLine(JSON.stringify({ 0: "worker", 1: "ready" }))?.subsystem).toBeUndefined(); }); + it("retains the exact plugin identity from logger binding metadata", () => { + const parsed = parseLogLine( + JSON.stringify({ + 0: '{"subsystem":"gateway/channels/external-chat"}', + 1: "sent", + _meta: { name: '{"plugin":"vendor-external-chat","feature":"delivery"}' }, + }), + ); + + expect(parsed?.plugin).toBe("vendor-external-chat"); + expect(parsed?.subsystem).toBe("gateway/channels/external-chat"); + }); + it("falls back to meta timestamp when top-level time is missing", () => { const line = JSON.stringify({ 0: "hello", diff --git a/src/logging/parse-log-line.ts b/src/logging/parse-log-line.ts index 67deef464b0e..41ed2b470898 100644 --- a/src/logging/parse-log-line.ts +++ b/src/logging/parse-log-line.ts @@ -3,14 +3,16 @@ import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/string-coerce"; // Parser for JSON LogTape lines emitted by the OpenClaw logger. -type ParsedLogLine = { +export type ParsedLogLine = { time?: string; level?: string; subsystem?: string; module?: string; + plugin?: string; message: string; raw: string; }; +type LogContext = Pick; function extractMessage(value: Record): string { const parts: string[] = []; @@ -28,7 +30,7 @@ function extractMessage(value: Record): string { return parts.join(" "); } -function parseMetaName(raw?: unknown): { subsystem?: string; module?: string } { +function parseMetaName(raw?: unknown): LogContext { if (typeof raw !== "string") { return {}; } @@ -37,6 +39,7 @@ function parseMetaName(raw?: unknown): { subsystem?: string; module?: string } { return { subsystem: typeof parsed.subsystem === "string" ? parsed.subsystem : undefined, module: typeof parsed.module === "string" ? parsed.module : undefined, + plugin: typeof parsed.plugin === "string" ? parsed.plugin : undefined, }; } catch { return {}; @@ -46,12 +49,14 @@ function parseMetaName(raw?: unknown): { subsystem?: string; module?: string } { function resolveContext( value: Record, meta: Record | undefined, -): { subsystem?: string; module?: string } { +): LogContext { const metadataContext = parseMetaName(meta?.name); - if (metadataContext.subsystem || metadataContext.module) { - return metadataContext; - } - return parseMetaName(value["0"]); + const positionalContext = parseMetaName(value["0"]); + return { + subsystem: metadataContext.subsystem ?? positionalContext.subsystem, + module: metadataContext.module ?? positionalContext.module, + plugin: metadataContext.plugin ?? positionalContext.plugin, + }; } /** Parses a raw log line into compact metadata and message text, or null for non-JSON lines. */ @@ -74,6 +79,7 @@ export function parseLogLine(raw: string): ParsedLogLine | null { level: normalizeOptionalLowercaseString(levelRaw), subsystem: context.subsystem, module: context.module, + plugin: context.plugin, message: typeof parsed.message === "string" ? parsed.message : extractMessage(parsed), raw, }; diff --git a/src/logging/redact.test.ts b/src/logging/redact.test.ts index a55249b72610..73c0bd3d52c8 100644 --- a/src/logging/redact.test.ts +++ b/src/logging/redact.test.ts @@ -1975,9 +1975,11 @@ describe("redactSensitiveLines", () => { expect(result[1]).toBe("normal log line"); }); - it("returns lines unmodified when mode is off", () => { + it("returns lines unmodified when redaction is off", () => { const resolved = resolveRedactOptions({ mode: "off", patterns: defaults }); - const lines = ["TOKEN=abcdef1234567890ghij"]; + const secret = "opaque-registry-value-1234567890"; + registerSecretValueForRedaction(secret); + const lines = [`TOKEN=abcdef1234567890ghij ${secret}`]; expect(redactSensitiveLines(lines, resolved)).toEqual(lines); }); @@ -2008,12 +2010,15 @@ describe("redactSensitiveLines", () => { ).toEqual(["Authorization: Digest", " ***; status=401"]); }); - it("returns lines unmodified when resolved patterns is empty — does not fall back to defaults", () => { + it("applies exact registered secrets without falling back from empty resolved patterns", () => { // Simulates the case where all user-configured patterns fail to compile. // The pre-resolved empty array must be honored, not silently replaced with defaults. const resolved = { mode: "tools" as const, patterns: [], redactFormBodies: false }; - const lines = ["TOKEN=abcdef1234567890ghij"]; - expect(redactSensitiveLines(lines, resolved)).toEqual(lines); + const secret = "opaque-registry-value-1234567890"; + registerSecretValueForRedaction(secret); + expect(redactSensitiveLines([`TOKEN=abcdef1234567890ghij ${secret}`], resolved)).toEqual([ + "TOKEN=abcdef1234567890ghij opaque…7890", + ]); }); it("returns empty array unchanged — does not produce a synthetic blank line", () => { diff --git a/src/logging/redact.ts b/src/logging/redact.ts index e87dd68b942d..26f549804ef7 100644 --- a/src/logging/redact.ts +++ b/src/logging/redact.ts @@ -1108,12 +1108,16 @@ export function getDefaultRedactPatterns(): string[] { // line boundaries, then split back. Use this instead of mapping redactSensitiveText when // options are resolved once per request. export function redactSensitiveLines(lines: string[], resolved: ResolvedRedactOptions): string[] { - if (resolved.mode === "off" || !resolved.patterns.length || lines.length === 0) { + if (lines.length === 0 || resolved.mode === "off") { return lines; } + const exactRedactedLines = lines.map((line) => redactRegisteredSecretValues(line, maskToken)); + if (!resolved.patterns.length) { + return exactRedactedLines; + } const redactedLines = resolved.redactFormBodies - ? lines.map((line) => redactFormBody(redactUrlQueryPairs(line))) - : lines; + ? exactRedactedLines.map((line) => redactFormBody(redactUrlQueryPairs(line))) + : exactRedactedLines; let redacted = redactedLines.join("\n"); if (resolved.redactStructuredAuthHeaders) { redacted = redactStructuredAuthHeaders(redacted, "***");