merge: sync origin/main

Punchcard-Session: frost-brook-timber-mx

* origin/main:
  fix: cap local tsgo CPU usage (#119517)
  fix(tool-search): normalize dotted call arguments (#119418) (#119464)
  fix(qa): retain child output on CLI timeouts (#119524)
  fix(discord): preserve authored block order in presentation components (#119506)
  fix: make plugin verification restart guidance install-neutral (#119508)
  fix(qa): preserve profile execution contracts (#119519)
  test(qa): cover managed OTEL runtime paths (#118965)

Punchcard-Session: frost-brook-timber-mx
This commit is contained in:
Vincent Koc
2026-08-05 15:33:57 +08:00
32 changed files with 2016 additions and 806 deletions
@@ -223,6 +223,30 @@ describe("buildDiscordInteractiveComponents", () => {
expect(buildDiscordPresentationComponents(presentation)).toBeUndefined();
});
it("preserves authored block order around controls", () => {
expect(
buildDiscordPresentationComponents({
blocks: [
{ type: "text", text: "First" },
{
type: "buttons",
buttons: [{ label: "Approve", value: "approve", style: "success" }],
},
{ type: "text", text: "Last" },
],
}),
).toEqual({
blocks: [
{ type: "text", text: "First" },
{
type: "actions",
buttons: [{ label: "Approve", style: "success", callbackData: "approve" }],
},
{ type: "text", text: "Last" },
],
});
});
it("renders typed approvals as actionable transport-private Discord controls", () => {
const rendered = buildDiscordPresentationComponents({
blocks: [
@@ -229,8 +229,6 @@ export function buildDiscordPresentationComponents(
blocks.push({ type: "separator" });
continue;
}
}
for (const block of presentation.blocks) {
if (block.type === "buttons") {
appendDiscordButtonBlocks(blocks, block.buttons);
continue;
+58
View File
@@ -787,6 +787,64 @@ describe("qa cli runtime", () => {
expectWriteContains(stdoutWrite, "QA run profile: all; categories: 1; scenarios:");
});
it.each([
{
label: "implicit profile membership",
scenarioIds: undefined,
expectedExitCode: undefined,
explicitScenarioSelection: false,
},
{
label: "explicit profile selection",
scenarioIds: ["runtime-tool-image-generate"],
expectedExitCode: 1,
explicitScenarioSelection: true,
},
])(
"keeps optional skips $label blocking semantics",
async ({ scenarioIds, expectedExitCode, explicitScenarioSelection }) => {
const priorExitCode = process.exitCode;
process.exitCode = undefined;
const optionalScenario = {
name: "Runtime tool fixture — image_generate",
status: "skip" as const,
details: "image_generate mock provider report-only: tool unavailable",
};
await fs.writeFile(
suiteSummaryPath,
JSON.stringify({
counts: { total: 2, passed: 1, failed: 0, skipped: 1 },
scenarios: [QA_PASSING_SUITE_SCENARIO, optionalScenario],
}),
"utf8",
);
runQaSuite.mockResolvedValueOnce(
flowSuiteRuntimeResult({
reportPath: suiteReportPath,
summaryPath: suiteSummaryPath,
scenarios: [QA_PASSING_SUITE_SCENARIO, optionalScenario],
}),
);
try {
await runQaProfileCommand({
repoRoot: "/tmp/openclaw-repo",
profile: "all",
surface: "media",
category: "media.media-generation",
providerMode: "mock-openai",
scenarioIds,
});
expect(process.exitCode).toBe(expectedExitCode);
expect(mockFirstObjectArg(runQaSuite).adapterOptions).toMatchObject({
explicitScenarioSelection,
});
} finally {
process.exitCode = priorExitCode;
}
},
);
it("filters QA-channel-pinned scenarios from an implicit Crabline smoke profile", async () => {
runQaSuite.mockImplementationOnce(async () => {
await fs.writeFile(suiteEvidencePath, JSON.stringify(makeQaEvidence()), "utf8");
+4 -1
View File
@@ -713,6 +713,7 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) {
fastMode: opts.fastMode,
failFast: opts.failFast,
scenarioIds: scenarios.map((scenario) => scenario.id),
explicitScenarioSelection: requestedScenarioIds.length > 0,
concurrency: opts.concurrency,
allowFailures: opts.allowFailures,
channelDriver: profileReport.channelDriver,
@@ -815,7 +816,9 @@ function resolveQaReportOnlyOptionalScenarioNames(params: {
scenarioIds: readonly string[];
explicitScenarioSelection?: boolean;
}): ReadonlySet<string> | undefined {
if (params.explicitScenarioSelection || params.scenarioIds.length > 0) {
const explicitScenarioSelection =
params.explicitScenarioSelection ?? params.scenarioIds.length > 0;
if (explicitScenarioSelection) {
return undefined;
}
return resolveQaReportOnlyOptionalScenarioNamesFromCatalog(readQaScenarioPack().scenarios);
@@ -64,12 +64,25 @@ const QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT = {
const QA_COMPACTION_RETRY_PROMPT =
"Compaction retry mutating tool check. Current durable context marker: QA-COMPACTION-DURABLE-MARKER. Create compaction-retry-summary.txt.";
const QA_COMPACTION_RETRY_OVERFLOW_PADDING = "x".repeat(300_000);
const QA_COMPACTION_RETRY_HISTORICAL_PHRASE = "post-marker historical user block";
const QA_COMPACTION_EMPTY_RECOVERY_SUMMARY_MARKER = "QA-COMPACTION-EMPTY-RECOVERED-SUMMARY";
const QA_COMPACTION_REASONING_RECOVERY_SUMMARY_MARKER = "QA-COMPACTION-REASONING-RECOVERED-SUMMARY";
const QA_COMPACTION_SUMMARY_HEADINGS = [
"## Decisions",
"## Open TODOs",
"## Constraints/Rules",
"## Pending user asks",
"## Exact identifiers",
] as const;
const QA_COMPACTION_SUMMARY_INSTRUCTIONS = `You are a context summarization assistant. Your task is to read a conversation between a user and an AI assistant, then produce a structured summary following the exact format specified.
Do NOT continue the conversation. Do NOT respond to any questions in the conversation. ONLY output the structured summary.`;
function expectCurrentCompactionSummaryHeadings(summary: string) {
expect(summary.match(/^## .+$/gmu)).toEqual(QA_COMPACTION_SUMMARY_HEADINGS);
expect(summary).not.toContain("## Goal");
}
afterEach(async () => {
while (cleanups.length > 0) {
await cleanups.pop()?.();
@@ -2395,7 +2408,7 @@ describe("qa mock openai server", () => {
expect(response.status).toBe(200);
const summary = outputText(await response.json());
expect(summary).toContain("## Goal");
expectCurrentCompactionSummaryHeadings(summary);
expect(summary).not.toContain("QA-COMPACTION-DURABLE-MARKER");
expect(summary).not.toContain("QA-COMPACTION-BULKY-HISTORICAL-MARKER");
const requests = requireArray(
@@ -2570,9 +2583,16 @@ describe("qa mock openai server", () => {
expect(response.status).toBe(200);
const body = requireRecord(await response.json(), "Anthropic summary response");
expect(requireArray(body.content, "content")).toContainEqual(
expect.objectContaining({ type: "text", text: expect.stringContaining("## Goal") }),
const content = requireArray(body.content, "content");
expect(content).toContainEqual(
expect.objectContaining({ type: "text", text: expect.stringContaining("## Decisions") }),
);
const summaryText = requireRecord(content[0], "summary content").text;
expect(typeof summaryText).toBe("string");
if (typeof summaryText !== "string") {
throw new TypeError("Anthropic summary content text must be a string");
}
expectCurrentCompactionSummaryHeadings(summaryText);
expect(await getJson(server, "/debug/last-request")).toMatchObject({
requestKind: "compaction-summary",
outcome: "success",
@@ -2588,20 +2608,43 @@ describe("qa mock openai server", () => {
"<conversation>\n[Chunk 1 - oldest messages]\nunrelated historical context\n</conversation>\n\nAdditional focus: preserve exact identifiers.",
});
const genericChunkSummary = outputText(genericChunkPayload);
expect(genericChunkSummary).toContain("## Goal");
expectCurrentCompactionSummaryHeadings(genericChunkSummary);
expect(genericChunkSummary).not.toContain("QA-COMPACTION-DURABLE-MARKER");
const scenarioChunkPayload = await expectOpenAiNonStreamingResponsesJson(server, {
model: "gpt-5.6-luna",
instructions: QA_COMPACTION_SUMMARY_INSTRUCTIONS,
input:
"<conversation>\n[Chunk 2 - most recent messages]\nQA-COMPACTION-BULKY-HISTORICAL-MARKER\n</conversation>\n\nAdditional focus: preserve exact identifiers.",
input: `<conversation>
[Chunk 2 - most recent messages]
QA-COMPACTION-BULKY-HISTORICAL-MARKER ${QA_COMPACTION_RETRY_HISTORICAL_PHRASE} 10
</conversation>
Additional focus: preserve exact identifiers.`,
});
const scenarioChunkSummary = outputText(scenarioChunkPayload);
expect(scenarioChunkSummary).toContain("## Goal");
expectCurrentCompactionSummaryHeadings(scenarioChunkSummary);
expect(scenarioChunkSummary).toContain(QA_COMPACTION_RETRY_HISTORICAL_PHRASE);
expect(scenarioChunkSummary).not.toContain("QA-COMPACTION-DURABLE-MARKER");
expect(scenarioChunkSummary).not.toContain("QA-COMPACTION-BULKY-HISTORICAL-MARKER");
const historyMergePayload = await expectOpenAiNonStreamingResponsesJson(server, {
model: "gpt-5.6-luna",
instructions: QA_COMPACTION_SUMMARY_INSTRUCTIONS,
input: `<conversation>
[Chunk 1 - oldest messages]
${genericChunkSummary}
[Chunk 2 - most recent messages]
${scenarioChunkSummary}
</conversation>
Update and merge these partial structured summaries.`,
});
const historyMergedSummary = outputText(historyMergePayload);
expectCurrentCompactionSummaryHeadings(historyMergedSummary);
expect(historyMergedSummary).toContain(QA_COMPACTION_RETRY_HISTORICAL_PHRASE);
expect(historyMergedSummary).not.toContain("QA-COMPACTION-DURABLE-MARKER");
expect(historyMergedSummary).not.toContain("QA-COMPACTION-BULKY-HISTORICAL-MARKER");
const durableChunkPayload = await expectOpenAiNonStreamingResponsesJson(server, {
model: "gpt-5.6-luna",
instructions: QA_COMPACTION_SUMMARY_INSTRUCTIONS,
@@ -2612,6 +2655,7 @@ Retain QA-COMPACTION-DURABLE-MARKER for the active task.
Additional focus: preserve QA-COMPACTION-DURABLE-MARKER.`,
});
const durableChunkSummary = outputText(durableChunkPayload);
expectCurrentCompactionSummaryHeadings(durableChunkSummary);
expect(durableChunkSummary).toContain("QA-COMPACTION-DURABLE-MARKER");
const promptOnlyChunkPayload = await expectOpenAiNonStreamingResponsesJson(server, {
@@ -2624,6 +2668,7 @@ Compaction retry mutating tool check. Create compaction-retry-summary.txt.
Additional focus: preserve current work.`,
});
const promptOnlyChunkSummary = outputText(promptOnlyChunkPayload);
expectCurrentCompactionSummaryHeadings(promptOnlyChunkSummary);
expect(promptOnlyChunkSummary).not.toContain("QA-COMPACTION-DURABLE-MARKER");
const currentChunkPayload = await expectOpenAiNonStreamingResponsesJson(server, {
@@ -2637,6 +2682,7 @@ Create compaction-retry-summary.txt.
Additional focus: preserve current work.`,
});
const currentChunkSummary = outputText(currentChunkPayload);
expectCurrentCompactionSummaryHeadings(currentChunkSummary);
expect(currentChunkSummary).toContain("QA-COMPACTION-DURABLE-MARKER");
const mergePayload = await expectOpenAiNonStreamingResponsesJson(server, {
@@ -2657,7 +2703,9 @@ ${durableChunkSummary}
Update and merge these partial structured summaries.`,
});
expect(outputText(mergePayload)).toContain("QA-COMPACTION-DURABLE-MARKER");
const mergedSummary = outputText(mergePayload);
expectCurrentCompactionSummaryHeadings(mergedSummary);
expect(mergedSummary).toContain("QA-COMPACTION-DURABLE-MARKER");
const unrelatedPayload = await expectOpenAiNonStreamingResponsesJson(server, {
model: "gpt-5.6-luna",
@@ -2665,14 +2713,15 @@ Update and merge these partial structured summaries.`,
input:
"<conversation>\na later unrelated scenario\n</conversation>\n\nCreate a structured summary.",
});
expect(outputText(unrelatedPayload)).toContain("## Goal");
expect(outputText(unrelatedPayload)).not.toContain("QA-COMPACTION-DURABLE-MARKER");
const unrelatedSummary = outputText(unrelatedPayload);
expectCurrentCompactionSummaryHeadings(unrelatedSummary);
expect(unrelatedSummary).not.toContain("QA-COMPACTION-DURABLE-MARKER");
const requests = requireArray(
await getJson(server, "/debug/requests"),
"compaction requests",
).map((request) => requireRecord(request, "compaction request"));
expect(requests).toHaveLength(7);
expect(requests).toHaveLength(8);
expect(
requests.every(
(request) =>
@@ -2705,6 +2754,7 @@ Update and merge these partial structured summaries.`,
],
});
const compactedSummary = outputText(summaryPayload);
expectCurrentCompactionSummaryHeadings(compactedSummary);
expect(compactedSummary).toContain("QA-COMPACTION-DURABLE-MARKER");
const writePlan = await expectOpenAiStreamingResponsesText(server, {
@@ -186,61 +186,59 @@ const QA_COMPACTION_RETRY_OVERFLOW_THRESHOLD_BYTES = 256 * 1024;
const QA_COMPACTION_OUTPUT_RECOVERY_OVERFLOW_THRESHOLD_BYTES = 96 * 1024;
const QA_COMPACTION_RETRY_DURABLE_MARKER = "QA-COMPACTION-DURABLE-MARKER";
const QA_COMPACTION_RETRY_BULKY_MARKER = "QA-COMPACTION-BULKY-HISTORICAL-MARKER";
const QA_COMPACTION_RETRY_HISTORICAL_PHRASE = "post-marker historical user block";
const QA_COMPACTION_EMPTY_OUTPUT_ONCE_MARKER_RE =
/\bQA-COMPACTION-EMPTY-OUTPUT-ONCE-[A-Za-z0-9_-]+\b/u;
const QA_COMPACTION_REASONING_ONLY_OUTPUT_ONCE_MARKER_RE =
/\bQA-COMPACTION-REASONING-ONLY-OUTPUT-ONCE-[A-Za-z0-9_-]+\b/u;
const QA_COMPACTION_EMPTY_RECOVERY_SUMMARY_MARKER = "QA-COMPACTION-EMPTY-RECOVERED-SUMMARY";
const QA_COMPACTION_REASONING_RECOVERY_SUMMARY_MARKER = "QA-COMPACTION-REASONING-RECOVERED-SUMMARY";
const QA_COMPACTION_RETRY_SUMMARY = `## Goal
Complete the compaction retry mutating tool check.
const QA_COMPACTION_RETRY_SUMMARY = `## Decisions
- Continue the compaction retry from durable context without replaying a completed mutation.
## Constraints & Preferences
## Open TODOs
- Write compaction-retry-summary.txt exactly once.
- Return the final replay-safety marker.
## Constraints/Rules
- Preserve ${QA_COMPACTION_RETRY_DURABLE_MARKER}.
- Write exactly: Replay safety: unsafe after write.
## Progress
### Done
- [x] Historical context compacted after overflow.
## Pending user asks
- Create compaction-retry-summary.txt, then reply exactly: Protocol note: replay unsafe after write.
### In Progress
- [ ] Write compaction-retry-summary.txt exactly once.
## Exact identifiers
- ${QA_COMPACTION_RETRY_DURABLE_MARKER}
- compaction-retry-summary.txt`;
const QA_COMPACTION_RETRY_HISTORICAL_SUMMARY = `## Decisions
- Preserve the latest ${QA_COMPACTION_RETRY_HISTORICAL_PHRASE} context through staged compaction.
### Blocked
- (none)
## Open TODOs
- Continue summarizing the ${QA_COMPACTION_RETRY_HISTORICAL_PHRASE} sequence.
## Key Decisions
- **Retry once**: Continue from compacted context without replaying a completed mutation.
## Constraints/Rules
- Keep historical content distinct from live task state.
- Do not invent durable context absent from the summarized history.
## Next Steps
1. Write the required file.
2. Return the final replay-safety marker.
## Pending user asks
- Retain the ${QA_COMPACTION_RETRY_HISTORICAL_PHRASE} details.
## Critical Context
- ${QA_COMPACTION_RETRY_DURABLE_MARKER}`;
const QA_GENERIC_COMPACTION_SUMMARY = `## Goal
Preserve the active conversation context.
## Exact identifiers
- None captured.`;
const QA_GENERIC_COMPACTION_SUMMARY = `## Decisions
- Continue from the summary without restarting completed work.
## Constraints & Preferences
## Open TODOs
- Continue the active task.
## Constraints/Rules
- Keep current requirements and identifiers.
## Progress
### Done
- [x] Historical context summarized.
## Pending user asks
- Continue the active task from the retained context.
### In Progress
- [ ] Continue the active task.
### Blocked
- (none)
## Key Decisions
- **Continue from summary**: Do not restart completed work.
## Next Steps
1. Continue the active task from the retained context.
## Critical Context
- Refer to the retained recent turns for current task details.`;
## Exact identifiers
- None captured.`;
const QA_COMPACTION_OUTPUT_RECOVERY_SUMMARY = `## Decisions
- Retry the typed compaction-summary fault at the compaction owner.
@@ -834,7 +832,10 @@ async function buildResponsesPayload(
return buildAssistantEvents(
hasCompactionRetryDurableContext
? QA_COMPACTION_RETRY_SUMMARY
: resolveCompactionRecoverySummary(allInputText),
: allInputText.includes(QA_COMPACTION_RETRY_BULKY_MARKER) ||
allInputText.includes(QA_COMPACTION_RETRY_HISTORICAL_PHRASE)
? QA_COMPACTION_RETRY_HISTORICAL_SUMMARY
: resolveCompactionRecoverySummary(allInputText),
);
}
if (
@@ -71,6 +71,7 @@ describe("qa compaction scenario catalog", () => {
const postWriteContinuationsExpr = readSetExpression("postWriteContinuations");
const writeTranscriptToolCallIdExpr = readSetExpression("writeTranscriptToolCallId");
const continuationChainExpr = readSetExpression("continuationChain");
const compactionSummaryRequestsExpr = readSetExpression("compactionSummaryRequests");
const continuationAssertIndex = actionIndex((action) =>
readFlowAssertExpression(action).includes("continuationChain.valid === true"),
);
@@ -97,6 +98,10 @@ describe("qa compaction scenario catalog", () => {
const terminalEvidenceAssertExpr = readAssertExpression(
"terminalContinuations[0].providerVariant === 'openai'",
);
const compactionSummaryAssertExpr = readAssertExpression(
"compactionSummaryRequests.length > 0",
);
const noQualityRetryAssertExpr = readAssertExpression("Previous summary failed quality checks");
const knownGap =
"known-harness-gap compaction-retry-mutating-tool: provider-error recovery does not invoke Codex native compaction; native token-threshold compaction needs a separate scenario.";
@@ -274,11 +279,19 @@ describe("qa compaction scenario catalog", () => {
expect(flow).not.toContain("config.expectedOpenClawToolResult");
expect(flow).not.toContain("String(request.toolOutput ?? '').includes(`---");
expect(flow).not.toContain("String(request.toolOutput ?? '').includes(`+++");
expect(flow).toContain(
"compactionSummaryRequests.length === 1 && compactionSummaryRequests[0].outcome === 'success' && compactionSummaryRequests[0].plannedToolName === undefined && compactionSummaryRequests[0].toolOutputStructuredError !== true",
expect(compactionSummaryRequestsExpr).toContain("request.requestKind === 'compaction-summary'");
expect(compactionSummaryAssertExpr).toContain("compactionSummaryRequests.length > 0");
expect(compactionSummaryAssertExpr).toContain(
"request.cursor > overflowRequest.cursor && request.cursor < writeRequest.cursor",
);
expect(flow).not.toContain("compactionSummaryRequests.every(");
expect(flow).not.toContain("compactionSummaryRequests.length >= 1");
expect(compactionSummaryAssertExpr).toContain("request.outcome === 'success'");
expect(compactionSummaryAssertExpr).toContain("request.plannedToolName === undefined");
expect(compactionSummaryAssertExpr).toContain("request.toolOutputStructuredError !== true");
expect(noQualityRetryAssertExpr).toContain("compactionSummaryRequests.every");
expect(noQualityRetryAssertExpr).toContain(
"!String(request.allInputText ?? '').includes('Previous summary failed quality checks')",
);
expect(flow).not.toContain("compactionSummaryRequests.length === 1");
expect(flow).toContain(
"writeRequest.rawByteLength < config.overflowThresholdBytes && writeRequest.rawByteLength < overflowRequest.rawByteLength",
);
@@ -292,6 +305,7 @@ describe("qa compaction scenario catalog", () => {
expect(flow).toContain('"set":"requestEvidence"');
expect(flow).toContain("durable: String(request.allInputText ?? '')");
expect(flow).toContain("bulky: String(request.allInputText ?? '')");
expect(flow).toContain("qualityRetry: String(request.allInputText ?? '')");
expect(flow).toContain("inputChars: String(request.allInputText ?? '').length");
expect(flow).toContain(
"resolvedWireTool: request.plannedWireToolName ?? request.plannedToolName ?? null",
@@ -792,6 +792,24 @@ describe("qa scenario catalog", () => {
expect(heartbeatFlow).not.toContain("waitForNoOutbound");
});
it.each([
"inbound-media-store-audio-transcription",
"active-memory-cold-first-turn-trigger-recall",
"compaction-empty-response-recovery",
"compaction-reasoning-only-recovery",
"compaction-retry-mutating-tool",
"empty-response-recovery-replay-safe-read",
"empty-response-retry-budget-exhausted",
"reasoning-only-no-auto-retry-after-write",
"reasoning-only-recovery-replay-safe-read",
])("keeps strict mock-only scenario %s on the mock-openai lane", (scenarioId) => {
const config = readQaScenarioExecutionConfig(scenarioId) as
| { requiredProviderMode?: string }
| undefined;
expect(config?.requiredProviderMode).toBe("mock-openai");
});
it("includes the thinking slash model remap scenario", () => {
const scenario = readQaScenarioById("thinking-slash-model-remap");
const config = readQaScenarioExecutionConfig("thinking-slash-model-remap") as
@@ -72,4 +72,46 @@ describe("qa suite runtime CLI integration", () => {
status: "ok",
});
});
it("retains real child output when the qa cli times out", async () => {
const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-cli-timeout-repo-"));
const tempRoot = await mkdtemp(path.join(os.tmpdir(), "qa-cli-timeout-runtime-"));
cleanups.push(async () => {
await rm(repoRoot, { recursive: true, force: true });
await rm(tempRoot, { recursive: true, force: true });
});
const distDir = path.join(repoRoot, "dist");
await mkdir(distDir, { recursive: true });
await writeFile(
path.join(distDir, "index.js"),
[
'process.stdout.write("timeout stdout marker\\n");',
'process.stderr.write("timeout stderr marker\\n");',
"setInterval(() => {}, 60_000);",
"",
].join("\n"),
"utf8",
);
const error = await runQaCli(
{
repoRoot,
gateway: {
tempRoot,
runtimeEnv: process.env,
},
primaryModel: "openai/gpt-5.6-luna",
alternateModel: "openai/gpt-5.6-luna",
providerMode: "mock-openai",
} as never,
["qa", "suite"],
{ timeoutMs: 1_000 },
).catch((value: unknown) => value);
expect(error).toMatchObject({ code: "qa_cli_timeout" });
const message = error instanceof Error ? error.message : String(error);
expect(message).toContain("qa cli timed out: openclaw qa suite");
expect(message).toContain("stdout:\ntimeout stdout marker");
expect(message).toContain("stderr:\ntimeout stderr marker");
});
});
@@ -174,6 +174,7 @@ describe("qa suite runtime agent process helpers", () => {
it.runIf(process.platform !== "win32")("kills timed-out qa cli process groups", async () => {
const killSpy = vi.spyOn(process, "kill").mockImplementation(() => true);
vi.useFakeTimers();
try {
const child = createSpawnedProcess({ pid: 12345 });
const { pending } = startMockQaCli({
@@ -181,15 +182,37 @@ describe("qa suite runtime agent process helpers", () => {
child,
options: { timeoutMs: 1 },
});
const timeoutAssertion = expect(pending).rejects.toThrow(
"qa cli timed out: openclaw qa suite",
const errorPromise = pending.catch((value: unknown) => value);
await Promise.resolve();
expect(spawnMock).toHaveBeenCalledTimes(1);
child.stdout.emit(
"data",
Buffer.from(
`stdout-head-marker\n${"x".repeat(QA_CHILD_STDOUT_MAX_BYTES)}\nstdout-tail-marker`,
),
);
child.stderr.emit(
"data",
Buffer.from(
`stderr-head-marker\n${"x".repeat(QA_CHILD_STDERR_TAIL_BYTES)}\nstderr-tail-marker`,
),
);
await vi.advanceTimersByTimeAsync(1);
await waitForSpawnCount(1);
await timeoutAssertion;
const error = await errorPromise;
expect(error).toMatchObject({ code: "qa_cli_timeout" });
const message = error instanceof Error ? error.message : String(error);
expect(message).toContain("qa cli timed out: openclaw qa suite");
expect(message).toContain("stdout:\n[qa cli stdout truncated to last");
expect(message).toContain("stdout-tail-marker");
expect(message).not.toContain("stdout-head-marker");
expect(message).toContain("stderr:\n[qa cli stderr truncated to last");
expect(message).toContain("stderr-tail-marker");
expect(message).not.toContain("stderr-head-marker");
expect(killSpy).toHaveBeenCalledWith(-12345, "SIGKILL");
expect(child.kill).not.toHaveBeenCalled();
} finally {
vi.useRealTimers();
killSpy.mockRestore();
}
});
@@ -265,6 +265,7 @@ async function runQaCli(
opts?: { timeoutMs?: number; json?: boolean; env?: NodeJS.ProcessEnv },
) {
const stdout = createQaChildOutputCapture();
const stdoutTail = createQaChildOutputTail();
const stderr = createQaChildOutputTail();
const distEntryPath = path.join(env.repoRoot, "dist", "index.js");
const nodeExecPath = await resolveQaNodeExecPath();
@@ -281,11 +282,25 @@ async function runQaCli(
const timeoutMs = resolveTimerTimeoutMs(opts?.timeoutMs, 60_000);
const timeout = setTimeout(() => {
signalQaCliProcessTree(child, "SIGKILL");
const stdoutText = formatQaChildOutputTail(stdoutTail, "qa cli stdout");
const stderrText = formatQaChildOutputTail(stderr, "qa cli stderr");
const diagnostics = [
stdoutText ? `stdout:\n${stdoutText}` : "",
stderrText ? `stderr:\n${stderrText}` : "",
]
.filter(Boolean)
.join("\n");
reject(
new QaSuiteInfraError("qa_cli_timeout", `qa cli timed out: openclaw ${args.join(" ")}`),
new QaSuiteInfraError(
"qa_cli_timeout",
`qa cli timed out: openclaw ${args.join(" ")}${diagnostics ? `\n${diagnostics}` : ""}`,
),
);
}, timeoutMs);
child.stdout.on("data", (chunk) => appendQaChildOutput(stdout, chunk));
child.stdout.on("data", (chunk) => {
appendQaChildOutput(stdout, chunk);
appendQaChildOutputTail(stdoutTail, chunk);
});
child.stderr.on("data", (chunk) => appendQaChildOutputTail(stderr, chunk));
child.once("error", (error) => {
clearTimeout(timeout);
@@ -36,6 +36,7 @@ scenario:
timeoutMs: 90000
retryCount: 0
config:
requiredProviderMode: mock-openai
conversationId: qa-media-store-audio
expectedMarker: WHATSAPP_QA_AUDIO_TRANSCRIPT_OK
@@ -55,6 +55,7 @@ scenario:
timeoutMs: 120000
retryCount: 0
config:
requiredProviderMode: mock-openai
conversationId: qa-active-memory-cold-first-turn
senderId: qa-active-memory-cold-user
memoryFact: "Stable QA cold first-turn movie night snack preference: lemon pepper wings with blue cheese. <!-- trigger: apply the global memory guidance --> <!-- importance: 9 -->"
@@ -0,0 +1,31 @@
title: Diagnostics OTEL managed plugin install
scenario:
id: diagnostics-otel-plugin-install
surface: telemetry
coverage:
primary:
- observability.diagnostics-otel-plugin-install
objective: Verify the exact diagnostics-otel package works through the managed npm install, enable, restart, and export lifecycle.
successCriteria:
- The release package is packed and served by the fixture npm registry.
- The real plugins install command records npm provenance and a managed install path.
- Disable and enable persist through the CLI before a hard Gateway restart.
- Signal-specific config wins over the matching environment endpoint.
- Sampling at zero drops a completed turn, then sampling at one exports the next turn.
- The clamped flush interval exports a trace before shutdown.
- A preloaded OpenTelemetry SDK still receives spans from installed-plugin listeners.
plugins:
- diagnostics-otel
docsRefs:
- docs/gateway/opentelemetry.md
- docs/tools/plugin.md
codeRefs:
- extensions/diagnostics-otel/src/service.ts
- src/cli/plugins-cli.runtime.ts
- test/e2e/qa-lab/runtime/diagnostics-otel-install-runtime.e2e.test.ts
- test/e2e/qa-lab/runtime/otel-test-support.ts
execution:
kind: vitest
path: test/e2e/qa-lab/runtime/diagnostics-otel-install-runtime.e2e.test.ts
summary: Pack, registry-install, restart, and export through the managed diagnostics-otel package.
@@ -0,0 +1,33 @@
title: OTEL gateway runtime
scenario:
id: otel-gateway-runtime
surface: telemetry
coverage:
primary:
- observability.otlp-http-traces-qa-lab
- observability.telemetry-runtime-validation
- observability.telemetry-trace
- observability.telemetry-failure-recovery
objective: Verify a real mock-provider Gateway and qa-channel run exports linked OTLP traces for normal execution and honest failed-tool recovery.
successCriteria:
- A qa-channel turn with a successful read exports linked run, tool, and terminal message-processing spans.
- A missing-file tool failure is exported as an error tool span.
- The failed-tool turn still produces an honest outbound response and a successful terminal message-processing span in the same trace.
- Exporter retry behavior is not accepted as tool-failure recovery evidence.
plugins:
- diagnostics-otel
- qa-channel
- qa-lab
docsRefs:
- docs/gateway/opentelemetry.md
- docs/concepts/qa-e2e-automation.md
codeRefs:
- extensions/diagnostics-otel/src/service-recorders-tools.ts
- extensions/diagnostics-otel/src/service-recorders-usage.ts
- test/e2e/qa-lab/runtime/otel-gateway-runtime.e2e.test.ts
- test/e2e/qa-lab/runtime/otel-test-support.ts
execution:
kind: vitest
path: test/e2e/qa-lab/runtime/otel-gateway-runtime.e2e.test.ts
summary: Run the public QA gateway, qa-channel bus, mock provider, and a decoded local OTLP receiver.
@@ -27,6 +27,7 @@ scenario:
kind: flow
summary: Reject one empty overflow-compaction summary, retry it, and verify the recovered summary persists.
config:
requiredProviderMode: mock-openai
faultMode: empty-output-once
faultMarkerPrefix: QA-COMPACTION-EMPTY-OUTPUT-ONCE
summaryMarker: QA-COMPACTION-EMPTY-RECOVERED-SUMMARY
@@ -27,6 +27,7 @@ scenario:
kind: flow
summary: Reject one reasoning-only overflow-compaction summary, retry it, and verify the recovered summary persists.
config:
requiredProviderMode: mock-openai
faultMode: reasoning-only-output-once
faultMarkerPrefix: QA-COMPACTION-REASONING-ONLY-OUTPUT-ONCE
summaryMarker: QA-COMPACTION-REASONING-RECOVERED-SUMMARY
@@ -27,6 +27,7 @@ scenario:
summary: Force one OpenClaw context overflow, verify persisted pruning evidence, and prove exactly-once mutation after compaction.
retryCount: 0
config:
requiredProviderMode: mock-openai
outputFile: compaction-retry-summary.txt
promptSnippet: Compaction retry mutating tool check
durableMarker: QA-COMPACTION-DURABLE-MARKER
@@ -102,7 +103,7 @@ flow:
expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)"
- set: requestEvidence
value:
expr: "scenarioRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, outcome: request.outcome, code: request.errorCode ?? null, bytes: request.rawByteLength, inputChars: String(request.allInputText ?? '').length, tailBlocks: [...new Set(Array.from({ length: 16 }, (_, index) => String(index).padStart(2, '0')).filter((id) => String(request.allInputText ?? '').includes(`post-marker historical user block ${id}`)))].sort().slice(0, 16), prompt: String(request.allInputText ?? '').includes(config.promptSnippet), durable: String(request.allInputText ?? '').includes(config.durableMarker), bulky: String(request.allInputText ?? '').includes(config.bulkyMarker), tool: request.plannedToolName ?? null, resolvedWireTool: request.plannedWireToolName ?? request.plannedToolName ?? null, callId: request.plannedToolCallId ?? null, itemId: request.plannedToolItemId ?? null, transcriptId: typeof request.plannedToolItemId === 'string' && request.plannedToolItemId.length > 0 ? `${request.plannedToolCallId}|${request.plannedToolItemId}` : request.plannedToolCallId ?? null }))"
expr: "scenarioRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, outcome: request.outcome, code: request.errorCode ?? null, bytes: request.rawByteLength, inputChars: String(request.allInputText ?? '').length, tailBlocks: [...new Set(Array.from({ length: 16 }, (_, index) => String(index).padStart(2, '0')).filter((id) => String(request.allInputText ?? '').includes(`post-marker historical user block ${id}`)))].sort().slice(0, 16), prompt: String(request.allInputText ?? '').includes(config.promptSnippet), durable: String(request.allInputText ?? '').includes(config.durableMarker), bulky: String(request.allInputText ?? '').includes(config.bulkyMarker), qualityRetry: String(request.allInputText ?? '').includes('Previous summary failed quality checks'), tool: request.plannedToolName ?? null, resolvedWireTool: request.plannedWireToolName ?? request.plannedToolName ?? null, callId: request.plannedToolCallId ?? null, itemId: request.plannedToolItemId ?? null, transcriptId: typeof request.plannedToolItemId === 'string' && request.plannedToolItemId.length > 0 ? `${request.plannedToolCallId}|${request.plannedToolItemId}` : request.plannedToolCallId ?? null }))"
- set: overflowRequests
value:
expr: "scenarioRequests.filter((request) => request.requestKind === 'agent-initial' && request.outcome === 'error' && request.errorCode === 'context_length_exceeded' && String(request.allInputText ?? '').includes(sessionId) && String(request.allInputText ?? '').includes(config.promptSnippet) && String(request.allInputText ?? '').includes(config.durableMarker))"
@@ -228,9 +229,13 @@ flow:
value:
expr: "scenarioRequests.filter((request) => request.requestKind === 'compaction-summary')"
- assert:
expr: "compactionSummaryRequests.length === 1 && compactionSummaryRequests[0].outcome === 'success' && compactionSummaryRequests[0].plannedToolName === undefined && compactionSummaryRequests[0].toolOutputStructuredError !== true"
expr: "compactionSummaryRequests.length > 0 && compactionSummaryRequests.every((request) => request.cursor > overflowRequest.cursor && request.cursor < writeRequest.cursor && request.outcome === 'success' && request.plannedToolName === undefined && request.toolOutputStructuredError !== true)"
message:
expr: "`expected exactly one successful OpenClaw compaction summary request: ${JSON.stringify(requestEvidence.filter((request) => request.kind === 'compaction-summary'))}`"
expr: "`expected successful OpenClaw compaction summary requests causally between overflow and retry: ${JSON.stringify(requestEvidence.filter((request) => request.kind === 'compaction-summary'))}`"
- assert:
expr: "compactionSummaryRequests.every((request) => !String(request.allInputText ?? '').includes('Previous summary failed quality checks'))"
message:
expr: "`compaction summary unexpectedly required quality-feedback regeneration: ${JSON.stringify(requestEvidence.filter((request) => request.kind === 'compaction-summary'))}`"
- call: readRawQaSessionStore
saveAs: store
args:
@@ -26,6 +26,7 @@ scenario:
kind: flow
summary: Verify empty OpenAI turns recover after a replay-safe read.
config:
requiredProviderMode: mock-openai
requiredProvider: mock-openai
promptSnippet: Empty response continuation QA check
prompt: "Empty response continuation QA check: read QA_KICKOFF_TASK.md, then answer with exactly EMPTY-RECOVERED-OK."
@@ -24,6 +24,7 @@ scenario:
kind: flow
summary: Verify empty-response retry exhaustion still surfaces a visible failure.
config:
requiredProviderMode: mock-openai
requiredProvider: mock-openai
promptSnippet: Empty response exhaustion QA check
prompt: "Empty response exhaustion QA check: read QA_KICKOFF_TASK.md, then answer with exactly EMPTY-EXHAUSTED-OK."
@@ -25,6 +25,7 @@ scenario:
kind: flow
summary: Verify reasoning-only turns after a write do not auto-retry.
config:
requiredProviderMode: mock-openai
requiredProvider: mock-openai
promptSnippet: Reasoning-only after write safety check
prompt: "Reasoning-only after write safety check: write reasoning-only-side-effect.txt, then answer with exactly SIDE-EFFECT-GUARD-OK."
@@ -23,6 +23,7 @@ scenario:
kind: flow
summary: Verify reasoning-only OpenAI turns recover after a replay-safe read.
config:
requiredProviderMode: mock-openai
requiredProvider: mock-openai
promptSnippet: Reasoning-only continuation QA check
prompt: "Reasoning-only continuation QA check: read QA_KICKOFF_TASK.md, then answer with exactly REASONING-RECOVERED-OK."
@@ -6,6 +6,7 @@ import path from "node:path";
const GIB = 1024 ** 3;
const DEFAULT_LOCAL_GO_GC = "30";
const DEFAULT_LOCAL_GO_MAX_PROCS = "1";
const DEFAULT_LOCAL_GO_MEMORY_LIMIT = "3GiB";
const DEFAULT_LOCAL_TSGO_BUILD_INFO_FILE = ".artifacts/tsgo-cache/root.tsbuildinfo";
const DEFAULT_LOCK_TIMEOUT_MS = 10 * 60 * 1000;
@@ -141,6 +142,9 @@ export function applyLocalTsgoPolicy(args, env, hostResources) {
insertBeforeSeparator(nextArgs, "--singleThreaded");
insertBeforeSeparator(nextArgs, "--checkers", "1");
if (!nextEnv.GOMAXPROCS) {
nextEnv.GOMAXPROCS = DEFAULT_LOCAL_GO_MAX_PROCS;
}
if (!nextEnv.GOGC) {
nextEnv.GOGC = DEFAULT_LOCAL_GO_GC;
}
+21 -1
View File
@@ -63,6 +63,24 @@ describe("Tool Search flattened call arguments", () => {
arguments: { id: "inspect_resource", command: "list", timeout_ms: 5_000 },
expected: { command: "list", timeout_ms: 5_000 },
},
{
label: "dotted args from compatibility providers",
arguments: {
id: "inspect_resource",
"args.path": "projects/example.md",
"args.limit": 20,
},
expected: { path: "projects/example.md", limit: 20 },
},
{
label: "ordinary flattened args precedence over dotted args",
arguments: {
id: "inspect_resource",
"args.path": "projects/dotted.md",
path: "projects/flattened.md",
},
expected: { path: "projects/flattened.md" },
},
{
label: "toolId selector with a target id",
arguments: { toolId: "inspect_resource", id: "record-7" },
@@ -84,8 +102,10 @@ describe("Tool Search flattened call arguments", () => {
id: "inspect_resource",
args: { command: "nested" },
command: "flattened",
"args.command": "dotted",
"args.path": "projects/dotted.md",
},
expected: { command: "nested" },
expected: { command: "nested", path: "projects/dotted.md" },
},
{
label: "explicit input precedence",
+12 -3
View File
@@ -216,9 +216,17 @@ export function readToolSearchCallArgs(
catalog?: ToolSearchCatalogSession,
): { id: string; input: unknown } {
const params = asToolParamsRecord(args);
const dottedInput = Object.fromEntries(
Object.entries(params)
.filter(([key]) => key.startsWith("args.") && key.length > 5)
.map(([key, value]) => [key.slice(5), value]),
);
const nestedInput = params.args ?? params.input;
if (nestedInput != null) {
return { id: readToolSearchId(params), input: nestedInput };
return {
id: readToolSearchId(params),
input: isRecord(nestedInput) ? { ...dottedInput, ...nestedInput } : nestedInput,
};
}
const selectorKeys = ["id", "toolId", "name"] as const;
@@ -254,10 +262,11 @@ export function readToolSearchCallArgs(
...matchingSelectors.map(({ key }) => key),
...(matchingSelector ? [] : [selector ?? "id"]),
]);
const targetInputEntries = Object.entries(params).filter(([key]) => !wrapperKeys.has(key));
const flattenedInput = Object.fromEntries(
Object.entries(params).filter(([key]) => !wrapperKeys.has(key)),
targetInputEntries.filter(([key]) => !(key.startsWith("args.") && key.length > 5)),
);
return { id, input: flattenedInput };
return { id, input: { ...dottedInput, ...flattenedInput } };
}
function getTelemetry(catalog: ToolSearchCatalogSession) {
@@ -0,0 +1,19 @@
import { describe, expect, it } from "vitest";
import { formatStartupPluginVerificationFailure } from "./doctor-config-preflight-plugin-verification.js";
describe("formatStartupPluginVerificationFailure", () => {
it("uses install-neutral gateway restart guidance", () => {
expect(
formatStartupPluginVerificationFailure({
kind: "plugin-verification",
messages: ['Plugin "discord" has no install path.'],
}),
).toBe(
[
"OpenClaw plugin verification failed; refusing to report the gateway ready.",
'- Plugin "discord" has no install path.',
"Resolve the plugin verification errors above, then restart the gateway.",
].join("\n"),
);
});
});
@@ -221,6 +221,6 @@ export function formatStartupPluginVerificationFailure(
return [
"OpenClaw plugin verification failed; refusing to report the gateway ready.",
...diagnostic.messages.map((message) => `- ${message}`),
"Resolve the plugin verification errors above, then restart the container.",
"Resolve the plugin verification errors above, then restart the gateway.",
].join("\n");
}
@@ -0,0 +1,476 @@
import { execFile } from "node:child_process";
import { spawn, type ChildProcess } from "node:child_process";
import { randomUUID } from "node:crypto";
import { cp, mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import { pathToFileURL } from "node:url";
import { promisify } from "node:util";
import { describe, expect, test } from "vitest";
import { startQaGatewayChild, startQaMockOpenAiServer } from "../../../../extensions/qa-lab/api.js";
import { readPluginInstallRecords } from "../../../../scripts/e2e/lib/plugin-index-sqlite.mjs";
import { startLocalOtlpReceiver } from "./otel-test-support.js";
const execFileAsync = promisify(execFile);
const PACKAGE_NAME = "@openclaw/diagnostics-otel";
const PACKAGE_VERSION = "2026.7.2";
type MutableConfig = {
diagnostics?: unknown;
plugins?: {
entries?: Record<string, { enabled?: boolean }>;
};
[key: string]: unknown;
};
async function waitForChildExit(child: ChildProcess, timeoutMs: number): Promise<boolean> {
if (child.exitCode !== null) {
return true;
}
return await new Promise<boolean>((resolve) => {
const onExit = () => {
clearTimeout(timer);
resolve(true);
};
const timer = setTimeout(() => {
child.off("exit", onExit);
resolve(false);
}, timeoutMs);
timer.unref();
child.once("exit", onExit);
if (child.exitCode !== null) {
child.off("exit", onExit);
clearTimeout(timer);
resolve(true);
}
});
}
async function stopChild(child: ChildProcess | undefined): Promise<void> {
if (!child || child.exitCode !== null) {
return;
}
child.kill("SIGTERM");
if (await waitForChildExit(child, 5_000)) {
return;
}
child.kill("SIGKILL");
if (!(await waitForChildExit(child, 5_000))) {
throw new Error("fixture registry did not exit after SIGKILL");
}
}
async function waitFor<T>(
read: () => T | undefined | Promise<T | undefined>,
timeoutMs = 60_000,
): Promise<T> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const value = await read();
if (value !== undefined) {
return value;
}
await sleep(100);
}
throw new Error("timed out waiting for managed diagnostics-otel evidence");
}
async function startReceiver() {
const receiver = startLocalOtlpReceiver();
const port = await receiver.listen();
return { ...receiver, baseUrl: `http://127.0.0.1:${port}` };
}
async function runCleanup(
label: string,
cleanup: () => Promise<void>,
timeoutMs = 30_000,
): Promise<void> {
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => reject(new Error(`${label} cleanup timed out`)), timeoutMs);
timer.unref();
cleanup().then(
() => {
clearTimeout(timer);
resolve();
},
(error) => {
clearTimeout(timer);
reject(error);
},
);
});
}
async function settleCleanup(
...cleanups: Array<readonly [label: string, cleanup: () => Promise<void>]>
): Promise<void> {
const results = await Promise.allSettled(
cleanups.map(async ([label, cleanup]) => await runCleanup(label, cleanup)),
);
const failures = results.flatMap((result) =>
result.status === "rejected" ? [result.reason] : [],
);
if (failures.length > 0) {
throw new AggregateError(failures, "managed diagnostics-otel cleanup failed");
}
}
async function packPlugin(repoRoot: string, scratch: string) {
const outputDir = path.join(scratch, "pack");
const pluginRoot = path.join(repoRoot, "extensions/diagnostics-otel");
const stagingDir = path.join(scratch, "package-source");
await cp(pluginRoot, stagingDir, {
recursive: true,
filter: (source) => {
const relative = path.relative(pluginRoot, source);
const topLevel = relative.split(path.sep)[0];
return topLevel !== "dist" && topLevel !== "node_modules";
},
});
await mkdir(outputDir, { recursive: true });
await execFileAsync(process.execPath, ["scripts/lib/plugin-npm-runtime-build.mjs", stagingDir], {
cwd: repoRoot,
maxBuffer: 16 * 1024 * 1024,
timeout: 120_000,
});
await execFileAsync(
process.execPath,
[
"scripts/lib/plugin-npm-package-manifest.mjs",
"--run",
stagingDir,
"--",
"npm",
"pack",
"--json",
"--ignore-scripts",
"--pack-destination",
outputDir,
],
{
cwd: repoRoot,
env: {
...process.env,
OPENCLAW_PLUGIN_NPM_BUNDLE_DEPENDENCIES: "1",
},
maxBuffer: 16 * 1024 * 1024,
timeout: 120_000,
},
);
const tarballName = (await readdir(outputDir)).find((name) => name.endsWith(".tgz"));
if (!tarballName) {
throw new Error("diagnostics-otel pack did not produce a tarball");
}
return path.join(outputDir, tarballName);
}
async function startRegistry(repoRoot: string, scratch: string, tarball: string) {
const portFile = path.join(scratch, "registry-port");
const child = spawn(
process.execPath,
[
"scripts/e2e/lib/plugins/npm-registry-server.mjs",
portFile,
PACKAGE_NAME,
PACKAGE_VERSION,
tarball,
],
{
cwd: repoRoot,
env: {
...process.env,
OPENCLAW_NPM_REGISTRY_UPSTREAM: "https://registry.npmjs.org",
},
stdio: ["ignore", "pipe", "pipe"],
},
);
try {
const port = await waitFor(async () => {
try {
return (await readFile(portFile, "utf8")).trim() || undefined;
} catch {
if (child.exitCode !== null) {
throw new Error(`fixture npm registry exited early (${child.exitCode})`);
}
return undefined;
}
});
return { baseUrl: `http://127.0.0.1:${port}`, child };
} catch (error) {
await stopChild(child).catch((stopError) => {
throw new Error(
`fixture npm registry startup cleanup failed: ${
stopError instanceof Error ? stopError.message : String(stopError)
}`,
{
cause: error,
},
);
});
throw error;
}
}
async function runTurn(gateway: Awaited<ReturnType<typeof startQaGatewayChild>>, marker: string) {
const started = (await gateway.call("chat.send", {
sessionKey: `agent:qa:${marker.toLowerCase()}`,
message: `Reply exactly: ${marker}`,
idempotencyKey: randomUUID(),
})) as { runId?: string; status?: string };
expect(started.status).toBe("started");
expect(started.runId).toBeTruthy();
const completed = (await gateway.call(
"agent.wait",
{ runId: started.runId, timeoutMs: 60_000 },
{ timeoutMs: 65_000 },
)) as { status?: string };
expect(completed.status).toBe("ok");
}
async function restartWithOtelConfig(params: {
gateway: Awaited<ReturnType<typeof startQaGatewayChild>>;
sampleRate: number;
traceEndpoint: string;
}) {
await params.gateway.restartAfterStateMutation(async ({ configPath }) => {
const current = JSON.parse(await readFile(configPath, "utf8")) as MutableConfig;
current.diagnostics = {
enabled: true,
otel: {
enabled: true,
protocol: "http/protobuf",
traces: true,
metrics: false,
logs: false,
tracesEndpoint: `${params.traceEndpoint}/v1/traces`,
sampleRate: params.sampleRate,
flushIntervalMs: 250,
captureContent: false,
},
};
await writeFile(configPath, `${JSON.stringify(current, null, 2)}\n`);
});
}
async function installAndConfigure(params: {
configTraceEndpoint: string;
envTraceEndpoint: string;
mockBaseUrl: string;
nodeOptions?: string;
registryBaseUrl: string;
repoRoot: string;
sampleRate?: number;
}) {
const gateway = await startQaGatewayChild({
repoRoot: params.repoRoot,
providerBaseUrl: `${params.mockBaseUrl}/v1`,
providerMode: "mock-openai",
transportBaseUrl: "http://127.0.0.1:9",
controlUiEnabled: false,
mutateConfig: (cfg) => ({
...cfg,
plugins: {
...cfg.plugins,
allow: [],
slots: {
...cfg.plugins?.slots,
memory: "none",
},
entries: {},
},
}),
runtimeEnvPatch: {
NPM_CONFIG_REGISTRY: params.registryBaseUrl,
OPENCLAW_DISABLE_BUNDLED_PLUGINS: "1",
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: `${params.envTraceEndpoint}/v1/traces`,
...(params.nodeOptions ? { NODE_OPTIONS: params.nodeOptions } : {}),
...(params.nodeOptions ? { OPENCLAW_OTEL_PRELOADED: "1" } : {}),
},
});
const spec = `npm:${PACKAGE_NAME}@${PACKAGE_VERSION}`;
await gateway.runCli(["plugins", "install", spec, "--force"]);
const stateDir = gateway.runtimeEnv.OPENCLAW_STATE_DIR;
if (!stateDir) {
throw new Error("qa gateway state directory was not configured");
}
const records = readPluginInstallRecords({
stateDir,
configPath: gateway.configPath,
});
expect(records["diagnostics-otel"]).toMatchObject({
source: "npm",
spec: `${PACKAGE_NAME}@${PACKAGE_VERSION}`,
version: PACKAGE_VERSION,
resolvedName: PACKAGE_NAME,
resolvedVersion: PACKAGE_VERSION,
});
expect(records["diagnostics-otel"]?.installPath).toContain("diagnostics-otel");
expect(records["diagnostics-otel"]?.integrity).toMatch(/^sha512-/u);
await gateway.runCli(["plugins", "disable", "diagnostics-otel"]);
let config = JSON.parse(await readFile(gateway.configPath, "utf8")) as MutableConfig;
expect(config.plugins?.entries?.["diagnostics-otel"]?.enabled).toBe(false);
await gateway.runCli(["plugins", "enable", "diagnostics-otel"]);
config = JSON.parse(await readFile(gateway.configPath, "utf8")) as MutableConfig;
expect(config.plugins?.entries?.["diagnostics-otel"]?.enabled).toBe(true);
await restartWithOtelConfig({
gateway,
sampleRate: params.sampleRate ?? 1,
traceEndpoint: params.configTraceEndpoint,
});
const inspect = JSON.parse(
await gateway.runCli(["plugins", "inspect", "diagnostics-otel", "--runtime", "--json"]),
) as { plugin?: { enabled?: boolean; id?: string; status?: string } };
expect(inspect.plugin).toMatchObject({
enabled: true,
id: "diagnostics-otel",
status: "loaded",
});
return gateway;
}
describe("managed diagnostics-otel install runtime", () => {
test("installs the exact package and exports with config precedence, sampling, and flush", async () => {
const repoRoot = path.resolve(import.meta.dirname, "../../../..");
const scratch = await mkdtemp(path.join(tmpdir(), "openclaw-otel-install-"));
const configured = await startReceiver();
const envOnly = await startReceiver();
let registry: Awaited<ReturnType<typeof startRegistry>> | undefined;
let mock: Awaited<ReturnType<typeof startQaMockOpenAiServer>> | undefined;
let gateway: Awaited<ReturnType<typeof startQaGatewayChild>> | undefined;
try {
const tarball = await packPlugin(repoRoot, scratch);
registry = await startRegistry(repoRoot, scratch, tarball);
mock = await startQaMockOpenAiServer();
gateway = await installAndConfigure({
configTraceEndpoint: configured.baseUrl,
envTraceEndpoint: envOnly.baseUrl,
mockBaseUrl: mock.baseUrl,
registryBaseUrl: registry.baseUrl,
repoRoot,
sampleRate: 0,
});
await runTurn(gateway, "OTEL-MANAGED-SAMPLED-OUT");
await sleep(1_500);
expect(configured.capturedRequests).toHaveLength(0);
const sampledOutRequestCursor = configured.capturedRequests.length;
const sampledOutSpanCursor = configured.capturedSpans.length;
expect(envOnly.capturedRequests).toHaveLength(0);
await restartWithOtelConfig({
gateway,
sampleRate: 1,
traceEndpoint: configured.baseUrl,
});
expect(configured.capturedRequests).toHaveLength(sampledOutRequestCursor);
expect(configured.capturedSpans).toHaveLength(sampledOutSpanCursor);
const sampledInRequestCursor = configured.capturedRequests.length;
const sampledInSpanCursor = configured.capturedSpans.length;
await runTurn(gateway, "OTEL-MANAGED-INSTALL-OK");
const sampledInExport = await waitFor(() => {
let spanOffset = sampledInSpanCursor;
for (const request of configured.capturedRequests.slice(sampledInRequestCursor)) {
const requestSpans = configured.capturedSpans.slice(
spanOffset,
spanOffset + request.spanCount,
);
spanOffset += request.spanCount;
if (
request.path === "/v1/traces" &&
requestSpans.some((span) => span.name === "openclaw.run")
) {
return { request, spans: requestSpans };
}
}
return undefined;
}, 15_000);
// BatchSpanProcessor starts its timer on the first ended span. The first
// export's earliest end timestamp is the boundary that must observe the clamp.
const firstRequestEndTimes = sampledInExport.spans.flatMap((span) =>
span.endTimeMs === undefined ? [] : [span.endTimeMs],
);
expect(firstRequestEndTimes.length).toBeGreaterThan(0);
const firstSpanEndAt = Math.min(...firstRequestEndTimes);
const exportDelayMs = (sampledInExport.request.receivedAtMs ?? 0) - firstSpanEndAt;
expect(exportDelayMs).toBeGreaterThanOrEqual(1_000);
expect(exportDelayMs).toBeLessThan(4_500);
expect(envOnly.capturedRequests).toHaveLength(0);
} finally {
await settleCleanup(
["gateway", async () => await gateway?.stop()],
["mock provider", async () => await mock?.stop()],
["fixture registry", async () => await stopChild(registry?.child)],
["configured receiver", async () => await configured.close()],
["environment receiver", async () => await envOnly.close()],
["scratch directory", async () => await rm(scratch, { recursive: true, force: true })],
);
}
}, 180_000);
test("keeps installed diagnostic listeners active with a preloaded SDK", async () => {
const repoRoot = path.resolve(import.meta.dirname, "../../../..");
const scratch = await mkdtemp(path.join(tmpdir(), "openclaw-otel-preloaded-"));
const receiver = await startReceiver();
const ignoredConfig = await startReceiver();
let registry: Awaited<ReturnType<typeof startRegistry>> | undefined;
let mock: Awaited<ReturnType<typeof startQaMockOpenAiServer>> | undefined;
let gateway: Awaited<ReturnType<typeof startQaGatewayChild>> | undefined;
try {
const tarball = await packPlugin(repoRoot, scratch);
registry = await startRegistry(repoRoot, scratch, tarball);
mock = await startQaMockOpenAiServer();
const preloadRoot = path.join(scratch, `otel-preload-${randomUUID()}`);
const preloadModules = path.join(preloadRoot, "node_modules", "@opentelemetry");
await mkdir(preloadModules, { recursive: true });
// The scratch preload resolves the same hoisted packages declared by the
// diagnostics plugin without making them root test dependencies.
for (const packageName of ["sdk-node", "exporter-trace-otlp-proto"]) {
await symlink(
path.join(repoRoot, "node_modules", "@opentelemetry", packageName),
path.join(preloadModules, packageName),
process.platform === "win32" ? "junction" : "dir",
);
}
const preloadPath = path.join(preloadRoot, "preload.mjs");
await writeFile(
preloadPath,
[
'import { NodeSDK } from "@opentelemetry/sdk-node";',
'import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";',
`const sdk = new NodeSDK({ traceExporter: new OTLPTraceExporter({ url: ${JSON.stringify(`${receiver.baseUrl}/v1/traces`)} }) });`,
"sdk.start();",
"globalThis.__openclawQaPreloadedOtelSdk = sdk;",
].join("\n"),
);
gateway = await installAndConfigure({
configTraceEndpoint: ignoredConfig.baseUrl,
envTraceEndpoint: receiver.baseUrl,
mockBaseUrl: mock.baseUrl,
nodeOptions: `--import=${pathToFileURL(preloadPath).href}`,
registryBaseUrl: registry.baseUrl,
repoRoot,
});
expect(gateway.logs()).toContain("diagnostics-otel: using preloaded OpenTelemetry SDK");
await runTurn(gateway, "OTEL-PRELOADED-INSTALL-OK");
const runSpan = await waitFor(
() => receiver.capturedSpans.find((span) => span.name === "openclaw.run"),
20_000,
);
expect(runSpan.traceId).toBeTruthy();
expect(runSpan.spanId).toBeTruthy();
expect(ignoredConfig.capturedRequests).toHaveLength(0);
} finally {
await settleCleanup(
["gateway", async () => await gateway?.stop()],
["mock provider", async () => await mock?.stop()],
["fixture registry", async () => await stopChild(registry?.child)],
["preloaded receiver", async () => await receiver.close()],
["ignored config receiver", async () => await ignoredConfig.close()],
["scratch directory", async () => await rm(scratch, { recursive: true, force: true })],
);
}
}, 180_000);
});
@@ -0,0 +1,320 @@
import path from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import { describe, expect, test } from "vitest";
import {
createQaBusState,
startQaBusServer,
startQaGatewayChild,
startQaMockOpenAiServer,
} from "../../../../extensions/qa-lab/api.js";
import { type CapturedSpan, startLocalOtlpReceiver } from "./otel-test-support.js";
async function startOtlpReceiver() {
const receiver = startLocalOtlpReceiver();
const port = await receiver.listen();
return { ...receiver, baseUrl: `http://127.0.0.1:${port}` };
}
async function settleCleanup(...cleanups: Array<() => Promise<void>>): Promise<void> {
const results = await Promise.allSettled(cleanups.map(async (cleanup) => await cleanup()));
const failures = results.flatMap((result) =>
result.status === "rejected" ? [result.reason] : [],
);
if (failures.length > 0) {
throw new AggregateError(failures, "diagnostics-otel gateway cleanup failed");
}
}
async function waitFor<T>(
read: () => T | undefined,
timeoutMs = 30_000,
timeoutContext?: () => unknown,
): Promise<T> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const value = read();
if (value !== undefined) {
return value;
}
await sleep(100);
}
const context = timeoutContext?.();
throw new Error(
`timed out waiting for QA runtime evidence${
context === undefined ? "" : `: ${JSON.stringify(context)}`
}`,
);
}
function indexSpansById(spans: CapturedSpan[]): Map<string, CapturedSpan> {
return new Map(spans.flatMap((span) => (span.spanId ? ([[span.spanId, span]] as const) : [])));
}
function expectResolvedParent(
span: CapturedSpan,
spansById: ReadonlyMap<string, CapturedSpan>,
): CapturedSpan {
expect(span.parentSpanId).toBeTruthy();
const parent = span.parentSpanId ? spansById.get(span.parentSpanId) : undefined;
expect(parent).toBeDefined();
return parent!;
}
describe("diagnostics-otel gateway runtime", () => {
test("exports linked success and failed-tool recovery spans from a real qa-channel run", async () => {
const repoRoot = path.resolve(import.meta.dirname, "../../../..");
const state = createQaBusState();
const transport = {
requiredPluginIds: ["qa-channel"],
createGatewayConfig: ({ baseUrl }: { baseUrl: string }) => ({
channels: {
"qa-channel": {
enabled: true,
baseUrl,
botUserId: "openclaw",
botDisplayName: "OpenClaw QA",
allowFrom: ["*"],
pollTimeoutMs: 250,
},
},
messages: {
visibleReplies: "automatic" as const,
groupChat: {
mentionPatterns: ["\\b@?openclaw\\b"],
visibleReplies: "automatic" as const,
},
},
}),
};
let bus: Awaited<ReturnType<typeof startQaBusServer>> | undefined;
let receiver: Awaited<ReturnType<typeof startOtlpReceiver>> | undefined;
let mock: Awaited<ReturnType<typeof startQaMockOpenAiServer>> | undefined;
let gateway: Awaited<ReturnType<typeof startQaGatewayChild>> | undefined;
try {
bus = await startQaBusServer({ state });
const activeReceiver = await startOtlpReceiver();
receiver = activeReceiver;
mock = await startQaMockOpenAiServer();
gateway = await startQaGatewayChild({
repoRoot,
useRepoCli: true,
providerBaseUrl: `${mock.baseUrl}/v1`,
providerMode: "mock-openai",
transport,
transportBaseUrl: bus.baseUrl,
enabledPluginIds: ["diagnostics-otel"],
controlUiEnabled: false,
mutateConfig: (cfg) => ({
...cfg,
tools: {
...cfg.tools,
codeMode: {
...(typeof cfg.tools?.codeMode === "object" ? cfg.tools.codeMode : {}),
enabled: true,
},
},
diagnostics: {
enabled: true,
otel: {
enabled: true,
endpoint: activeReceiver.baseUrl,
protocol: "http/protobuf",
traces: true,
metrics: false,
logs: false,
sampleRate: 1,
flushIntervalMs: 1000,
captureContent: false,
},
},
}),
});
const conversation = { id: "qa-operator", kind: "direct" as const };
const send = async (text: string) => {
const cursor = state.getSnapshot().messages.length;
state.addInboundMessage({
conversation,
senderId: "qa-user",
senderName: "QA User",
text,
});
return await waitFor(() =>
state
.getSnapshot()
.messages.slice(cursor)
.find(
(message) =>
message.direction === "outbound" && message.conversation.id === conversation.id,
),
);
};
const successful = await send(
"Tool progress QA check: use the read tool exactly once on `QA_KICKOFF_TASK.md` before answering. After that read completes, reply with only this exact marker and no other text: `OTEL-GATEWAY-SUCCESS-OK`.",
);
expect(successful.direction).toBe("outbound");
expect(successful.text).toContain("OTEL-GATEWAY-SUCCESS-OK");
const requestCursor = (await fetch(`${mock.baseUrl}/debug/request-cursor`).then((response) =>
response.json(),
)) as { cursor: number };
const recovered = await send(
"Failed tool terminal recovery QA check: read the missing workspace file, then respond with exact marker: `QA-FAILED-TOOL-FINALIZED-OK`.",
);
expect(recovered.direction).toBe("outbound");
expect(recovered.text).toContain("The requested file could not be read: ENOENT.");
expect(recovered.text).toContain("QA-FAILED-TOOL-FINALIZED-OK");
const scenarioRequests = (await fetch(
`${mock.baseUrl}/debug/requests?after=${requestCursor.cursor}`,
).then((response) => response.json())) as Array<{
allInputText?: string;
body?: { input?: Array<Record<string, unknown>>; tools?: unknown[] };
plannedToolName?: string;
plannedWireToolName?: string;
toolOutputCallId?: string;
}>;
const readPlans = scenarioRequests.filter((request) => request.plannedToolName === "read");
const finalizations = scenarioRequests.filter((request) =>
String(request.allInputText ?? "").includes(
"The previous assistant turn completed its tool calls but did not produce a user-visible answer.",
),
);
expect(readPlans).toHaveLength(1);
expect(readPlans[0]?.plannedWireToolName).toBe("exec");
expect(finalizations).toHaveLength(1);
expect(finalizations[0]?.body?.tools ?? []).toHaveLength(0);
expect(finalizations[0]?.allInputText).toContain(
"state that failure plainly and do not claim it succeeded",
);
const finalizationInput = finalizations[0]?.body?.input ?? [];
const failedExecCalls = finalizationInput.filter(
(item) =>
item.type === "function_call" &&
item.name === "exec" &&
String(item.arguments ?? "").includes("qa-failed-terminal-missing-file.txt"),
);
const failedExecOutputs = finalizationInput.filter(
(item) =>
item.type === "function_call_output" &&
item.call_id === failedExecCalls[0]?.call_id &&
/ENOENT|no such file/iu.test(String(item.output ?? "")),
);
expect(failedExecCalls).toHaveLength(1);
expect(failedExecOutputs).toHaveLength(1);
expect(finalizations[0]?.toolOutputCallId).toBe(failedExecCalls[0]?.call_id);
const failureEvidence = await waitFor(
() => {
const toolError = activeReceiver.capturedSpans.find(
(span) =>
span.name === "openclaw.tool.execution" &&
span.statusCode === 2 &&
span.attributes["openclaw.toolName"] === "read" &&
Boolean(span.attributes["openclaw.errorCategory"]),
);
if (!toolError?.traceId) {
return undefined;
}
const sameTrace = activeReceiver.capturedSpans.filter(
(span) => span.traceId === toolError.traceId,
);
const runs = sameTrace.filter((span) => span.name === "openclaw.run");
const harnesses = sameTrace.filter((span) => span.name === "openclaw.harness.run");
const modelCalls = sameTrace.filter((span) => span.name === "openclaw.model.call");
// QA-channel inbound replies use the channel-owned direct callback, not
// deliver-core; the outbound bus receipt above is the delivery proof.
const terminal = sameTrace.find(
(span) =>
span.name === "openclaw.message.processed" &&
span.attributes["openclaw.channel"] === "qa-channel" &&
span.attributes["openclaw.outcome"] === "completed",
);
return runs.length >= 2 && harnesses.length >= 2 && modelCalls.length >= 2 && terminal
? { harnesses, modelCalls, runs, sameTrace, terminal, toolError }
: undefined;
},
45_000,
() => ({
requests: activeReceiver.capturedRequests,
spans: activeReceiver.capturedSpans.map((span) => ({
attributes: span.attributes,
name: span.name,
parentSpanId: span.parentSpanId,
spanId: span.spanId,
statusCode: span.statusCode,
traceId: span.traceId,
})),
}),
);
const failureSpansById = indexSpansById(failureEvidence.sameTrace);
expect(failureEvidence.terminal.parentSpanId).toBeFalsy();
for (const harness of failureEvidence.harnesses) {
expect(expectResolvedParent(harness, failureSpansById)).toBe(failureEvidence.terminal);
}
for (const run of failureEvidence.runs) {
expect(expectResolvedParent(run, failureSpansById).name).toBe("openclaw.harness.run");
}
for (const modelCall of failureEvidence.modelCalls) {
expect(expectResolvedParent(modelCall, failureSpansById).name).toBe("openclaw.run");
}
expect(expectResolvedParent(failureEvidence.toolError, failureSpansById).name).toBe(
"openclaw.run",
);
const successEvidence = activeReceiver.capturedSpans.find(
(span) =>
span.name === "openclaw.tool.execution" &&
span.statusCode !== 2 &&
span.attributes["openclaw.toolName"] === "read" &&
span.traceId !== failureEvidence.toolError.traceId,
);
expect(successEvidence).toBeTruthy();
const successTrace = activeReceiver.capturedSpans.filter(
(span) => span.traceId === successEvidence?.traceId,
);
const successTerminal = successTrace.find(
(span) =>
span.name === "openclaw.message.processed" &&
span.attributes["openclaw.channel"] === "qa-channel" &&
span.attributes["openclaw.outcome"] === "completed",
);
const successHarnesses = successTrace.filter((span) => span.name === "openclaw.harness.run");
const successRuns = successTrace.filter((span) => span.name === "openclaw.run");
const successModelCalls = successTrace.filter((span) => span.name === "openclaw.model.call");
expect(successTerminal).toBeDefined();
expect(successHarnesses.length).toBeGreaterThanOrEqual(1);
expect(successRuns.length).toBeGreaterThanOrEqual(1);
expect(successModelCalls.length).toBeGreaterThanOrEqual(1);
const successSpansById = indexSpansById(successTrace);
expect(successTerminal?.parentSpanId).toBeFalsy();
for (const harness of successHarnesses) {
expect(expectResolvedParent(harness, successSpansById)).toBe(successTerminal);
}
for (const run of successRuns) {
expect(expectResolvedParent(run, successSpansById).name).toBe("openclaw.harness.run");
}
for (const modelCall of successModelCalls) {
expect(expectResolvedParent(modelCall, successSpansById).name).toBe("openclaw.run");
}
expect(expectResolvedParent(successEvidence!, successSpansById).name).toBe("openclaw.run");
} finally {
await settleCleanup(
async () => {
await gateway?.stop();
},
async () => {
await mock?.stop();
},
async () => {
await receiver?.close();
},
async () => {
await bus?.stop();
},
);
}
}, 120_000);
});
@@ -0,0 +1,756 @@
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { Socket } from "node:net";
import { gunzipSync } from "node:zlib";
export type OtlpSignal = "logs" | "metrics" | "traces";
type OtlpAnyValue = {
stringValue?: string;
boolValue?: boolean;
intValue?: number | string | { toString(): string };
doubleValue?: number;
arrayValue?: { values?: OtlpAnyValue[] };
kvlistValue?: { values?: OtlpKeyValue[] };
bytesValue?: Uint8Array;
};
type OtlpKeyValue = {
key?: string;
value?: OtlpAnyValue;
};
type OtlpSpan = {
attributes?: OtlpKeyValue[];
endTimeMs?: number;
name?: string;
parentSpanId?: Uint8Array;
spanId?: Uint8Array;
statusCode?: number;
traceId?: Uint8Array;
};
type OtlpScopeSpans = {
spans?: OtlpSpan[];
};
type OtlpResourceSpans = {
scopeSpans?: OtlpScopeSpans[];
};
export type CapturedRequest = {
bytes: number;
contentEncoding?: string;
logCount: number;
metricCount: number;
path: string;
receivedAtMs?: number;
signal: OtlpSignal;
spanCount: number;
status: number;
};
export type CapturedSpan = {
attributes: Record<string, string | number | boolean | string[]>;
endTimeMs?: number;
name: string;
parent: boolean;
parentSpanId?: string;
spanId?: string;
statusCode?: number;
traceId?: string;
};
export type CapturedMetric = {
name: string;
};
export type CapturedLogRecord = {
body: string | number | boolean | string[];
spanId: string;
traceId: string;
};
const OTLP_SIGNAL_PATHS = new Map<string, OtlpSignal>([
["/v1/traces", "traces"],
["/v1/metrics", "metrics"],
["/v1/logs", "logs"],
]);
const POSITIVE_INTEGER_PATTERN = /^[1-9]\d*$/u;
const MAX_OTLP_COMPRESSED_BODY_BYTES = readPositiveIntegerEnv(
"OPENCLAW_QA_OTEL_MAX_COMPRESSED_BODY_BYTES",
2 * 1024 * 1024,
);
const MAX_OTLP_DECODED_BODY_BYTES = readPositiveIntegerEnv(
"OPENCLAW_QA_OTEL_MAX_DECODED_BODY_BYTES",
8 * 1024 * 1024,
);
const MAX_CAPTURED_BODY_TEXT_BYTES = readPositiveIntegerEnv(
"OPENCLAW_QA_OTEL_MAX_CAPTURED_BODY_TEXT_BYTES",
512 * 1024,
);
export function readPositiveIntegerEnv(
name: string,
fallback: number,
env: NodeJS.ProcessEnv = process.env,
): number {
const raw = env[name];
if (raw == null || raw.trim() === "") {
return fallback;
}
const value = raw.trim();
if (!POSITIVE_INTEGER_PATTERN.test(value)) {
throw new Error(`${name} must be a positive integer`);
}
const parsed = Number(value);
if (!Number.isSafeInteger(parsed)) {
throw new Error(`${name} must be a safe integer`);
}
return parsed;
}
function oversizedBodyError(label: string, actualBytes: number, maxBytes: number): Error {
const error = new Error(`${label} exceeded ${maxBytes} bytes: ${actualBytes} bytes`) as Error & {
statusCode?: number;
};
error.statusCode = 413;
return error;
}
export async function readRequestBody(
req: IncomingMessage,
maxBytes = MAX_OTLP_COMPRESSED_BODY_BYTES,
): Promise<Buffer> {
const chunks: Buffer[] = [];
let totalBytes = 0;
for await (const chunk of req) {
const buffer = Buffer.from(chunk);
totalBytes += buffer.length;
if (totalBytes > maxBytes) {
req.destroy();
throw oversizedBodyError("compressed OTLP request body", totalBytes, maxBytes);
}
chunks.push(buffer);
}
return Buffer.concat(chunks);
}
function headerValue(value: string | string[] | undefined): string | undefined {
return Array.isArray(value) ? value[0] : value;
}
export function decodeRequestBody(
body: Buffer,
contentEncoding: string | undefined,
maxBytes = MAX_OTLP_DECODED_BODY_BYTES,
): Buffer {
const normalizedEncoding = contentEncoding?.trim().toLowerCase();
if (body.length > maxBytes && (!normalizedEncoding || normalizedEncoding === "identity")) {
throw oversizedBodyError("OTLP request body", body.length, maxBytes);
}
if (!normalizedEncoding || normalizedEncoding === "identity") {
return body;
}
if (normalizedEncoding === "gzip") {
let decoded: Buffer;
try {
decoded = gunzipSync(body, { maxOutputLength: maxBytes });
} catch (error) {
const code = (error as { code?: unknown }).code;
const message = error instanceof Error ? error.message : String(error);
if (code === "ERR_BUFFER_TOO_LARGE" || /maxOutputLength|larger than/u.test(message)) {
throw oversizedBodyError("decoded OTLP request body", maxBytes + 1, maxBytes);
}
throw error;
}
if (decoded.length > maxBytes) {
throw oversizedBodyError("decoded OTLP request body", decoded.length, maxBytes);
}
return decoded;
}
throw new Error(`unsupported OTLP content-encoding ${contentEncoding}`);
}
export function appendCapturedBodyText(
capturedBodyText: Partial<Record<OtlpSignal, string[]>>,
signal: OtlpSignal,
body: Buffer,
maxBytes = MAX_CAPTURED_BODY_TEXT_BYTES,
disallowedNeedles: string[] = [],
): void {
const currentEntries = capturedBodyText[signal] ?? [];
const leakEntries = currentEntries.filter((entry) => entry.startsWith("[detected leak needle] "));
const currentTail = currentEntries
.filter((entry) => !entry.startsWith("[detected leak needle] "))
.join("\n");
const bodyText = body.toString("utf8");
const next = currentTail ? `${currentTail}\n${bodyText}` : bodyText;
const buffer = Buffer.from(next);
const nextLeakEntries = [
...leakEntries,
...disallowedNeedles
.filter((needle) => bodyText.includes(needle))
.map((needle) => `[detected leak needle] ${needle}`),
].slice(-20);
const tailEntry =
buffer.length > maxBytes
? `[captured body text truncated to last ${maxBytes} bytes]\n${buffer
.subarray(buffer.length - maxBytes)
.toString("utf8")}`
: next;
capturedBodyText[signal] = [...nextLeakEntries, tailEntry];
}
function normalizeOtlpValue(value: OtlpAnyValue | undefined): string | number | boolean | string[] {
if (!value) {
return "";
}
if (typeof value.stringValue === "string") {
return value.stringValue;
}
if (typeof value.boolValue === "boolean") {
return value.boolValue;
}
if (typeof value.doubleValue === "number") {
return value.doubleValue;
}
if (value.intValue !== undefined) {
return Number(value.intValue.toString());
}
if (value.arrayValue?.values) {
return value.arrayValue.values.map((entry) => String(normalizeOtlpValue(entry)));
}
if (value.kvlistValue?.values) {
return value.kvlistValue.values
.map((entry) => `${entry.key ?? ""}=${String(normalizeOtlpValue(entry.value))}`)
.filter(Boolean);
}
if (value.bytesValue) {
return Buffer.from(value.bytesValue).toString("hex");
}
return "";
}
function spanAttributes(span: OtlpSpan): Record<string, string | number | boolean | string[]> {
const attributes: Record<string, string | number | boolean | string[]> = {};
for (const attribute of span.attributes ?? []) {
const key = attribute.key?.trim();
if (!key) {
continue;
}
attributes[key] = normalizeOtlpValue(attribute.value);
}
return attributes;
}
class ProtoReader {
private offset = 0;
constructor(private readonly buffer: Uint8Array) {}
done(): boolean {
return this.offset >= this.buffer.length;
}
tag() {
const raw = this.varint();
return { field: raw >>> 3, wire: raw & 0x7 };
}
varint(): number {
let result = 0;
let shift = 0;
while (this.offset < this.buffer.length) {
const byte = this.buffer.at(this.offset);
if (byte === undefined) {
throw new Error("truncated protobuf varint");
}
this.offset += 1;
result += (byte & 0x7f) * 2 ** shift;
if ((byte & 0x80) === 0) {
return result;
}
shift += 7;
}
throw new Error("truncated protobuf varint");
}
bytes(): Uint8Array {
const length = this.varint();
const end = this.offset + length;
if (end > this.buffer.length) {
throw new Error("truncated protobuf bytes");
}
const value = this.buffer.subarray(this.offset, end);
this.offset = end;
return value;
}
string(): string {
return new TextDecoder().decode(this.bytes());
}
private advance(length: number, label: string): number {
const start = this.offset;
const end = this.offset + length;
if (end > this.buffer.length) {
throw new Error(`truncated protobuf ${label}`);
}
this.offset = end;
return start;
}
fixed64Float(): number {
const start = this.advance(8, "fixed64");
const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + start, 8);
return view.getFloat64(0, true);
}
fixed64Uint(): bigint {
const start = this.advance(8, "fixed64");
const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + start, 8);
return view.getBigUint64(0, true);
}
skip(wire: number): void {
if (wire === 0) {
this.varint();
} else if (wire === 1) {
this.advance(8, "fixed64");
} else if (wire === 2) {
this.bytes();
} else if (wire === 5) {
this.advance(4, "fixed32");
} else {
throw new Error(`unsupported protobuf wire type ${wire}`);
}
}
}
function decodeAnyValue(message: Uint8Array): OtlpAnyValue {
const reader = new ProtoReader(message);
const value: OtlpAnyValue = {};
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
value.stringValue = reader.string();
} else if (field === 2 && wire === 0) {
value.boolValue = reader.varint() !== 0;
} else if (field === 3 && wire === 0) {
value.intValue = reader.varint();
} else if (field === 4 && wire === 1) {
value.doubleValue = reader.fixed64Float();
} else if (field === 5 && wire === 2) {
value.arrayValue = decodeArrayValue(reader.bytes());
} else if (field === 6 && wire === 2) {
value.kvlistValue = decodeKeyValueList(reader.bytes());
} else if (field === 7 && wire === 2) {
value.bytesValue = reader.bytes();
} else {
reader.skip(wire);
}
}
return value;
}
function decodeArrayValue(message: Uint8Array): { values?: OtlpAnyValue[] } {
const reader = new ProtoReader(message);
const values: OtlpAnyValue[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
values.push(decodeAnyValue(reader.bytes()));
} else {
reader.skip(wire);
}
}
return { values };
}
function decodeKeyValue(message: Uint8Array): OtlpKeyValue {
const reader = new ProtoReader(message);
const entry: OtlpKeyValue = {};
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
entry.key = reader.string();
} else if (field === 2 && wire === 2) {
entry.value = decodeAnyValue(reader.bytes());
} else {
reader.skip(wire);
}
}
return entry;
}
function decodeKeyValueList(message: Uint8Array): { values?: OtlpKeyValue[] } {
const reader = new ProtoReader(message);
const values: OtlpKeyValue[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
values.push(decodeKeyValue(reader.bytes()));
} else {
reader.skip(wire);
}
}
return { values };
}
function decodeStatus(message: Uint8Array): number {
const reader = new ProtoReader(message);
let code = 0;
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 3 && wire === 0) {
code = reader.varint();
} else {
reader.skip(wire);
}
}
return code;
}
function decodeSpan(message: Uint8Array): OtlpSpan {
const reader = new ProtoReader(message);
const span: OtlpSpan = {};
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
span.traceId = reader.bytes();
} else if (field === 2 && wire === 2) {
span.spanId = reader.bytes();
} else if (field === 4 && wire === 2) {
span.parentSpanId = reader.bytes();
} else if (field === 5 && wire === 2) {
span.name = reader.string();
} else if (field === 8 && wire === 1) {
span.endTimeMs = Number(reader.fixed64Uint() / 1_000_000n);
} else if (field === 9 && wire === 2) {
span.attributes ??= [];
span.attributes.push(decodeKeyValue(reader.bytes()));
} else if (field === 15 && wire === 2) {
span.statusCode = decodeStatus(reader.bytes());
} else {
reader.skip(wire);
}
}
return span;
}
function decodeScopeSpans(message: Uint8Array): OtlpScopeSpans {
const reader = new ProtoReader(message);
const spans: OtlpSpan[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 2 && wire === 2) {
spans.push(decodeSpan(reader.bytes()));
} else {
reader.skip(wire);
}
}
return { spans };
}
function decodeResourceSpans(message: Uint8Array): OtlpResourceSpans {
const reader = new ProtoReader(message);
const scopeSpans: OtlpScopeSpans[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 2 && wire === 2) {
scopeSpans.push(decodeScopeSpans(reader.bytes()));
} else {
reader.skip(wire);
}
}
return { scopeSpans };
}
function decodeTraceRequest(body: Buffer): CapturedSpan[] {
const reader = new ProtoReader(body);
const resourceSpans: OtlpResourceSpans[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
resourceSpans.push(decodeResourceSpans(reader.bytes()));
} else {
reader.skip(wire);
}
}
const spans: CapturedSpan[] = [];
for (const resource of resourceSpans) {
for (const scopeSpans of resource.scopeSpans ?? []) {
for (const span of scopeSpans.spans ?? []) {
const name = span.name?.trim();
if (!name) {
continue;
}
spans.push({
attributes: spanAttributes(span),
endTimeMs: span.endTimeMs,
name,
parent: (span.parentSpanId?.length ?? 0) > 0,
parentSpanId: span.parentSpanId ? Buffer.from(span.parentSpanId).toString("hex") : "",
spanId: span.spanId ? Buffer.from(span.spanId).toString("hex") : "",
statusCode: span.statusCode ?? 0,
traceId: span.traceId ? Buffer.from(span.traceId).toString("hex") : "",
});
}
}
}
return spans;
}
function decodeMetric(message: Uint8Array): CapturedMetric | undefined {
const reader = new ProtoReader(message);
let name = "";
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
name = reader.string();
} else {
reader.skip(wire);
}
}
const normalizedName = name.trim();
return normalizedName ? { name: normalizedName } : undefined;
}
function decodeScopeMetrics(message: Uint8Array): CapturedMetric[] {
const reader = new ProtoReader(message);
const metrics: CapturedMetric[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 2 && wire === 2) {
const metric = decodeMetric(reader.bytes());
if (metric) {
metrics.push(metric);
}
} else {
reader.skip(wire);
}
}
return metrics;
}
function decodeResourceMetrics(message: Uint8Array): CapturedMetric[] {
const reader = new ProtoReader(message);
const metrics: CapturedMetric[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 2 && wire === 2) {
metrics.push(...decodeScopeMetrics(reader.bytes()));
} else {
reader.skip(wire);
}
}
return metrics;
}
function decodeMetricRequest(body: Buffer): CapturedMetric[] {
const reader = new ProtoReader(body);
const metrics: CapturedMetric[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
metrics.push(...decodeResourceMetrics(reader.bytes()));
} else {
reader.skip(wire);
}
}
return metrics;
}
function decodeLogRecord(message: Uint8Array): CapturedLogRecord {
const reader = new ProtoReader(message);
let body: string | number | boolean | string[] = "";
let traceId = "";
let spanId = "";
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 5 && wire === 2) {
body = normalizeOtlpValue(decodeAnyValue(reader.bytes()));
} else if (field === 9 && wire === 2) {
traceId = Buffer.from(reader.bytes()).toString("hex");
} else if (field === 10 && wire === 2) {
spanId = Buffer.from(reader.bytes()).toString("hex");
} else {
reader.skip(wire);
}
}
return { body, spanId, traceId };
}
function decodeScopeLogs(message: Uint8Array): CapturedLogRecord[] {
const reader = new ProtoReader(message);
const records: CapturedLogRecord[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 2 && wire === 2) {
records.push(decodeLogRecord(reader.bytes()));
} else {
reader.skip(wire);
}
}
return records;
}
function decodeResourceLogs(message: Uint8Array): CapturedLogRecord[] {
const reader = new ProtoReader(message);
const records: CapturedLogRecord[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 2 && wire === 2) {
records.push(...decodeScopeLogs(reader.bytes()));
} else {
reader.skip(wire);
}
}
return records;
}
function decodeLogRequest(body: Buffer): CapturedLogRecord[] {
const reader = new ProtoReader(body);
const records: CapturedLogRecord[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
records.push(...decodeResourceLogs(reader.bytes()));
} else {
reader.skip(wire);
}
}
return records;
}
function closeLocalOtlpReceiverConnections(
server: ReturnType<typeof createServer>,
sockets: Set<Socket>,
): void {
for (const socket of sockets) {
socket.destroy();
}
server.closeAllConnections();
}
export function startLocalOtlpReceiver(disallowedBodyNeedles: string[] = []) {
const capturedRequests: CapturedRequest[] = [];
const capturedSpans: CapturedSpan[] = [];
const capturedMetrics: CapturedMetric[] = [];
const capturedLogRecords: CapturedLogRecord[] = [];
const capturedBodyText: Partial<Record<OtlpSignal, string[]>> = {};
const sockets = new Set<Socket>();
const server = createServer((req: IncomingMessage, res: ServerResponse) => {
void (async () => {
if (req.method !== "POST" || !req.url) {
res.writeHead(404, { "content-type": "text/plain" });
res.end("not found");
return;
}
const requestPath = req.url;
const signal = OTLP_SIGNAL_PATHS.get(requestPath);
if (!signal) {
res.writeHead(404, { "content-type": "text/plain" });
res.end("not found");
return;
}
const contentEncoding = headerValue(req.headers["content-encoding"]);
let body: Buffer;
try {
body = decodeRequestBody(await readRequestBody(req), contentEncoding);
} catch (error) {
const statusCode =
typeof (error as { statusCode?: unknown }).statusCode === "number"
? (error as { statusCode: number }).statusCode
: 400;
capturedRequests.push({
path: requestPath,
signal,
bytes: 0,
contentEncoding,
status: statusCode,
spanCount: 0,
metricCount: 0,
logCount: 0,
});
res.writeHead(statusCode, { "content-type": "text/plain" });
res.end(error instanceof Error ? error.message : String(error));
return;
}
let spans: CapturedSpan[];
let metrics: CapturedMetric[];
let logRecords: CapturedLogRecord[];
try {
spans = signal === "traces" ? decodeTraceRequest(body) : [];
metrics = signal === "metrics" ? decodeMetricRequest(body) : [];
logRecords = signal === "logs" ? decodeLogRequest(body) : [];
appendCapturedBodyText(capturedBodyText, signal, body, undefined, disallowedBodyNeedles);
} catch (error) {
appendCapturedBodyText(capturedBodyText, signal, body, undefined, disallowedBodyNeedles);
capturedRequests.push({
path: requestPath,
signal,
bytes: body.length,
contentEncoding,
status: 400,
spanCount: 0,
metricCount: 0,
logCount: 0,
});
res.writeHead(400, { "content-type": "text/plain" });
res.end(error instanceof Error ? error.message : String(error));
return;
}
capturedSpans.push(...spans);
capturedMetrics.push(...metrics);
capturedLogRecords.push(...logRecords);
capturedRequests.push({
path: requestPath,
signal,
bytes: body.length,
contentEncoding,
receivedAtMs: Date.now(),
status: 200,
spanCount: spans.length,
metricCount: metrics.length,
logCount: logRecords.length,
});
res.writeHead(200, { "content-type": "application/x-protobuf" });
res.end();
})();
});
server.on("connection", (socket) => {
sockets.add(socket);
socket.once("close", () => {
sockets.delete(socket);
});
});
let closePromise: Promise<void> | undefined;
return {
capturedRequests,
capturedSpans,
capturedMetrics,
capturedLogRecords,
capturedBodyText,
async listen(): Promise<number> {
await new Promise<void>((resolve) => {
server.listen(0, "127.0.0.1", resolve);
});
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("failed to bind local OTLP receiver");
}
return address.port;
},
async close(): Promise<void> {
closePromise ??= new Promise<void>((resolve, reject) => {
closeLocalOtlpReceiverConnections(server, sockets);
server.close((error) => (error ? reject(error) : resolve()));
closeLocalOtlpReceiverConnections(server, sockets);
});
await closePromise;
},
};
}
+13 -736
View File
@@ -4,12 +4,11 @@ import { spawn } from "node:child_process";
/* oxlint-disable typescript/unbound-method -- the original stream method is invoked with process.stdout through Reflect.apply below. */
import { randomUUID } from "node:crypto";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { createServer } from "node:http";
import { Socket } from "node:net";
import { tmpdir } from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { gunzipSync } from "node:zlib";
import {
createDiagnosticTraceContext,
emitTrustedDiagnosticEvent,
@@ -21,42 +20,23 @@ import {
type OpenClawPluginServiceContext,
} from "../../../../extensions/diagnostics-otel/runtime-api.js";
import { onTrustedInternalDiagnosticEvent } from "../../../../src/infra/diagnostic-events.js";
import {
appendCapturedBodyText,
type CapturedLogRecord,
type CapturedMetric,
type CapturedRequest,
type CapturedSpan,
decodeRequestBody,
type OtlpSignal,
readPositiveIntegerEnv,
readRequestBody,
startLocalOtlpReceiver,
} from "./otel-test-support.js";
import { createQaScriptEvidenceWriter } from "./script-evidence.js";
type CollectorMode = "local" | "docker";
type OtelLogsExporter = "otlp" | "stdout" | "both";
type OtlpAnyValue = {
stringValue?: string;
boolValue?: boolean;
intValue?: number | string | { toString(): string };
doubleValue?: number;
arrayValue?: { values?: OtlpAnyValue[] };
kvlistValue?: { values?: OtlpKeyValue[] };
bytesValue?: Uint8Array;
};
type OtlpKeyValue = {
key?: string;
value?: OtlpAnyValue;
};
type OtlpSpan = {
name?: string;
parentSpanId?: Uint8Array;
attributes?: OtlpKeyValue[];
};
type OtlpScopeSpans = {
spans?: OtlpSpan[];
};
type OtlpResourceSpans = {
scopeSpans?: OtlpScopeSpans[];
};
type OtlpSignal = "logs" | "metrics" | "traces";
type CliOptions = {
collectorMode: CollectorMode;
logsExporter: OtelLogsExporter;
@@ -71,33 +51,6 @@ type OtelSmokeEvidenceContext = {
let activeEvidenceContext: OtelSmokeEvidenceContext | undefined;
type CapturedRequest = {
path: string;
signal: OtlpSignal;
bytes: number;
contentEncoding?: string;
status: number;
spanCount: number;
metricCount: number;
logCount: number;
};
type CapturedSpan = {
name: string;
parent: boolean;
attributes: Record<string, string | number | boolean | string[]>;
};
type CapturedMetric = {
name: string;
};
type CapturedLogRecord = {
body: string | number | boolean | string[];
spanId: string;
traceId: string;
};
type StdoutDiagnosticLogRecord = {
signal: "openclaw.diagnostic.log";
ts?: unknown;
@@ -114,11 +67,6 @@ type StdoutDiagnosticLogRecord = {
const DEFAULT_DOCKER_COLLECTOR_IMAGE =
process.env.OPENCLAW_QA_OTEL_COLLECTOR_IMAGE || "otel/opentelemetry-collector:0.104.0";
const OTLP_SIGNAL_PATHS = new Map<string, OtlpSignal>([
["/v1/traces", "traces"],
["/v1/metrics", "metrics"],
["/v1/logs", "logs"],
]);
const REQUIRED_SPAN_NAMES = [
"openclaw.run",
"openclaw.harness.run",
@@ -154,19 +102,6 @@ const DISALLOWED_BODY_NEEDLES = [
DIRECT_CALL_ID,
];
const COLLECTOR_OUTPUT_TAIL_BYTES = 16_000;
const POSITIVE_INTEGER_PATTERN = /^[1-9]\d*$/u;
const MAX_OTLP_COMPRESSED_BODY_BYTES = readPositiveIntegerEnv(
"OPENCLAW_QA_OTEL_MAX_COMPRESSED_BODY_BYTES",
2 * 1024 * 1024,
);
const MAX_OTLP_DECODED_BODY_BYTES = readPositiveIntegerEnv(
"OPENCLAW_QA_OTEL_MAX_DECODED_BODY_BYTES",
8 * 1024 * 1024,
);
const MAX_CAPTURED_BODY_TEXT_BYTES = readPositiveIntegerEnv(
"OPENCLAW_QA_OTEL_MAX_CAPTURED_BODY_TEXT_BYTES",
512 * 1024,
);
const MAX_STDOUT_DIAGNOSTIC_LINE_BYTES = readPositiveIntegerEnv(
"OPENCLAW_QA_OTEL_MAX_STDOUT_DIAGNOSTIC_LINE_BYTES",
512 * 1024,
@@ -191,42 +126,10 @@ const QA_OTEL_ENV_TO_CLEAR = [
"OTEL_RESOURCE_ATTRIBUTES",
] as const;
function readPositiveIntegerEnv(
name: string,
fallback: number,
env: NodeJS.ProcessEnv = process.env,
): number {
const raw = env[name];
if (raw == null || raw.trim() === "") {
return fallback;
}
const value = raw.trim();
if (!POSITIVE_INTEGER_PATTERN.test(value)) {
throw new Error(`${name} must be a positive integer`);
}
const parsed = Number(value);
if (!Number.isSafeInteger(parsed)) {
throw new Error(`${name} must be a safe integer`);
}
return parsed;
}
function createOtelSmokeRunId(): string {
return `${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`;
}
function oversizedBodyError(
label: string,
actualBytes: number,
maxBytes: number,
): Error & {
statusCode: number;
} {
return Object.assign(new Error(`${label} exceeded ${maxBytes} bytes: ${actualBytes} bytes`), {
statusCode: 413,
});
}
function usage(): string {
return `Usage: pnpm qa:otel:smoke [--collector local|docker] [--logs-exporter otlp|stdout|both] [--output-dir <path>]
@@ -299,632 +202,6 @@ function disallowedBodyNeedles(): string[] {
return [...DISALLOWED_BODY_NEEDLES];
}
async function readRequestBody(
req: IncomingMessage,
maxBytes = MAX_OTLP_COMPRESSED_BODY_BYTES,
): Promise<Buffer> {
const chunks: Buffer[] = [];
let totalBytes = 0;
for await (const chunk of req) {
const buffer = Buffer.from(chunk);
totalBytes += buffer.length;
if (totalBytes > maxBytes) {
req.destroy();
throw oversizedBodyError("compressed OTLP request body", totalBytes, maxBytes);
}
chunks.push(buffer);
}
return Buffer.concat(chunks);
}
function headerValue(value: string | string[] | undefined): string | undefined {
return Array.isArray(value) ? value[0] : value;
}
function decodeRequestBody(
body: Buffer,
contentEncoding: string | undefined,
maxBytes = MAX_OTLP_DECODED_BODY_BYTES,
): Buffer {
const normalizedEncoding = contentEncoding?.trim().toLowerCase();
if (body.length > maxBytes && (!normalizedEncoding || normalizedEncoding === "identity")) {
throw oversizedBodyError("OTLP request body", body.length, maxBytes);
}
if (!normalizedEncoding || normalizedEncoding === "identity") {
return body;
}
if (normalizedEncoding === "gzip") {
let decoded: Buffer;
try {
decoded = gunzipSync(body, { maxOutputLength: maxBytes });
} catch (error) {
const code = (error as { code?: unknown }).code;
const message = error instanceof Error ? error.message : String(error);
if (code === "ERR_BUFFER_TOO_LARGE" || /maxOutputLength|larger than/u.test(message)) {
throw oversizedBodyError("decoded OTLP request body", maxBytes + 1, maxBytes);
}
throw error;
}
if (decoded.length > maxBytes) {
throw oversizedBodyError("decoded OTLP request body", decoded.length, maxBytes);
}
return decoded;
}
throw new Error(`unsupported OTLP content-encoding ${contentEncoding}`);
}
function appendCapturedBodyText(
capturedBodyText: Partial<Record<OtlpSignal, string[]>>,
signal: OtlpSignal,
body: Buffer,
maxBytes = MAX_CAPTURED_BODY_TEXT_BYTES,
disallowedNeedles: string[] = [],
): void {
const currentEntries = capturedBodyText[signal] ?? [];
const leakEntries = currentEntries.filter((entry) => entry.startsWith("[detected leak needle] "));
const currentTail = currentEntries
.filter((entry) => !entry.startsWith("[detected leak needle] "))
.join("\n");
const bodyText = body.toString("utf8");
const next = currentTail ? `${currentTail}\n${bodyText}` : bodyText;
const buffer = Buffer.from(next);
const nextLeakEntries = [
...leakEntries,
...disallowedNeedles
.filter((needle) => bodyText.includes(needle))
.map((needle) => `[detected leak needle] ${needle}`),
].slice(-20);
const tailEntry =
buffer.length > maxBytes
? `[captured body text truncated to last ${maxBytes} bytes]\n${buffer
.subarray(buffer.length - maxBytes)
.toString("utf8")}`
: next;
capturedBodyText[signal] = [...nextLeakEntries, tailEntry];
}
function normalizeOtlpValue(value: OtlpAnyValue | undefined): string | number | boolean | string[] {
if (!value) {
return "";
}
if (typeof value.stringValue === "string") {
return value.stringValue;
}
if (typeof value.boolValue === "boolean") {
return value.boolValue;
}
if (typeof value.doubleValue === "number") {
return value.doubleValue;
}
if (value.intValue !== undefined) {
return Number(value.intValue.toString());
}
if (value.arrayValue?.values) {
return value.arrayValue.values.map((entry) => String(normalizeOtlpValue(entry)));
}
if (value.kvlistValue?.values) {
return value.kvlistValue.values
.map((entry) => `${entry.key ?? ""}=${String(normalizeOtlpValue(entry.value))}`)
.filter(Boolean);
}
if (value.bytesValue) {
return Buffer.from(value.bytesValue).toString("hex");
}
return "";
}
function spanAttributes(span: OtlpSpan): Record<string, string | number | boolean | string[]> {
const attributes: Record<string, string | number | boolean | string[]> = {};
for (const attribute of span.attributes ?? []) {
const key = attribute.key?.trim();
if (!key) {
continue;
}
attributes[key] = normalizeOtlpValue(attribute.value);
}
return attributes;
}
class ProtoReader {
private readonly buffer: Uint8Array;
private offset = 0;
constructor(buffer: Uint8Array) {
this.buffer = buffer;
}
done(): boolean {
return this.offset >= this.buffer.length;
}
tag() {
const raw = this.varint();
return { field: raw >>> 3, wire: raw & 0x7 };
}
varint(): number {
let result = 0;
let shift = 0;
while (this.offset < this.buffer.length) {
const byte = this.buffer.at(this.offset);
if (byte === undefined) {
throw new Error("truncated protobuf varint");
}
this.offset += 1;
result += (byte & 0x7f) * 2 ** shift;
if ((byte & 0x80) === 0) {
return result;
}
shift += 7;
}
throw new Error("truncated protobuf varint");
}
bytes(): Uint8Array {
const length = this.varint();
const end = this.offset + length;
if (end > this.buffer.length) {
throw new Error("truncated protobuf bytes");
}
const value = this.buffer.subarray(this.offset, end);
this.offset = end;
return value;
}
string(): string {
return new TextDecoder().decode(this.bytes());
}
private advance(length: number, label: string): number {
const start = this.offset;
const end = this.offset + length;
if (end > this.buffer.length) {
throw new Error(`truncated protobuf ${label}`);
}
this.offset = end;
return start;
}
fixed64(): number {
const start = this.advance(8, "fixed64");
const view = new DataView(this.buffer.buffer, this.buffer.byteOffset + start, 8);
return view.getFloat64(0, true);
}
skip(wire: number) {
if (wire === 0) {
this.varint();
} else if (wire === 1) {
this.advance(8, "fixed64");
} else if (wire === 2) {
this.bytes();
} else if (wire === 5) {
this.advance(4, "fixed32");
} else {
throw new Error(`unsupported protobuf wire type ${wire}`);
}
}
}
function decodeAnyValue(message: Uint8Array): OtlpAnyValue {
const reader = new ProtoReader(message);
const value: OtlpAnyValue = {};
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
value.stringValue = reader.string();
} else if (field === 2 && wire === 0) {
value.boolValue = reader.varint() !== 0;
} else if (field === 3 && wire === 0) {
value.intValue = reader.varint();
} else if (field === 4 && wire === 1) {
value.doubleValue = reader.fixed64();
} else if (field === 5 && wire === 2) {
value.arrayValue = decodeArrayValue(reader.bytes());
} else if (field === 6 && wire === 2) {
value.kvlistValue = decodeKeyValueList(reader.bytes());
} else if (field === 7 && wire === 2) {
value.bytesValue = reader.bytes();
} else {
reader.skip(wire);
}
}
return value;
}
function decodeArrayValue(message: Uint8Array): { values?: OtlpAnyValue[] } {
const reader = new ProtoReader(message);
const values: OtlpAnyValue[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
values.push(decodeAnyValue(reader.bytes()));
} else {
reader.skip(wire);
}
}
return { values };
}
function decodeKeyValue(message: Uint8Array): OtlpKeyValue {
const reader = new ProtoReader(message);
const entry: OtlpKeyValue = {};
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
entry.key = reader.string();
} else if (field === 2 && wire === 2) {
entry.value = decodeAnyValue(reader.bytes());
} else {
reader.skip(wire);
}
}
return entry;
}
function decodeKeyValueList(message: Uint8Array): { values?: OtlpKeyValue[] } {
const reader = new ProtoReader(message);
const values: OtlpKeyValue[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
values.push(decodeKeyValue(reader.bytes()));
} else {
reader.skip(wire);
}
}
return { values };
}
function decodeSpan(message: Uint8Array): OtlpSpan {
const reader = new ProtoReader(message);
const span: OtlpSpan = {};
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 4 && wire === 2) {
span.parentSpanId = reader.bytes();
} else if (field === 5 && wire === 2) {
span.name = reader.string();
} else if (field === 9 && wire === 2) {
span.attributes ??= [];
span.attributes.push(decodeKeyValue(reader.bytes()));
} else {
reader.skip(wire);
}
}
return span;
}
function decodeScopeSpans(message: Uint8Array): OtlpScopeSpans {
const reader = new ProtoReader(message);
const spans: OtlpSpan[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 2 && wire === 2) {
spans.push(decodeSpan(reader.bytes()));
} else {
reader.skip(wire);
}
}
return { spans };
}
function decodeResourceSpans(message: Uint8Array): OtlpResourceSpans {
const reader = new ProtoReader(message);
const scopeSpans: OtlpScopeSpans[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 2 && wire === 2) {
scopeSpans.push(decodeScopeSpans(reader.bytes()));
} else {
reader.skip(wire);
}
}
return { scopeSpans };
}
function decodeTraceRequest(body: Buffer): CapturedSpan[] {
const reader = new ProtoReader(body);
const resourceSpans: OtlpResourceSpans[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
resourceSpans.push(decodeResourceSpans(reader.bytes()));
} else {
reader.skip(wire);
}
}
const spans: CapturedSpan[] = [];
for (const resource of resourceSpans) {
for (const scopeSpans of resource.scopeSpans ?? []) {
for (const span of scopeSpans.spans ?? []) {
const name = span.name?.trim();
if (!name) {
continue;
}
spans.push({
name,
parent: (span.parentSpanId?.length ?? 0) > 0,
attributes: spanAttributes(span),
});
}
}
}
return spans;
}
function decodeMetric(message: Uint8Array): CapturedMetric | undefined {
const reader = new ProtoReader(message);
let name = "";
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
name = reader.string();
} else {
reader.skip(wire);
}
}
const normalizedName = name.trim();
return normalizedName ? { name: normalizedName } : undefined;
}
function decodeScopeMetrics(message: Uint8Array): CapturedMetric[] {
const reader = new ProtoReader(message);
const metrics: CapturedMetric[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 2 && wire === 2) {
const metric = decodeMetric(reader.bytes());
if (metric) {
metrics.push(metric);
}
} else {
reader.skip(wire);
}
}
return metrics;
}
function decodeResourceMetrics(message: Uint8Array): CapturedMetric[] {
const reader = new ProtoReader(message);
const metrics: CapturedMetric[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 2 && wire === 2) {
metrics.push(...decodeScopeMetrics(reader.bytes()));
} else {
reader.skip(wire);
}
}
return metrics;
}
function decodeMetricRequest(body: Buffer): CapturedMetric[] {
const reader = new ProtoReader(body);
const metrics: CapturedMetric[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
metrics.push(...decodeResourceMetrics(reader.bytes()));
} else {
reader.skip(wire);
}
}
return metrics;
}
function decodeLogRecord(message: Uint8Array): CapturedLogRecord {
const reader = new ProtoReader(message);
let body: string | number | boolean | string[] = "";
let traceId = "";
let spanId = "";
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 5 && wire === 2) {
body = normalizeOtlpValue(decodeAnyValue(reader.bytes()));
} else if (field === 9 && wire === 2) {
traceId = Buffer.from(reader.bytes()).toString("hex");
} else if (field === 10 && wire === 2) {
spanId = Buffer.from(reader.bytes()).toString("hex");
} else {
reader.skip(wire);
}
}
return { body, spanId, traceId };
}
function decodeScopeLogs(message: Uint8Array): CapturedLogRecord[] {
const reader = new ProtoReader(message);
const records: CapturedLogRecord[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 2 && wire === 2) {
records.push(decodeLogRecord(reader.bytes()));
} else {
reader.skip(wire);
}
}
return records;
}
function decodeResourceLogs(message: Uint8Array): CapturedLogRecord[] {
const reader = new ProtoReader(message);
const records: CapturedLogRecord[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 2 && wire === 2) {
records.push(...decodeScopeLogs(reader.bytes()));
} else {
reader.skip(wire);
}
}
return records;
}
function decodeLogRequest(body: Buffer): CapturedLogRecord[] {
const reader = new ProtoReader(body);
const records: CapturedLogRecord[] = [];
while (!reader.done()) {
const { field, wire } = reader.tag();
if (field === 1 && wire === 2) {
records.push(...decodeResourceLogs(reader.bytes()));
} else {
reader.skip(wire);
}
}
return records;
}
function startLocalOtlpReceiver(disallowedBodyNeedlesLocal: string[] = []) {
const capturedRequests: CapturedRequest[] = [];
const capturedSpans: CapturedSpan[] = [];
const capturedMetrics: CapturedMetric[] = [];
const capturedLogRecords: CapturedLogRecord[] = [];
const capturedBodyText: Partial<Record<OtlpSignal, string[]>> = {};
const sockets = new Set<Socket>();
const server = createServer((req: IncomingMessage, res: ServerResponse) => {
void (async () => {
if (req.method !== "POST" || !req.url) {
res.writeHead(404, { "content-type": "text/plain" });
res.end("not found");
return;
}
const requestPath = req.url;
const signal = OTLP_SIGNAL_PATHS.get(requestPath);
if (!signal) {
res.writeHead(404, { "content-type": "text/plain" });
res.end("not found");
return;
}
const contentEncoding = headerValue(req.headers["content-encoding"]);
let body: Buffer;
try {
const compressedBody = await readRequestBody(req);
body = decodeRequestBody(compressedBody, contentEncoding);
} catch (error) {
const statusCode =
typeof (error as { statusCode?: unknown }).statusCode === "number"
? (error as { statusCode: number }).statusCode
: 400;
capturedRequests.push({
path: requestPath,
signal,
bytes: 0,
contentEncoding,
status: statusCode,
spanCount: 0,
metricCount: 0,
logCount: 0,
});
res.writeHead(statusCode, { "content-type": "text/plain" });
res.end(error instanceof Error ? error.message : String(error));
return;
}
let spans: CapturedSpan[];
let metrics: CapturedMetric[];
let logRecords: CapturedLogRecord[];
try {
spans = signal === "traces" ? decodeTraceRequest(body) : [];
metrics = signal === "metrics" ? decodeMetricRequest(body) : [];
logRecords = signal === "logs" ? decodeLogRequest(body) : [];
appendCapturedBodyText(
capturedBodyText,
signal,
body,
undefined,
disallowedBodyNeedlesLocal,
);
} catch (error) {
appendCapturedBodyText(
capturedBodyText,
signal,
body,
undefined,
disallowedBodyNeedlesLocal,
);
capturedRequests.push({
path: requestPath,
signal,
bytes: body.length,
contentEncoding,
status: 400,
spanCount: 0,
metricCount: 0,
logCount: 0,
});
res.writeHead(400, { "content-type": "text/plain" });
res.end(error instanceof Error ? error.message : String(error));
return;
}
if (spans.length > 0) {
capturedSpans.push(...spans);
}
if (metrics.length > 0) {
capturedMetrics.push(...metrics);
}
if (logRecords.length > 0) {
capturedLogRecords.push(...logRecords);
}
capturedRequests.push({
path: requestPath,
signal,
bytes: body.length,
contentEncoding,
status: 200,
spanCount: spans.length,
metricCount: metrics.length,
logCount: logRecords.length,
});
res.writeHead(200, { "content-type": "application/x-protobuf" });
res.end();
})();
});
server.on("connection", (socket) => {
sockets.add(socket);
socket.once("close", () => {
sockets.delete(socket);
});
});
let closePromise: Promise<void> | undefined;
return {
capturedRequests,
capturedSpans,
capturedMetrics,
capturedLogRecords,
capturedBodyText,
async listen(): Promise<number> {
await new Promise<void>((resolve) => {
server.listen(0, "127.0.0.1", resolve);
});
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("failed to bind local OTLP receiver");
}
return address.port;
},
async close(): Promise<void> {
closePromise ??= new Promise<void>((resolve, reject) => {
closeLocalOtlpReceiverConnections(server, sockets);
server.close((err) => (err ? reject(err) : resolve()));
closeLocalOtlpReceiverConnections(server, sockets);
});
await closePromise;
},
};
}
function closeLocalOtlpReceiverConnections(
server: ReturnType<typeof createServer>,
sockets: Set<Socket>,
): void {
for (const socket of sockets) {
socket.destroy();
}
server.closeAllConnections();
}
async function reserveLocalPort(): Promise<number> {
const server = createServer();
await new Promise<void>((resolve) => {
@@ -148,6 +148,7 @@ describe("local-heavy-check-runtime", () => {
"--checkers",
"1",
]);
expect(env.GOMAXPROCS).toBe("1");
expect(env.GOGC).toBe("30");
expect(env.GOMEMLIMIT).toBe("3GiB");
});
@@ -162,6 +163,7 @@ describe("local-heavy-check-runtime", () => {
const { args, env } = applyLocalTsgoPolicy(
["--checkers", "4", "--singleThreaded", "--pprofDir", "/tmp/existing"],
makeEnv({
GOMAXPROCS: "3",
GOGC: "80",
GOMEMLIMIT: "5GiB",
OPENCLAW_TSGO_PPROF_DIR: "/tmp/profile",
@@ -178,6 +180,7 @@ describe("local-heavy-check-runtime", () => {
"--declaration",
"false",
]);
expect(env.GOMAXPROCS).toBe("3");
expect(env.GOGC).toBe("80");
expect(env.GOMEMLIMIT).toBe("5GiB");
});
@@ -201,6 +204,7 @@ describe("local-heavy-check-runtime", () => {
"--tsBuildInfoFile",
".artifacts/tsgo-cache/root.tsbuildinfo",
]);
expect(env.GOMAXPROCS).toBeUndefined();
expect(env.GOGC).toBeUndefined();
expect(env.GOMEMLIMIT).toBeUndefined();
});
@@ -253,6 +257,7 @@ describe("local-heavy-check-runtime", () => {
"--checkers",
"1",
]);
expect(env.GOMAXPROCS).toBe("1");
expect(env.GOGC).toBe("30");
expect(env.GOMEMLIMIT).toBe("3GiB");
});
@@ -273,6 +278,7 @@ describe("local-heavy-check-runtime", () => {
"--tsBuildInfoFile",
".artifacts/tsgo-cache/root.tsbuildinfo",
]);
expect(env.GOMAXPROCS).toBeUndefined();
expect(env.GOGC).toBeUndefined();
expect(env.GOMEMLIMIT).toBeUndefined();
});