From c4ae2be947c76d01a82cfad17b7679c12ecc123f Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Thu, 18 Jun 2026 15:16:58 -0700 Subject: [PATCH] fix: taxonomy coverage id cleanup (#94304) * fix: split taxonomy coverage id features * fix: clean taxonomy feature row names * docs: clarify taxonomy coverage id semantics * docs: tighten coverage id guidance * fix: keep taxonomy features product shaped * fix: narrow sdk artifact coverage bundle * fix: name taxonomy coverage ids clearly * fix: polish taxonomy feature descriptions --- .agents/skills/claw-score/SKILL.md | 6 ++ docs/concepts/qa-e2e-automation.md | 2 + qa/scenarios/index.yaml | 3 + taxonomy.yaml | 140 ++++++++++++++--------------- 4 files changed, 78 insertions(+), 73 deletions(-) diff --git a/.agents/skills/claw-score/SKILL.md b/.agents/skills/claw-score/SKILL.md index 3ed1a7014eb6..1177d704c066 100644 --- a/.agents/skills/claw-score/SKILL.md +++ b/.agents/skills/claw-score/SKILL.md @@ -31,6 +31,12 @@ out of this repo. If a score needs private evidence, use the redacted - `taxonomy.yaml` is the hand-edited source of truth for surfaces, levels, QA profiles, categories, feature coverage IDs, docs refs, LTS overrides, and completeness-instruction paths. +- Feature `coverageIds` are ANDed proof targets, not aliases. A feature may + list multiple IDs when each ID proves part of one capability. +- Keep categories and feature names unique, product-shaped, and broader than raw + coverage IDs. Do not promote generic IDs into standalone feature names. +- Avoid duplicate coverage-ID bundles under different feature names in one + category. - `docs/maturity-scores.yaml` is the aggregate score source committed in this repo. It is the only committed score data; do not add generated inventory directories. diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index c831e409bf7f..9a5377e19571 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -57,6 +57,8 @@ the resolved scenarios through `qa suite`. `--surface` and The resulting `qa-evidence.json` includes a profile scorecard summary with selected-category counts and missing coverage IDs; the individual evidence entries remain the source of truth for the tests, coverage roles, and results. +Taxonomy feature coverage IDs are exact proof targets, not aliases. Primary +scenario coverage fulfills matching IDs; secondary coverage stays advisory. Slim evidence omits per-entry `execution` and sets `evidenceMode: "slim"`; `smoke-ci` defaults to slim, and `--evidence-mode full` restores full entries: diff --git a/qa/scenarios/index.yaml b/qa/scenarios/index.yaml index 8ca79dc1a781..96457549abe7 100644 --- a/qa/scenarios/index.yaml +++ b/qa/scenarios/index.yaml @@ -19,6 +19,9 @@ title: OpenClaw QA Scenario Pack # - add `coverage.secondary` only when a scenario intentionally protects another behavior # - keep IDs behavior-shaped, broad enough to reuse, lowercase, and dotted or dashed # - use the exact values listed under feature `coverageIds` in `taxonomy.yaml` +# - taxonomy feature coverage IDs are exact proof targets, not aliases +# - scenario primary can list multiple IDs only when this scenario is primary +# proof for each # - prefer reusing an existing coverage ID over minting a scenario-shaped ID # - avoid copying the scenario title into coverage IDs # - use `pnpm openclaw qa coverage` to render the current inventory diff --git a/taxonomy.yaml b/taxonomy.yaml index 9e9038e18487..468a48c3748a 100644 --- a/taxonomy.yaml +++ b/taxonomy.yaml @@ -1335,13 +1335,13 @@ surfaces: coverageIds: [sandboxed-exec-behavior] description: Exec behavior, sandbox roots, and workspace constraints visible to operators. - name: Approval flow - coverageIds: [personal.approval-denial, personal.tool-safety, runtime.approvals, tools.followthrough, tools.safety] + coverageIds: [personal.approval-denial, runtime.approvals, tools.followthrough] description: Operator approval gates for tool execution. - name: Elevated execution coverageIds: [elevated-execution] description: Elevated host execution rules and related controls. - name: Tool safety controls - coverageIds: [personal.approval-denial, personal.tool-safety, runtime.approvals, tools.followthrough, tools.safety] + coverageIds: [personal.tool-safety, tools.safety] description: Before-tool-call hooks and related guardrails that shape operator-visible tool behavior. - name: Delegated tool access coverageIds: [delegated-tool-access] @@ -2318,10 +2318,10 @@ surfaces: coverageIds: [channels.qa-channel, personal.reminders, scheduling.cron] description: Covers Agent cron tool across cron job creation, listing, inspection, editing, and related cron job lifecycle behavior. - name: Manual cron runs - coverageIds: [channels.qa-channel, personal.reminders, scheduling.cron, scheduling.dedup] + coverageIds: [scheduling.cron] description: Covers Manual cron runs across scheduler dispatch, timer arming, manual/due runs, isolated agent execution, and related cron runs and diagnostics behavior. - name: Isolated cron execution - coverageIds: [channels.qa-channel, personal.reminders, scheduling.cron, scheduling.dedup] + coverageIds: [scheduling.cron, scheduling.dedup] description: Covers Isolated cron execution across scheduler dispatch, timer arming, manual/due runs, isolated agent execution, and related cron runs and diagnostics behavior. - name: Model/provider preflight coverageIds: [model-provider-preflight] @@ -2842,9 +2842,9 @@ surfaces: - name: Image generation tool invocation coverageIds: [channels.qa-channel, media.image-generation, tools.image-generate, tools.native-image-generation] description: Covers Image generation tool invocation across `image_generate`, model selection, provider registration, provider capability listing, and related image generation tool and provider routing behavior. - - name: Provider and model selection + - name: Image generation provider routing coverageIds: [media.image-generation, tools.native-image-generation] - description: Covers Provider and model selection across `image_generate`, model selection, provider registration, provider capability listing, and related image generation tool and provider routing behavior. + description: Covers image generation provider and model routing across `image_generate`, provider registration, provider capability listing, and related image generation tool behavior. - name: Reference image editing coverageIds: [reference-image-editing] description: Covers Reference image editing across `image_generate`, model selection, provider registration, provider capability listing, and related image generation tool and provider routing behavior. @@ -2857,9 +2857,9 @@ surfaces: - name: Music generation tool invocation coverageIds: [music-generation-tool-invocation] description: Covers Music generation tool invocation across `music_generate`, provider/model config, lyrics/instrumental/duration/format controls, image reference inputs where supported, and related music generation tool and provider routing behavior. - - name: Provider and model selection - coverageIds: [provider-and-model-selection-2] - description: Covers Provider and model selection across `music_generate`, provider/model config, lyrics/instrumental/duration/format controls, image reference inputs where supported, and related music generation tool and provider routing behavior. + - name: Music generation provider controls + coverageIds: [music-generation-provider-controls] + description: Covers music generation provider and model controls across `music_generate`, provider/model config, lyrics/instrumental/duration/format controls, image reference inputs where supported, and related music generation routing behavior. - name: Lyrics, instrumental, duration, and format controls coverageIds: [lyrics-instrumental-duration-and-format-controls] description: Covers Lyrics, instrumental, duration, and format controls across `music_generate`, provider/model config, lyrics/instrumental/duration/format controls, image reference inputs where supported, and related music generation tool and provider routing behavior. @@ -2898,7 +2898,7 @@ surfaces: - docs/tools/video-generation.md search_anchors: - Image generation tool invocation - - Provider and model selection + - Image generation provider routing - Reference image editing - Generated image task lifecycle - Generated image persistence and delivery @@ -2906,6 +2906,7 @@ surfaces: - Provider + CLI fallback - Mention detection in groups - Music generation tool invocation + - Music generation provider controls - Lyrics, instrumental, duration, and format controls - Reference inputs where supported - Music task lifecycle and duplicate status @@ -3690,15 +3691,15 @@ surfaces: - name: Catalog Discovery id: catalog-discovery features: - - name: openclaw plugins search as the ClawHub + - name: Plugin catalog search coverageIds: [openclaw-plugins-search-as-the-clawhub] - description: openclaw plugins search as the ClawHub plugin lookup command + description: OpenClaw plugin search resolves ClawHub package lookup results. - name: Search result metadata coverageIds: [search-result-metadata] - description: package name, family, channel, version, summary, and - - name: Distinction between plugin search + description: Search results include package identity, channel, version, summary, compatibility, and source metadata. + - name: Plugin and skill search separation coverageIds: [distinction-between-plugin-search] - description: Distinction between plugin search and skill search + description: Plugin search and skill search stay distinct even when both use ClawHub metadata. - name: Catalog lookup failure coverageIds: [catalog-lookup-failure] description: Catalog lookup failure and empty-result behavior @@ -3781,8 +3782,8 @@ surfaces: features: - name: Source prefixes coverageIds: [source-prefixes] - description: Source prefixes and shorthand resolution for clawhub:, npm:, - - name: Bare package behavior during the launch + description: Source prefixes and shorthand resolution cover ClawHub, npm, local, archive, and marketplace plugin sources. + - name: Bare package launch behavior coverageIds: [bare-package-behavior-during-the-launch] description: Bare package behavior during the launch cutover - name: Explicit pinned versions @@ -3818,7 +3819,7 @@ surfaces: - name: Uninstall config/index/policy/file cleanup coverageIds: [uninstall-config-index-policy-file-cleanup] description: Evidence scope for Uninstall config/index/policy/file cleanup. - - name: Gateway restart/reload requirements after + - name: Gateway restart and reload requirements coverageIds: [gateway-restart-reload-requirements-after] description: Gateway restart/reload requirements after install/update/uninstall - name: Per-plugin managed npm project @@ -3836,15 +3837,15 @@ surfaces: - name: Legacy dependency root cleanup coverageIds: [legacy-dependency-root-cleanup] description: Legacy dependency root cleanup and doctor repair - - name: plugins list + - name: Plugin inventory commands coverageIds: [plugins-list] - description: plugins list, plugins inspect, runtime inspect, plugins doctor, and + description: Plugin list, inspect, runtime inspect, and doctor commands expose operator inventory and diagnostics. - name: Local plugin index coverageIds: [local-plugin-index] description: Local plugin index and persisted cold registry state - name: Troubleshooting stale config coverageIds: [troubleshooting-stale-config] - description: Troubleshooting stale config, blocked paths, dependencies, missing plugins, + description: Troubleshooting covers stale config, blocked paths, dependencies, missing plugins, and repair guidance. - name: Runtime verification after Gateway coverageIds: [runtime-verification-after-gateway] description: Runtime verification after Gateway restart @@ -4029,9 +4030,9 @@ surfaces: - name: ToolSpace coverageIds: [toolspace] description: Tool discovery and invocation abstraction for external apps. - - name: Artifacts - coverageIds: [character.persona, personal.task-followthrough, tools.followthrough, workspace.artifacts, workspace.builds, workspace.long-running-task, workspace.repo-discovery] - description: Artifact summaries, retention metadata, and download behavior. + - name: Artifact resources + coverageIds: [workspace.artifacts, workspace.builds] + description: SDK helpers for workspace artifact summaries, generated build outputs, retention metadata, and downloads. - name: Tasks coverageIds: [tasks] description: SDK helpers around Gateway task APIs. @@ -6197,7 +6198,7 @@ surfaces: - name: Container update/rebuild/restart guidance for Docker coverageIds: [container-update-rebuild-restart-guidance-for-docker] description: Container update/rebuild/restart guidance for Docker and Podman hosts - - name: Docker Compose + - name: Docker Compose mounts and secrets coverageIds: [docker-compose] description: Docker Compose and Podman config/workspace/auth-profile secret mounts - name: Gateway token generation @@ -6206,8 +6207,8 @@ surfaces: - name: Ownership coverageIds: [ownership] description: Ownership, permissions, SELinux mount behavior, and state survival across container replacement - - name: Docker Compose - coverageIds: [docker-compose-2] + - name: Docker Compose network access + coverageIds: [docker-compose-network-access] description: Docker Compose and Podman port publishing, bind mode, host local provider access, Bonjour, Tailscale, and Control UI origins - name: Container health endpoints coverageIds: [container-health-endpoints] @@ -8500,85 +8501,77 @@ surfaces: - name: Channel Setup and Operations id: channel-setup-and-operations features: - - name: Voice Call Channel - coverageIds: [voice-call-channel] - description: Cli, Gateway Rpc, and Agent Tool - - name: Voice Call Channel - coverageIds: [voice-call-channel-2] + - name: Voice call CLI, RPC, and agent tool + coverageIds: [voice-call-cli-rpc-agent-tool] + description: CLI, Gateway RPC, and agent tool + - name: Voice call setup smoke + coverageIds: [voice-call-setup-smoke] description: Setup, Configuration, and Smoke docs: - docs/cli/voicecall.md - docs/plugins/voice-call.md - docs/gateway/protocol.md search_anchors: - - 'voice call channel voice call channel: cli, gateway rpc, and agent tool' - - 'voice call channel: cli, gateway rpc, and agent tool' - - 'voice call channel voice call channel: setup, configuration, and smoke' - - 'voice call channel: setup, configuration, and smoke' + - voice call CLI, RPC, and agent tool + - voice call setup smoke category_note: setup-configuration-and-smoke.md human_lts_override: false - name: Access and Identity id: access-and-identity features: - - name: Voice Call Channel - coverageIds: [voice-call-channel] + - name: Voice call webhook security + coverageIds: [voice-call-webhook-security] description: Webhook Exposure and Security docs: - docs/plugins/voice-call.md - docs/cli/voicecall.md search_anchors: - - 'voice call channel voice call channel: webhook exposure and security' - - 'voice call channel: webhook exposure and security' + - voice call webhook security category_note: webhook-exposure-and-security.md human_lts_override: false - name: Conversation Routing and Delivery id: conversation-routing-and-delivery features: - - name: Voice Call Channel - coverageIds: [voice-call-channel] + - name: Voice call inbound routing + coverageIds: [voice-call-inbound-routing] description: Inbound Routing, Sessions, and Lifecycle docs: - docs/plugins/voice-call.md search_anchors: - - 'voice call channel voice call channel: inbound routing, sessions, and lifecycle' - - 'voice call channel: inbound routing, sessions, and lifecycle' + - voice call inbound routing, sessions, and lifecycle category_note: inbound-routing-sessions-and-lifecycle.md human_lts_override: false - name: Media and Rich Content id: media-and-rich-content features: - - name: Voice Call Channel - coverageIds: [voice-call-channel] + - name: Voice call provider transports + coverageIds: [voice-call-provider-transports] description: Provider Transports and Call Control - - name: Voice Call Channel - coverageIds: [voice-call-channel-2] - description: Telephony Tts, Playback, Dtmf, and Audio + - name: Voice call telephony audio + coverageIds: [voice-call-telephony-audio] + description: Telephony TTS, playback, DTMF, and audio docs: - docs/plugins/voice-call.md - docs/plugins/plugin-inventory.md search_anchors: - - 'voice call channel voice call channel: provider transports and call control' - - 'voice call channel: provider transports and call control' - - 'voice call channel voice call channel: telephony tts, playback, dtmf, and audio' - - 'voice call channel: telephony tts, playback, dtmf, and audio' + - voice call provider transports and call control + - voice call telephony TTS, playback, DTMF, and audio category_note: provider-transports-and-call-control.md human_lts_override: false - name: Realtime Voice and Calls id: realtime-voice-and-calls features: - - name: Voice Call Channel - coverageIds: [voice-call-channel] + - name: Voice call realtime consult + coverageIds: [voice-call-realtime-consult] description: Realtime Voice and Agent Consult - - name: Voice Call Channel - coverageIds: [voice-call-channel-2] + - name: Voice call streaming transcription + coverageIds: [voice-call-streaming-transcription] description: Streaming Transcription and Auto-response docs: - docs/plugins/voice-call.md search_anchors: - - 'voice call channel voice call channel: realtime voice and agent consult' - - 'voice call channel: realtime voice and agent consult' - - 'voice call channel voice call channel: streaming transcription and auto-response' - - 'voice call channel: streaming transcription and auto-response' + - voice call realtime voice and agent consult + - voice call streaming transcription and auto-response category_note: realtime-voice-and-agent-consult.md human_lts_override: false last_score_run: @@ -8645,10 +8638,10 @@ surfaces: coverageIds: [codex-responses-transport] description: Covers Codex Responses Transport across low-level provider request/streaming path for `openai-codex-responses` and the shared OpenAI Responses conversion code used by direct OpenAI and Codex-auth compatibility routes. - name: Payload Compatibility - coverageIds: [runtime.codex-native-workspace.read, runtime.prompt-compatibility, tools.fs.read] + coverageIds: [runtime.prompt-compatibility, tools.fs.read] description: Covers Payload Compatibility across low-level provider request/streaming path for `openai-codex-responses` and the shared OpenAI Responses conversion code used by direct OpenAI and Codex-auth compatibility routes. - name: Tool Context - coverageIds: [runtime.codex-native-workspace.read, runtime.prompt-compatibility, tools.fs.read] + coverageIds: [runtime.codex-native-workspace.read, tools.fs.read] description: Covers Tool Context across provider-facing tools, context injection, media inputs, native-vs-client tool ownership, OpenAI Responses compatibility, and how OpenAI/Codex models receive OpenClaw runtime context. - name: Capability Compatibility coverageIds: [capability-compatibility] @@ -9080,15 +9073,15 @@ surfaces: - name: Usage and stop reasons coverageIds: [usage-and-stop-reasons] description: 'Covers Usage and stop reasons across direct `google-generative-ai` Gemini transport and shared Google message/stream conversion: request URL construction, request config, text/image/audio/video/tool payload conversion, function response handling, and related direct gemini api behavior.' - - name: Thought-signature replay - coverageIds: [thought-signature-replay] - description: 'Covers Thought-signature replay across direct `google-generative-ai` Gemini transport and shared Google message/stream conversion: request URL construction, request config, text/image/audio/video/tool payload conversion, function response handling, and related direct gemini api behavior.' + - name: Direct Gemini transport payloads + coverageIds: [direct-gemini-transport-payloads] + description: 'Covers direct `google-generative-ai` Gemini transport and shared Google message/stream conversion: request URL construction, request config, text/image/audio/video/tool payload conversion, function response handling, and related direct Gemini API behavior.' - name: Thinking-level mapping coverageIds: [thinking-level-mapping] description: Covers Thinking-level mapping across Gemini thinking-level mapping, adaptive thinking request shape, `thoughtSignature` capture/replay/sanitization, Google replay policy, and related thinking and turn recovery behavior. - - name: Thought-signature replay - coverageIds: [thought-signature-replay-2] - description: Covers Thought-signature replay across Gemini thinking-level mapping, adaptive thinking request shape, `thoughtSignature` capture/replay/sanitization, Google replay policy, and related thinking and turn recovery behavior. + - name: Thought-signature replay policy + coverageIds: [thought-signature-replay] + description: Covers Gemini thinking-level mapping, adaptive thinking request shape, `thoughtSignature` capture/replay/sanitization, Google replay policy, and related thinking and turn recovery behavior. - name: Tool turn ordering coverageIds: [tool-turn-ordering] description: Covers Tool turn ordering across Gemini thinking-level mapping, adaptive thinking request shape, `thoughtSignature` capture/replay/sanitization, Google replay policy, and related thinking and turn recovery behavior. @@ -9105,8 +9098,9 @@ surfaces: - Multimodal inputs - Tool-call streaming - Usage and stop reasons - - Thought-signature replay + - Direct Gemini transport payloads - Thinking-level mapping + - Thought-signature replay policy - Tool turn ordering - Incomplete-turn recovery category_note: direct-gemini-api-transport-streaming-and-multimodal-payloads.md @@ -10085,7 +10079,7 @@ surfaces: coverageIds: [snapshots] description: Covers Snapshots across browser tool action schemas, navigate/act/snapshot/screenshot operations, AI/role/ARIA snapshot formats, action ref storage, and related browser actions, snapshots, and artifacts behavior. - name: Artifacts - coverageIds: [character.persona, personal.task-followthrough, tools.followthrough, workspace.artifacts, workspace.builds, workspace.long-running-task, workspace.repo-discovery] + coverageIds: [workspace.artifacts] description: Covers Artifacts across browser tool action schemas, navigate/act/snapshot/screenshot operations, AI/role/ARIA snapshot formats, action ref storage, and related browser actions, snapshots, and artifacts behavior. - name: Browser Plugin Service coverageIds: [browser-plugin-service] @@ -10132,10 +10126,10 @@ surfaces: coverageIds: [tools.bash, tools.exec] description: Covers Exec Routing across `exec` foreground and background execution, `yieldMs`, timeouts, PTY, and related exec routing and process lifecycle behavior. - name: Process Lifecycle - coverageIds: [workspace.artifacts, workspace.builds, workspace.long-running-task, workspace.repo-discovery] + coverageIds: [workspace.long-running-task] description: Covers Process Lifecycle across `exec` foreground and background execution, `yieldMs`, timeouts, PTY, and related exec routing and process lifecycle behavior. - name: Direct Tool Invoke API - coverageIds: [plugins.mcp-tools, plugins.skills, tools.invocation] + coverageIds: [plugins.mcp-tools, tools.invocation] description: Covers Direct Tool Invoke API across HTTP `POST /tools/invoke`, Gateway RPC `tools.invoke`, request body and auth semantics, shared-secret operator scope restoration, and related direct tool invoke api and node system.run behavior. - name: Node System.run coverageIds: [node-system-run]