diff --git a/.github/workflows/maturity-scorecard.yml b/.github/workflows/maturity-scorecard.yml index 2d2286aef0f1..0c00c82b6b36 100644 --- a/.github/workflows/maturity-scorecard.yml +++ b/.github/workflows/maturity-scorecard.yml @@ -329,6 +329,7 @@ jobs: uses: ./.github/workflows/qa-profile-evidence.yml with: ref: ${{ needs.validate_selected_ref.outputs.selected_revision }} + trusted_ref: ${{ inputs.ref }} expected_sha: ${{ needs.validate_selected_ref.outputs.selected_revision }} qa_profile: all allow_failures: ${{ inputs.allow_failures }} diff --git a/.github/workflows/qa-profile-evidence.yml b/.github/workflows/qa-profile-evidence.yml index db93d8cb4317..a9d5ef32a0f4 100644 --- a/.github/workflows/qa-profile-evidence.yml +++ b/.github/workflows/qa-profile-evidence.yml @@ -10,6 +10,11 @@ on: required: true default: main type: string + trusted_ref: + description: Optional trusted branch, tag, or SHA identity for an immutable ref + required: false + default: "" + type: string expected_sha: description: Optional full SHA that ref must resolve to required: false @@ -31,6 +36,11 @@ on: description: OpenClaw branch, tag, or SHA to run required: true type: string + trusted_ref: + description: Optional trusted branch, tag, or SHA identity for an immutable ref + required: false + default: "" + type: string expected_sha: description: Optional full SHA that ref must resolve to required: false @@ -141,6 +151,7 @@ jobs: if: needs.authorize_actor.outputs.authorized == 'true' runs-on: blacksmith-8vcpu-ubuntu-2404 outputs: + protocol_base_revision: ${{ steps.validate.outputs.protocol_base_revision }} selected_revision: ${{ steps.validate.outputs.selected_revision }} trusted_reason: ${{ steps.validate.outputs.trusted_reason }} steps: @@ -155,13 +166,15 @@ jobs: id: validate env: EXPECTED_SHA: ${{ inputs.expected_sha }} - INPUT_REF: ${{ inputs.ref }} + INPUT_REF: ${{ inputs.trusted_ref || inputs.ref }} shell: bash run: | set -euo pipefail selected_revision="$(git rev-parse HEAD)" expected_sha="${EXPECTED_SHA,,}" + branch_candidate="${INPUT_REF#refs/heads/}" + tag_candidate="${INPUT_REF#refs/tags/}" trusted_reason="" if [[ -n "${expected_sha// }" && ! "$expected_sha" =~ ^[0-9a-f]{40}$ ]]; then @@ -175,16 +188,22 @@ jobs: timeout --signal=TERM --kill-after=10s 120s git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main - if git merge-base --is-ancestor "$selected_revision" refs/remotes/origin/main; then - trusted_reason="main-ancestor" - elif git tag --points-at "$selected_revision" | grep -Eq '^v'; then - trusted_reason="release-tag" - elif [[ "$INPUT_REF" =~ ^release/[0-9]{4}\.[0-9]+\.[0-9]+$ ]]; then - timeout --signal=TERM --kill-after=10s 120s git fetch --no-tags origin "+refs/heads/${INPUT_REF}:refs/remotes/origin/${INPUT_REF}" - release_branch_sha="$(git rev-parse "refs/remotes/origin/${INPUT_REF}")" + if [[ "$tag_candidate" =~ ^v ]]; then + timeout --signal=TERM --kill-after=10s 120s git fetch --no-tags origin "+refs/tags/${tag_candidate}:refs/tags/${tag_candidate}" + release_tag_sha="$(git rev-parse "refs/tags/${tag_candidate}^{commit}")" + if [[ "$selected_revision" == "$release_tag_sha" ]]; then + trusted_reason="release-tag" + fi + elif [[ "$branch_candidate" =~ ^release/[0-9]{4}\.[0-9]+\.[0-9]+$ ]]; then + timeout --signal=TERM --kill-after=10s 120s git fetch --no-tags origin "+refs/heads/${branch_candidate}:refs/remotes/origin/${branch_candidate}" + release_branch_sha="$(git rev-parse "refs/remotes/origin/${branch_candidate}")" if [[ "$selected_revision" == "$release_branch_sha" ]]; then trusted_reason="release-branch-head" fi + elif git merge-base --is-ancestor "$selected_revision" refs/remotes/origin/main; then + trusted_reason="main-ancestor" + elif git tag --points-at "$selected_revision" | grep -Eq '^v'; then + trusted_reason="release-tag" fi if [[ -z "$trusted_reason" ]]; then @@ -193,6 +212,25 @@ jobs: exit 1 fi + case "$trusted_reason" in + main-ancestor) + protocol_base_revision="$(git rev-parse "${selected_revision}^1")" + ;; + release-branch-head | release-tag) + protocol_base_revision="$(git merge-base "$selected_revision" refs/remotes/origin/main)" + ;; + *) + echo "Unsupported trusted ref classification: ${trusted_reason}" >&2 + exit 1 + ;; + esac + if [[ ! "$protocol_base_revision" =~ ^[0-9a-f]{40}$ ]]; then + echo "Protocol comparison base must resolve to a full commit SHA." >&2 + exit 1 + fi + git cat-file -e "${protocol_base_revision}^{commit}" + + echo "protocol_base_revision=$protocol_base_revision" >> "$GITHUB_OUTPUT" echo "selected_revision=$selected_revision" >> "$GITHUB_OUTPUT" echo "trusted_reason=$trusted_reason" >> "$GITHUB_OUTPUT" { @@ -201,6 +239,7 @@ jobs: echo "- Requested ref: \`${INPUT_REF}\`" echo "- Resolved SHA: \`$selected_revision\`" echo "- Trust reason: \`$trusted_reason\`" + echo "- Protocol base: \`$protocol_base_revision\`" } >> "$GITHUB_STEP_SUMMARY" run_qa_profile: @@ -229,6 +268,25 @@ jobs: ref: ${{ needs.validate_selected_ref.outputs.selected_revision }} fetch-depth: 1 + - name: Fetch protocol comparison base + env: + PROTOCOL_SINCE_BASE_SHA: ${{ needs.validate_selected_ref.outputs.protocol_base_revision }} + shell: bash + run: | + set -euo pipefail + if [[ ! "$PROTOCOL_SINCE_BASE_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "Protocol comparison base must be a full commit SHA." >&2 + exit 1 + fi + timeout --signal=TERM --kill-after=10s 120s git fetch --no-tags --no-recurse-submodules --depth=1 origin \ + "+${PROTOCOL_SINCE_BASE_SHA}:refs/remotes/origin/qa-protocol-base" + resolved_protocol_base="$(git rev-parse refs/remotes/origin/qa-protocol-base^{commit})" + if [[ "$resolved_protocol_base" != "$PROTOCOL_SINCE_BASE_SHA" ]]; then + echo "Protocol comparison base resolved to ${resolved_protocol_base}, expected ${PROTOCOL_SINCE_BASE_SHA}." >&2 + exit 1 + fi + git cat-file -e "${resolved_protocol_base}^{commit}" + - name: Setup Node environment uses: ./.github/actions/setup-node-env with: @@ -290,17 +348,21 @@ jobs: - name: Build private QA runtime env: NODE_OPTIONS: --max-old-space-size=8192 - run: node --import tsx scripts/build-all.mts qaRuntime + run: pnpm build qaRuntime - name: Ensure Playwright Chromium - run: node --import tsx scripts/ensure-playwright-chromium.mts + run: | + playwright_script="scripts/ensure-playwright-chromium.mts" + [[ -f "$playwright_script" ]] || playwright_script="scripts/ensure-playwright-chromium.mjs" + node --import tsx "$playwright_script" - name: Run QA profile id: run_profile env: QA_PROFILE: ${{ steps.profile.outputs.profile }} - REQUESTED_REF: ${{ inputs.ref }} + REQUESTED_REF: ${{ inputs.trusted_ref || inputs.ref }} TARGET_SHA: ${{ needs.validate_selected_ref.outputs.selected_revision }} + PROTOCOL_SINCE_BASE_SHA: ${{ needs.validate_selected_ref.outputs.protocol_base_revision }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} @@ -376,6 +438,7 @@ jobs: target: { ref: process.env.REQUESTED_REF, sha: process.env.TARGET_SHA, + protocolBaseSha: process.env.PROTOCOL_SINCE_BASE_SHA, }, profile: process.env.QA_PROFILE, run: { @@ -404,7 +467,8 @@ jobs: OUTPUT_DIR: ${{ steps.run_profile.outputs.output_dir }} QA_EXIT_CODE: ${{ steps.run_profile.outputs.qa_exit_code }} QA_PROFILE: ${{ steps.profile.outputs.profile }} - REQUESTED_REF: ${{ inputs.ref }} + PROTOCOL_BASE_SHA: ${{ needs.validate_selected_ref.outputs.protocol_base_revision }} + REQUESTED_REF: ${{ inputs.trusted_ref || inputs.ref }} TARGET_SHA: ${{ needs.validate_selected_ref.outputs.selected_revision }} TRUSTED_REASON: ${{ needs.validate_selected_ref.outputs.trusted_reason }} shell: bash @@ -458,6 +522,7 @@ jobs: allowFailures: process.env.ALLOW_FAILURES === "true", requestedRef: process.env.REQUESTED_REF, targetSha: process.env.TARGET_SHA, + protocolBaseSha: process.env.PROTOCOL_BASE_SHA, trustedReason: process.env.TRUSTED_REASON, evidenceMode: payload.evidenceMode, profilePlanSha256, @@ -494,6 +559,7 @@ jobs: echo "- QA exit code: \`${QA_EXIT_CODE}\`" echo "- QA failures allowed: \`${ALLOW_FAILURES}\`" echo "- Target SHA: \`${TARGET_SHA}\`" + echo "- Protocol base SHA: \`${PROTOCOL_BASE_SHA}\`" echo "- Evidence path: \`${OUTPUT_DIR}/qa-evidence.json\`" echo "- Manifest: \`${OUTPUT_DIR}/qa-profile-evidence-manifest.json\`" } >> "$GITHUB_STEP_SUMMARY" diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 4e9deb47ac58..4e2bd214c4cd 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -587,6 +587,7 @@ timeout_outcome="none"`, GITHUB_RUN_ID: "42", LC_ALL: "POSIX", PATH: fixturePath, + PROTOCOL_SINCE_BASE_SHA: "b".repeat(40), QA_PROFILE: "all", REQUESTED_REF: "fixture", SUPERVISOR_READY_FILE: path.join(root, "supervisor-ready"), @@ -599,6 +600,7 @@ timeout_outcome="none"`, readFileSync(path.join(outputDir, "qa-profile-run-status.json"), "utf8"), ) as { exitCode: number; + target: { protocolBaseSha: string }; timedOut: boolean; timeoutOutcome: "none" | "term" | "kill"; }; @@ -778,6 +780,164 @@ function writeExecutable(filePath: string, lines: string[]): void { chmodSync(filePath, 0o755); } +function writeProtocolDescriptor( + repo: string, + additions: Array<{ name: string; since?: string }> = [], +): void { + const rows = [{ name: "health", since: "2026.7" }, ...additions].map(({ name, since }) => { + const sinceProperty = since === undefined ? "" : `, since: ${JSON.stringify(since)}`; + return ` { name: ${JSON.stringify(name)}${sinceProperty} },`; + }); + const descriptor = path.join(repo, "src/gateway/methods/core-descriptors.ts"); + mkdirSync(path.dirname(descriptor), { recursive: true }); + writeFileSync( + descriptor, + `export const CORE_GATEWAY_METHOD_SPECS = [\n${rows.join("\n")}\n] as const;\n`, + ); +} + +function commitProtocolFixture(repo: string, message: string): string { + runGit(repo, ["add", "-A"]); + runGit(repo, ["commit", "-q", "-m", message]); + return runGit(repo, ["rev-parse", "HEAD"]); +} + +function createQaProtocolTopology() { + const root = tempDirs.make("openclaw-qa-protocol-topology-"); + const origin = path.join(root, "origin"); + const checkout = path.join(root, "checkout"); + const releaseBranch = "release/2026.8.1"; + const releaseTag = "v2026.8.1"; + const mainReleaseTag = "v2026.8.2"; + + runGit(root, ["init", "-q", "-b", "main", origin]); + runGit(origin, ["config", "commit.gpgsign", "false"]); + runGit(origin, ["config", "user.email", "qa-protocol@example.invalid"]); + runGit(origin, ["config", "user.name", "QA Protocol Fixture"]); + writeFileSync( + path.join(origin, "package.json"), + '{"name":"qa-protocol-fixture","version":"2026.8.0"}\n', + ); + writeProtocolDescriptor(origin); + const mainBase = commitProtocolFixture(origin, "base protocol"); + + writeProtocolDescriptor(origin, [{ name: "sessions.patchMany", since: "2026.8" }]); + const mainHead = commitProtocolFixture(origin, "add main protocol method"); + runGit(origin, ["tag", mainReleaseTag]); + writeFileSync(path.join(origin, "main-tip.txt"), "later main tip\n"); + commitProtocolFixture(origin, "advance main"); + + runGit(origin, ["checkout", "-q", "-b", releaseBranch, mainBase]); + writeProtocolDescriptor(origin, [{ name: "sessions.releaseOnly" }]); + const releaseHead = commitProtocolFixture(origin, "add release protocol method"); + + runGit(origin, ["checkout", "-q", "--detach", mainBase]); + writeFileSync(path.join(origin, "tag.txt"), "release tag\n"); + const releaseTagHead = commitProtocolFixture(origin, "create release tag target"); + runGit(origin, ["tag", releaseTag]); + + runGit(origin, ["checkout", "-q", "-b", "feature/untrusted", mainBase]); + writeFileSync(path.join(origin, "feature.txt"), "untrusted\n"); + const featureHead = commitProtocolFixture(origin, "add untrusted feature"); + runGit(origin, ["checkout", "-q", "main"]); + + runGit(root, ["clone", "-q", "--no-local", origin, checkout]); + const fakeBin = path.join(root, "bin"); + mkdirSync(fakeBin); + writeExecutable(path.join(fakeBin, "timeout"), ["#!/usr/bin/env bash", "shift 3", 'exec "$@"']); + + return { + checkout, + fakeBin, + featureHead, + mainBase, + mainHead, + mainReleaseTag, + origin, + releaseBranch, + releaseHead, + releaseTag, + releaseTagHead, + }; +} + +function readWorkflowOutputs(outputPath: string): Record { + if (!existsSync(outputPath)) { + return {}; + } + const output = readFileSync(outputPath, "utf8").trim(); + return output + ? Object.fromEntries( + output.split("\n").map((line) => { + const separator = line.indexOf("="); + return [line.slice(0, separator), line.slice(separator + 1)]; + }), + ) + : {}; +} + +function runQaSelectedRefValidation( + topology: ReturnType, + inputRef: string, + revision: string, + expectedSha = revision, +) { + runGit(topology.checkout, ["checkout", "-q", "--detach", revision]); + const githubOutput = path.join(topology.checkout, "github-output"); + rmSync(githubOutput, { force: true }); + const validateStep = expectDefined( + readQaProfileEvidenceWorkflow().jobs.validate_selected_ref.steps.find( + (step: WorkflowStep) => step.name === "Validate selected ref", + ), + "QA profile selected-ref validation step", + ); + const result = runWorkflowShellScript(expectDefined(validateStep.run, "validation script"), { + cwd: topology.checkout, + env: { + ...process.env, + EXPECTED_SHA: expectedSha, + GITHUB_OUTPUT: githubOutput, + GITHUB_STEP_SUMMARY: path.join(topology.checkout, "github-summary"), + INPUT_REF: inputRef, + PATH: `${topology.fakeBin}:${process.env.PATH ?? ""}`, + }, + }); + return { ...result, outputs: readWorkflowOutputs(githubOutput) }; +} + +function runProtocolSinceFixture(checkout: string, baseSha: string) { + for (const scriptPath of [ + "packages/normalization-core/src/record-coerce.ts", + "scripts/check-protocol-since.mts", + "scripts/lib/repo-root.mjs", + ]) { + const target = path.join(checkout, scriptPath); + mkdirSync(path.dirname(target), { recursive: true }); + writeFileSync(target, readFileSync(scriptPath, "utf8")); + } + writeFileSync( + path.join(checkout, "tsconfig.json"), + JSON.stringify({ + compilerOptions: { + paths: { + "@openclaw/normalization-core/record-coerce": [ + "./packages/normalization-core/src/record-coerce.ts", + ], + }, + }, + }), + ); + const nodeModules = path.join(checkout, "node_modules"); + if (!existsSync(nodeModules)) { + symlinkSync(path.resolve("node_modules"), nodeModules, "dir"); + } + return spawnSync(process.execPath, ["--import", "tsx", "scripts/check-protocol-since.mts"], { + cwd: checkout, + encoding: "utf8", + env: { ...process.env, PROTOCOL_SINCE_BASE_SHA: baseSha }, + }); +} + function runDependencyCheckFixture(options: { historicalTarget: boolean; scripts: string[] }): { calls: string[]; output: string; @@ -5863,6 +6023,152 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" }, ); + it.skipIf(process.platform === "win32")( + "resolves topology-aware protocol bases and drives the real guard", + () => { + const topology = createQaProtocolTopology(); + const cases = [ + ["main", topology.mainHead, "main-ancestor", topology.mainBase], + [topology.releaseBranch, topology.releaseHead, "release-branch-head", topology.mainBase], + [ + `refs/heads/${topology.releaseBranch}`, + topology.releaseHead, + "release-branch-head", + topology.mainBase, + ], + [topology.releaseTag, topology.releaseTagHead, "release-tag", topology.mainBase], + [topology.releaseTagHead, topology.releaseTagHead, "release-tag", topology.mainBase], + [topology.mainReleaseTag, topology.mainHead, "release-tag", topology.mainHead], + ] as const; + + for (const [inputRef, revision, trustedReason, protocolBase] of cases) { + const result = runQaSelectedRefValidation(topology, inputRef, revision); + expect(result.status, `${result.stdout}${result.stderr}`).toBe(0); + expect(result.outputs).toEqual({ + protocol_base_revision: protocolBase, + selected_revision: revision, + trusted_reason: trustedReason, + }); + } + + runGit(topology.checkout, ["checkout", "-q", "--detach", topology.mainHead]); + const mainCheck = runProtocolSinceFixture(topology.checkout, topology.mainBase); + expect(mainCheck.status, `${mainCheck.stdout}${mainCheck.stderr}`).toBe(0); + expect(mainCheck.stdout).toContain("1 new core method"); + + runGit(topology.checkout, ["checkout", "-q", "--detach", topology.releaseHead]); + const releaseCheck = runProtocolSinceFixture(topology.checkout, topology.mainBase); + expect(releaseCheck.status).not.toBe(0); + expect(releaseCheck.stderr).toContain("sessions.releaseOnly is missing since metadata"); + + for (const [expectedSha, inputRef, revision] of [ + ["not-a-sha", "main", topology.mainHead], + [topology.featureHead, topology.featureHead, topology.featureHead], + [topology.mainHead, topology.releaseTag, topology.releaseTagHead], + ] as const) { + const result = runQaSelectedRefValidation(topology, inputRef, revision, expectedSha); + expect(result.status, `${result.stdout}${result.stderr}`).not.toBe(0); + expect(result.outputs).toEqual({}); + } + }, + ); + + it.skipIf(process.platform === "win32")( + "wires and fetches one explicit protocol base before QA execution", + () => { + const qaWorkflow = readQaProfileEvidenceWorkflow(); + const maturityWorkflow = readMaturityScorecardWorkflow(); + const validateJob = qaWorkflow.jobs.validate_selected_ref; + const runJob = qaWorkflow.jobs.run_qa_profile; + const stepNames = runJob.steps.map((step: WorkflowStep) => step.name); + const buildStep = expectDefined( + runJob.steps.find((step: WorkflowStep) => step.name === "Build private QA runtime"), + "private QA runtime build", + ); + const fetchStep = expectDefined( + runJob.steps.find((step: WorkflowStep) => step.name === "Fetch protocol comparison base"), + "protocol comparison base fetch", + ); + const runStep = expectDefined( + runJob.steps.find((step: WorkflowStep) => step.name === "Run QA profile"), + "QA profile run", + ); + const evidenceStep = expectDefined( + runJob.steps.find((step: WorkflowStep) => step.name === "Validate QA profile evidence"), + "QA profile evidence validation", + ); + const protocolOutput = "${{ needs.validate_selected_ref.outputs.protocol_base_revision }}"; + const trustedInput = "${{ inputs.trusted_ref || inputs.ref }}"; + + expect(qaWorkflow.on.workflow_call.inputs.trusted_ref).toEqual({ + description: "Optional trusted branch, tag, or SHA identity for an immutable ref", + required: false, + default: "", + type: "string", + }); + expect(validateJob.outputs.protocol_base_revision).toBe( + "${{ steps.validate.outputs.protocol_base_revision }}", + ); + expect(validateJob.steps[1].env.INPUT_REF).toBe(trustedInput); + const ordered = [ + "Checkout selected ref", + "Fetch protocol comparison base", + "Setup Node environment", + "Build private QA runtime", + "Run QA profile", + ].map((name) => stepNames.indexOf(name)); + expect(ordered.every((index, position) => index > (ordered[position - 1] ?? -1))).toBe(true); + expect(fetchStep.env?.PROTOCOL_SINCE_BASE_SHA).toBe(protocolOutput); + expect(buildStep.run).toBe("pnpm build qaRuntime"); + expect(runStep.env?.PROTOCOL_SINCE_BASE_SHA).toBe(protocolOutput); + expect(runStep.env?.REQUESTED_REF).toBe(trustedInput); + expect(runStep.run).toContain("protocolBaseSha: process.env.PROTOCOL_SINCE_BASE_SHA"); + expect(evidenceStep.env?.PROTOCOL_BASE_SHA).toBe(protocolOutput); + expect(evidenceStep.env?.REQUESTED_REF).toBe(trustedInput); + expect(evidenceStep.run).toContain("protocolBaseSha: process.env.PROTOCOL_BASE_SHA"); + expect(maturityWorkflow.jobs.generate_qa_evidence.with.trusted_ref).toBe("${{ inputs.ref }}"); + + const topology = createQaProtocolTopology(); + const checkout = tempDirs.make("openclaw-qa-protocol-fetch-"); + runGit(checkout, ["init", "-q", "-b", "main"]); + runGit(checkout, ["remote", "add", "origin", topology.origin]); + runGit(checkout, [ + "fetch", + "-q", + "--depth=1", + "origin", + `+${topology.mainHead}:refs/remotes/origin/selected`, + ]); + runGit(checkout, ["checkout", "-q", "--detach", "refs/remotes/origin/selected"]); + const sentinel = path.join(checkout, "qa-sentinel"); + const runFetch = (baseSha: string) => + runWorkflowShellScript( + `${expectDefined(fetchStep.run, "protocol fetch script")}\nprintf 'ran\\n' > "$QA_SENTINEL"\n`, + { + cwd: checkout, + env: { + ...process.env, + PATH: `${topology.fakeBin}:${process.env.PATH ?? ""}`, + PROTOCOL_SINCE_BASE_SHA: baseSha, + QA_SENTINEL: sentinel, + }, + }, + ); + + const success = runFetch(topology.mainBase); + expect(success.status, `${success.stdout}${success.stderr}`).toBe(0); + expect(runGit(checkout, ["rev-parse", "refs/remotes/origin/qa-protocol-base"])).toBe( + topology.mainBase, + ); + expect(existsSync(sentinel)).toBe(true); + + rmSync(sentinel); + const failure = runFetch("f".repeat(40)); + expect(failure.status, `${failure.stdout}${failure.stderr}`).not.toBe(0); + expect(existsSync(sentinel)).toBe(false); + }, + ); + it("bounds QA profile selected-ref fetches", () => { const validateSelectedRef = expectDefined( readQaProfileEvidenceWorkflow().jobs.validate_selected_ref.steps.find( @@ -5874,12 +6180,13 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" .split("\n") .filter((line: string) => line.includes("git fetch")); - expect(gitFetchLines).toHaveLength(2); + expect(gitFetchLines).toHaveLength(3); expect( gitFetchLines.every((line: string) => line.trimStart().startsWith("timeout --signal=TERM --kill-after=10s 120s git fetch"), ), ).toBe(true); + expect(gitFetchLines.some((line: string) => line.includes("+refs/tags/"))).toBe(true); }); it.skipIf(process.platform !== "linux")( @@ -5928,6 +6235,7 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" expect(result.commandStatus, `${result.stdout}\n${result.stderr}`).toBe(0); expect(result.status).toMatchObject({ exitCode: scenario.exitCode, + target: { protocolBaseSha: "b".repeat(40) }, timedOut: scenario.timedOut, timeoutOutcome: scenario.timeoutOutcome, }); @@ -6062,15 +6370,17 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" const ensurePlaywrightStep = qaRunJob.steps.find( (step: WorkflowStep) => step.name === "Ensure Playwright Chromium", ); - expect(ensurePlaywrightStep.run).toBe( - "node --import tsx scripts/ensure-playwright-chromium.mts", - ); + expect(ensurePlaywrightStep.run).toContain("scripts/ensure-playwright-chromium.mts"); + expect(ensurePlaywrightStep.run).toContain("scripts/ensure-playwright-chromium.mjs"); const runProfileStep = qaRunJob.steps.find( (step: WorkflowStep) => step.name === "Run QA profile", ); expect(runProfileStep.env?.OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF).toBe("1"); expect(runProfileStep.env?.OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS).toBe("120000"); - expect(runProfileStep.env?.REQUESTED_REF).toBe("${{ inputs.ref }}"); + expect(runProfileStep.env?.PROTOCOL_SINCE_BASE_SHA).toBe( + "${{ needs.validate_selected_ref.outputs.protocol_base_revision }}", + ); + expect(runProfileStep.env?.REQUESTED_REF).toBe("${{ inputs.trusted_ref || inputs.ref }}"); expect(runProfileStep.env?.TARGET_SHA).toBe( "${{ needs.validate_selected_ref.outputs.selected_revision }}", ); @@ -6113,6 +6423,7 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" expect(runProfileStep.run).not.toContain('case "$qa_exit_code"'); expect(runProfileStep.run).toContain('TIMEOUT_OUTCOME="$timeout_outcome"'); expect(runProfileStep.run).toContain("qa-profile-run-status.json"); + expect(runProfileStep.run).toContain("protocolBaseSha: process.env.PROTOCOL_SINCE_BASE_SHA"); expect(runProfileStep.run).toContain("exitCode: Number(process.env.QA_EXIT_CODE)"); expect(runProfileStep.run).toContain('timedOut: timeoutOutcome !== "none"'); expect(runProfileStep.run).toContain("timeoutOutcome,"); @@ -6136,8 +6447,8 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" ); expect(generateJob.uses).toBe("./.github/workflows/qa-profile-evidence.yml"); expect(generateJob.with).toMatchObject({ - // Reusable jobs start later, so forward the immutable revision instead of a moving branch. ref: "${{ needs.validate_selected_ref.outputs.selected_revision }}", + trusted_ref: "${{ inputs.ref }}", expected_sha: "${{ needs.validate_selected_ref.outputs.selected_revision }}", qa_profile: "all", allow_failures: "${{ inputs.allow_failures }}", @@ -6297,10 +6608,15 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" expect(qaEvidenceStep.run).toContain("qaProfileEvidencePlan.attest"); expect(qaEvidenceStep.run).toContain("profilePlanSha256"); expect(qaEvidenceStep.run).toContain("rerun the QA Profile Evidence workflow"); + expect(qaEvidenceStep.env.PROTOCOL_BASE_SHA).toBe( + "${{ needs.validate_selected_ref.outputs.protocol_base_revision }}", + ); + expect(qaEvidenceStep.env.REQUESTED_REF).toBe("${{ inputs.trusted_ref || inputs.ref }}"); expect(qaEvidenceStep.env.ALLOW_FAILURES).toBe("${{ inputs.allow_failures }}"); expect(qaEvidenceStep.run).toContain("qaExitCode: Number(process.env.QA_EXIT_CODE)"); expect(qaEvidenceStep.run).toContain('qaPassed: process.env.QA_EXIT_CODE === "0"'); expect(qaEvidenceStep.run).toContain('allowFailures: process.env.ALLOW_FAILURES === "true"'); + expect(qaEvidenceStep.run).toContain("protocolBaseSha: process.env.PROTOCOL_BASE_SHA"); expect(qaEvidenceStep.run).toContain("QA failures allowed:"); const qaUploadStep = qaRunJob.steps.find( @@ -6446,6 +6762,7 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" const root = tempDirs.make("openclaw-qa-profile-artifact-"); const evidencePath = path.join(root, "qa-evidence.json"); const manifestPath = path.join(root, "qa-profile-evidence-manifest.json"); + const protocolBaseSha = "b".repeat(40); const targetSha = "a".repeat(40); const expectedCell = { scenarioId: "scenario-one", @@ -6531,6 +6848,7 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" GITHUB_OUTPUT: path.join(root, "github-output"), GITHUB_STEP_SUMMARY: path.join(root, "github-summary"), OUTPUT_DIR: root, + PROTOCOL_BASE_SHA: protocolBaseSha, QA_EXIT_CODE: qaExitCode, QA_PROFILE: "all", REQUESTED_REF: targetSha, @@ -6554,6 +6872,10 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" completeProducer.status, `${completeProducer.stdout}${completeProducer.stderr}`, ).toBe(0); + expect(JSON.parse(readFileSync(manifestPath, "utf8"))).toMatchObject({ + protocolBaseSha, + targetSha, + }); const completeConsumer = runConsumer(); expect( completeConsumer.status, diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index 0a9b9a0ef863..b6554a5dc266 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -1944,6 +1944,7 @@ describe("scripts/test-projects changed-target routing", () => { "scripts/lib/repo-root.mjs": [ "test/scripts/ts-guard-utils.test.ts", "test/scripts/android-release-signing.test.ts", + "test/scripts/ci-workflow-guards.test.ts", ], "scripts/lib/ts-guard-utils.mts": ["test/scripts/ts-guard-utils.test.ts"], "scripts/lib/tsgo-sparse-guard.mts": [