fix(ci): isolate performance candidate trust (#127160)

2026-08-25 20:05:46 -06:00 · 2026-08-21 07:54:56 -07:00
parent 03d4b8a9c3
commit 7833e242cb
2 changed files with 279 additions and 22 deletions
@@ -74,6 +74,46 @@ function kovaMatrixEntries(): Array<Record<string, string>> {
  return readWorkflow().jobs?.kova?.strategy?.matrix?.include ?? [];
 }

+function runCandidateTrustClassification({
+  candidateSha,
+  eventName,
+  ref,
+  workflowSha,
+}: {
+  candidateSha: string;
+  eventName: "schedule" | "workflow_dispatch";
+  ref: string;
+  workflowSha: string;
+}) {
+  const step = findStep("Classify performance candidate trust", "resolve_target");
+  const root = tempDirs.make("openclaw-performance-candidate-trust-");
+  const output = join(root, "output");
+  const result = spawnSync("bash", ["-c", step.run ?? ""], {
+    encoding: "utf8",
+    env: {
+      ...process.env,
+      CANDIDATE_SHA: candidateSha,
+      DEFAULT_BRANCH: "main",
+      GITHUB_EVENT_NAME: eventName,
+      GITHUB_OUTPUT: output,
+      GITHUB_REF: ref,
+      WORKFLOW_SHA: workflowSha,
+    },
+  });
+  const outputs = Object.fromEntries(
+    existsSync(output)
+      ? readFileSync(output, "utf8")
+          .trim()
+          .split("\n")
+          .map((line) => {
+            const separator = line.indexOf("=");
+            return [line.slice(0, separator), line.slice(separator + 1)];
+          })
+      : [],
+  );
+  return { outputs, result };
+}
+
 describe("OpenClaw performance workflow", () => {
  it("uses an optional dispatch identifier to name parent-owned runs", () => {
    const workflow = readFileSync(WORKFLOW, "utf8");
@@ -195,13 +235,13 @@ describe("OpenClaw performance workflow", () => {
      decideLane.run?.indexOf('echo "run=$run_lane"') ?? -1,
    );
    expect(configureLiveAuth.if).toBe(
-      "${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }}",
+      "${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' }}",
    );
    expect(runKova.env?.OPENAI_API_KEY).toBe(
-      "${{ matrix.live == 'true' && secrets.OPENAI_API_KEY || '' }}",
+      "${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_API_KEY || '' }}",
    );
    expect(runKova.env?.OPENAI_BASE_URL).toBe(
-      "${{ matrix.live == 'true' && secrets.OPENAI_BASE_URL || '' }}",
+      "${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_BASE_URL || '' }}",
    );

    try {
@@ -212,6 +252,7 @@ describe("OpenClaw performance workflow", () => {
          GITHUB_OUTPUT: output,
          KOVA_REF_TRUSTED_FOR_LIVE: "false",
          LANE_ID: "live-openai-candidate",
+          SECRET_ELIGIBLE: "true",
        },
      });
      expect(rejected.status).toBe(1);
@@ -226,6 +267,7 @@ describe("OpenClaw performance workflow", () => {
          GITHUB_OUTPUT: output,
          KOVA_REF_TRUSTED_FOR_LIVE: "true",
          LANE_ID: "live-openai-candidate",
+          SECRET_ELIGIBLE: "true",
        },
      });
      expect(accepted.status).toBe(0);
@@ -235,6 +277,157 @@ describe("OpenClaw performance workflow", () => {
    }
  });

+  it("keeps arbitrary performance candidates secretless and cacheless", () => {
+    const workflow = readWorkflow();
+    const trust = findStep("Classify performance candidate trust", "resolve_target");
+    const decideLane = findStep("Decide lane");
+    const kovaHarness = findStep("Checkout performance workflow helpers");
+    const kovaStage = findStep("Stage trusted setup action graph");
+    const kovaSetup = findStep("Set up Node environment");
+    const sourceHarness = findStep("Checkout source performance helpers", "source_performance");
+    const sourceStage = findStep("Stage trusted source setup action graph", "source_performance");
+    const sourceSetup = findStep("Set up source performance environment", "source_performance");
+    const publisherHarness = findStep("Checkout performance publisher helper", "publish");
+
+    expect(workflow.jobs?.resolve_target?.outputs).toMatchObject({
+      secret_eligible: "${{ steps.candidate_trust.outputs.secret_eligible }}",
+      cache_write_allowed: "${{ steps.candidate_trust.outputs.cache_write_allowed }}",
+    });
+    expect(trust.env).toMatchObject({
+      CANDIDATE_SHA: "${{ steps.resolve.outputs.tested_sha }}",
+      DEFAULT_BRANCH: "${{ github.event.repository.default_branch }}",
+      WORKFLOW_SHA: "${{ github.workflow_sha }}",
+    });
+    expect(trust.run).toContain("secret_eligible=false");
+    expect(trust.run).toContain("cache_write_allowed=false");
+    expect(trust.run).toContain('"$GITHUB_REF" == "refs/heads/${DEFAULT_BRANCH}"');
+    expect(trust.run).toContain('"$CANDIDATE_SHA" == "$WORKFLOW_SHA"');
+    expect(trust.run).toContain("secret_eligible=true");
+    expect(trust.run).toContain("cache_write_allowed=true");
+
+    for (const harness of [kovaHarness, sourceHarness, publisherHarness]) {
+      expect(harness.with?.ref).toBe("${{ github.workflow_sha }}");
+      expect(harness.with?.["persist-credentials"]).toBe(false);
+    }
+    for (const setup of [kovaSetup, sourceSetup]) {
+      expect(setup.uses).toBe("./.artifacts/performance-workflow/.github/actions/setup-node-env");
+      expect(setup.with?.["cache-mode"]).toBe(
+        "${{ needs.resolve_target.outputs.cache_write_allowed == 'true' && 'restore' || 'off' }}",
+      );
+    }
+    expect(kovaStage.run).toBe(sourceStage.run);
+    for (const stage of [kovaStage, sourceStage]) {
+      expect(stage.run).toContain(
+        'trusted_action="$PERFORMANCE_HELPER_DIR/.github/actions/setup-pnpm-store-cache"',
+      );
+      expect(stage.run).toContain('rm -rf -- "$actions_dir/setup-pnpm-store-cache"');
+      expect(stage.run).toContain(
+        'cp -R -- "$trusted_action" "$actions_dir/setup-pnpm-store-cache"',
+      );
+      expect(stage.run).toContain(
+        'cmp "$trusted_action/action.yml" "$actions_dir/setup-pnpm-store-cache/action.yml"',
+      );
+      expect(stage.run).toContain(
+        'cmp "$trusted_action/ensure-node.sh" "$actions_dir/setup-pnpm-store-cache/ensure-node.sh"',
+      );
+    }
+    const kovaSteps = workflow.jobs?.kova?.steps ?? [];
+    const sourceSteps = workflow.jobs?.source_performance?.steps ?? [];
+    expect(kovaSteps.findIndex((step) => step.name === kovaStage.name)).toBeLessThan(
+      kovaSteps.findIndex((step) => step.name === kovaSetup.name),
+    );
+    expect(sourceSteps.findIndex((step) => step.name === sourceStage.name)).toBeLessThan(
+      sourceSteps.findIndex((step) => step.name === sourceSetup.name),
+    );
+    expect(decideLane.run).toContain(
+      'if [[ "$LANE_ID" == "live-openai-candidate" && "$run_lane" == "true" && "$SECRET_ELIGIBLE" != "true" ]]; then',
+    );
+    expect(decideLane.run).toContain('reason="candidate is not eligible for live credentials"');
+
+    const trustedSha = "a".repeat(40);
+    for (const eventName of ["schedule", "workflow_dispatch"] as const) {
+      const trusted = runCandidateTrustClassification({
+        candidateSha: trustedSha,
+        eventName,
+        ref: "refs/heads/main",
+        workflowSha: trustedSha,
+      });
+      expect(trusted.result.status, trusted.result.stderr).toBe(0);
+      expect(trusted.outputs).toEqual({
+        secret_eligible: "true",
+        cache_write_allowed: "true",
+      });
+    }
+
+    for (const candidate of [
+      {
+        candidateSha: "b".repeat(40),
+        eventName: "workflow_dispatch" as const,
+        ref: "refs/heads/main",
+        workflowSha: trustedSha,
+      },
+      {
+        candidateSha: trustedSha,
+        eventName: "workflow_dispatch" as const,
+        ref: "refs/heads/release/2026.8.1",
+        workflowSha: trustedSha,
+      },
+    ]) {
+      const untrusted = runCandidateTrustClassification(candidate);
+      expect(untrusted.result.status, untrusted.result.stderr).toBe(0);
+      expect(untrusted.outputs).toEqual({
+        secret_eligible: "false",
+        cache_write_allowed: "false",
+      });
+    }
+  });
+
+  it("replaces candidate-owned nested setup actions with the trusted workflow copy", () => {
+    const stages = [
+      findStep("Stage trusted setup action graph"),
+      findStep("Stage trusted source setup action graph", "source_performance"),
+    ];
+
+    for (const stage of stages) {
+      const root = tempDirs.make("openclaw-performance-action-graph-");
+      const workspace = join(root, "candidate");
+      const helper = join(root, "workflow");
+      const candidateAction = join(workspace, ".github/actions/setup-pnpm-store-cache/action.yml");
+      const candidateEnsureNode = join(
+        workspace,
+        ".github/actions/setup-pnpm-store-cache/ensure-node.sh",
+      );
+      const trustedAction = join(helper, ".github/actions/setup-pnpm-store-cache/action.yml");
+      const trustedEnsureNode = join(
+        helper,
+        ".github/actions/setup-pnpm-store-cache/ensure-node.sh",
+      );
+      mkdirSync(join(workspace, ".github/actions/setup-pnpm-store-cache"), {
+        recursive: true,
+      });
+      mkdirSync(join(helper, ".github/actions/setup-pnpm-store-cache"), {
+        recursive: true,
+      });
+      writeFileSync(candidateAction, "candidate action\n");
+      writeFileSync(candidateEnsureNode, "candidate script\n");
+      writeFileSync(trustedAction, "trusted action\n");
+      writeFileSync(trustedEnsureNode, "trusted script\n");
+
+      const result = spawnSync("bash", ["-c", stage.run ?? ""], {
+        encoding: "utf8",
+        env: {
+          ...process.env,
+          GITHUB_WORKSPACE: workspace,
+          PERFORMANCE_HELPER_DIR: helper,
+        },
+      });
+
+      expect(result.status, result.stderr).toBe(0);
+      expect(readFileSync(candidateAction, "utf8")).toBe("trusted action\n");
+      expect(readFileSync(candidateEnsureNode, "utf8")).toBe("trusted script\n");
+    }
+  });
+
  it("pins the OCM release archive and checksum", () => {
    const workflow = readFileSync(WORKFLOW, "utf8");
    const installRun = findStep("Install OCM and Kova").run ?? "";
@@ -425,7 +618,7 @@ describe("OpenClaw performance workflow", () => {

    expect(publisher?.needs).toEqual(["resolve_target", "kova", "source_performance"]);
    expect(publisher?.if).toBe(
-      "${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish_reports == true)) && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' && needs.source_performance.result != 'cancelled' }}",
+      "${{ always() && needs.resolve_target.outputs.secret_eligible == 'true' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish_reports == true)) && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' && needs.source_performance.result != 'cancelled' }}",
    );
    expect(publisher?.["runs-on"]).toBe("ubuntu-24.04");
    expect(publisher?.permissions?.actions).toBe("read");
@@ -488,7 +681,7 @@ describe("OpenClaw performance workflow", () => {

    expect(appToken.id).toBe("clawgrit_app_token");
    expect(appToken.if).toBe(
-      "${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}",
+      "${{ needs.resolve_target.outputs.secret_eligible == 'true' && steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}",
    );
    expect(appToken.uses).toBe(
      "actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1",
@@ -506,7 +699,7 @@ describe("OpenClaw performance workflow", () => {
    expect(workflowText.split(appTokenOutput)).toHaveLength(2);
    expect(workflowText.split("${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}")).toHaveLength(2);
    expect(publish.if).toBe(
-      "${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}",
+      "${{ needs.resolve_target.outputs.secret_eligible == 'true' && steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}",
    );
    expect(workflowText).not.toContain("CLAWGRIT_REPORTS_TOKEN");
    expect(workflowText).not.toContain("secrets.GH_APP_PRIVATE_KEY");
@@ -622,7 +815,7 @@ describe("OpenClaw performance workflow", () => {
      "scripts/lib/kova-report-selector.mjs",
    );
    expect(helper.with).toMatchObject({
-      ref: "${{ github.sha }}",
+      ref: "${{ github.workflow_sha }}",
      path: ".artifacts/performance-publisher",
      "sparse-checkout":
        "scripts/lib/kova-report-publish-files.mjs\nscripts/lib/kova-report-selector.mjs\n",
@@ -1110,16 +1303,17 @@ esac
    const runKova = findStep("Run Kova");

    expect(configureAuth.if).toContain("matrix.live == 'true'");
+    expect(configureAuth.if).toContain("needs.resolve_target.outputs.secret_eligible == 'true'");
    expect(configureAuth.env?.OPENAI_API_KEY).toBe("${{ secrets.OPENAI_API_KEY }}");
    expect(configureAuth.run).toContain('if [[ -z "${OPENAI_API_KEY:-}" ]]; then');
    expect(configureAuth.run).toContain("cannot run without live evidence");
    expect(configureAuth.run).toContain("exit 1");
    expect(configureAuth.run).not.toContain("will be skipped");
    expect(runKova.env?.OPENAI_API_KEY).toBe(
-      "${{ matrix.live == 'true' && secrets.OPENAI_API_KEY || '' }}",
+      "${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_API_KEY || '' }}",
    );
    expect(runKova.env?.OPENAI_BASE_URL).toBe(
-      "${{ matrix.live == 'true' && secrets.OPENAI_BASE_URL || '' }}",
+      "${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_BASE_URL || '' }}",
    );
    expect(runKova.run).not.toContain('echo "skipped=true" >> "$GITHUB_OUTPUT"');
  });