ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-23 10:55:31 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(ci): isolate performance candidate trust (#127160)

Browse Source
This commit is contained in:
Vincent Koc
2026-08-21 07:54:56 -07:00
committed by GitHub
parent 03d4b8a9c3
commit 7833e242cb
2 changed files with 279 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/openclaw-performance.yml
+76 -13
View File
@@ -93,6 +93,8 @@ jobs:
kova_ref: ${{ steps.resolve.outputs.kova_ref }}
kova_config_contract: ${{ steps.resolve.outputs.kova_config_contract }}
kova_ref_trusted_for_live: ${{ steps.resolve.outputs.kova_ref_trusted_for_live }}
secret_eligible: ${{ steps.candidate_trust.outputs.secret_eligible }}
cache_write_allowed: ${{ steps.candidate_trust.outputs.cache_write_allowed }}
steps:
- name: Checkout target metadata
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
@@ -177,6 +179,34 @@ jobs:
echo "kova_ref_trusted_for_live=false" >> "$GITHUB_OUTPUT"
fi
- name: Classify performance candidate trust
id: candidate_trust
env:
CANDIDATE_SHA: ${{ steps.resolve.outputs.tested_sha }}
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
secret_eligible=false
cache_write_allowed=false
if [[
"$GITHUB_EVENT_NAME" == "schedule" ||
"$GITHUB_EVENT_NAME" == "workflow_dispatch"
]] && [[
"$GITHUB_REF" == "refs/heads/${DEFAULT_BRANCH}" &&
"$CANDIDATE_SHA" == "$WORKFLOW_SHA"
]]; then
secret_eligible=true
cache_write_allowed=true
fi
{
echo "secret_eligible=$secret_eligible"
echo "cache_write_allowed=$cache_write_allowed"
} >> "$GITHUB_OUTPUT"
kova:
name: ${{ matrix.title }}
needs: resolve_target
@@ -234,6 +264,7 @@ jobs:
MATRIX_DEEP_PROFILE: ${{ matrix.deep_profile }}
MATRIX_LIVE: ${{ matrix.live }}
KOVA_REF_TRUSTED_FOR_LIVE: ${{ needs.resolve_target.outputs.kova_ref_trusted_for_live }}
SECRET_ELIGIBLE: ${{ needs.resolve_target.outputs.secret_eligible }}
steps:
- name: Decide lane
id: lane
@@ -250,6 +281,10 @@ jobs:
run_lane=false
reason="live_openai_candidate input is false"
fi
if [[ "$LANE_ID" == "live-openai-candidate" && "$run_lane" == "true" && "$SECRET_ELIGIBLE" != "true" ]]; then
run_lane=false
reason="candidate is not eligible for live credentials"
fi
if [[ "$LANE_ID" == "live-openai-candidate" && "$run_lane" == "true" && "$KOVA_REF_TRUSTED_FOR_LIVE" != "true" ]]; then
echo "::error::The live OpenAI lane only executes a reviewed immutable Kova default. Omit kova_ref or update the pinned workflow defaults after review."
exit 1
@@ -271,7 +306,7 @@ jobs:
if: steps.lane.outputs.run == 'true'
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.sha }}
ref: ${{ github.workflow_sha }}
path: .artifacts/performance-workflow
fetch-depth: 1
persist-credentials: false
@@ -295,11 +330,35 @@ jobs:
echo "Workflow SHA: ${GITHUB_SHA}"
} >> "$GITHUB_STEP_SUMMARY"
- name: Stage trusted setup action graph
if: steps.lane.outputs.run == 'true'
shell: bash
run: &stage_trusted_performance_setup |
set -euo pipefail
github_dir="$GITHUB_WORKSPACE/.github"
actions_dir="$github_dir/actions"
trusted_action="$PERFORMANCE_HELPER_DIR/.github/actions/setup-pnpm-store-cache"
test -f "$trusted_action/action.yml"
test -f "$trusted_action/ensure-node.sh"
if [[ -L "$github_dir" || ( -e "$github_dir" && ! -d "$github_dir" ) ]]; then
rm -rf -- "$github_dir"
fi
mkdir -p "$github_dir"
if [[ -L "$actions_dir" || ( -e "$actions_dir" && ! -d "$actions_dir" ) ]]; then
rm -rf -- "$actions_dir"
fi
mkdir -p "$actions_dir"
rm -rf -- "$actions_dir/setup-pnpm-store-cache"
cp -R -- "$trusted_action" "$actions_dir/setup-pnpm-store-cache"
cmp "$trusted_action/action.yml" "$actions_dir/setup-pnpm-store-cache/action.yml"
cmp "$trusted_action/ensure-node.sh" "$actions_dir/setup-pnpm-store-cache/ensure-node.sh"
- name: Set up Node environment
if: steps.lane.outputs.run == 'true'
uses: ./.github/actions/setup-node-env
uses: ./.artifacts/performance-workflow/.github/actions/setup-node-env
with:
cache-mode: restore
cache-mode: ${{ needs.resolve_target.outputs.cache_write_allowed == 'true' && 'restore' || 'off' }}
install-bun: "false"
- name: Prepare systemd user session
@@ -448,7 +507,7 @@ jobs:
echo "KOVA_LANE_REPEAT=$repeat" >> "$GITHUB_ENV"
- name: Configure live OpenAI auth
if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }}
if: ${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' }}
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
@@ -466,8 +525,8 @@ jobs:
id: kova
if: steps.lane.outputs.run == 'true'
env:
OPENAI_API_KEY: ${{ matrix.live == 'true' && secrets.OPENAI_API_KEY || '' }}
OPENAI_BASE_URL: ${{ matrix.live == 'true' && secrets.OPENAI_BASE_URL || '' }}
OPENAI_API_KEY: ${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_API_KEY || '' }}
OPENAI_BASE_URL: ${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_BASE_URL || '' }}
shell: bash
run: |
set -euo pipefail
@@ -646,7 +705,7 @@ jobs:
- name: Checkout source performance helpers
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.sha }}
ref: ${{ github.workflow_sha }}
path: .artifacts/performance-workflow
fetch-depth: 1
persist-credentials: false
@@ -667,10 +726,14 @@ jobs:
echo "Workflow SHA: ${GITHUB_SHA}"
} >> "$GITHUB_STEP_SUMMARY"
- name: Stage trusted source setup action graph
shell: bash
run: *stage_trusted_performance_setup
- name: Set up source performance environment
uses: ./.github/actions/setup-node-env
uses: ./.artifacts/performance-workflow/.github/actions/setup-node-env
with:
cache-mode: restore
cache-mode: ${{ needs.resolve_target.outputs.cache_write_allowed == 'true' && 'restore' || 'off' }}
install-bun: "false"
- name: Fetch previous source performance baseline
@@ -925,7 +988,7 @@ jobs:
publish:
name: Publish ${{ matrix.title }} report
needs: [resolve_target, kova, source_performance]
if: ${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish_reports == true)) && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' && needs.source_performance.result != 'cancelled' }}
if: ${{ always() && needs.resolve_target.outputs.secret_eligible == 'true' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish_reports == true)) && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' && needs.source_performance.result != 'cancelled' }}
runs-on: ubuntu-24.04
timeout-minutes: 30
permissions:
@@ -966,7 +1029,7 @@ jobs:
if: steps.lane.outputs.run == 'true'
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.sha }}
ref: ${{ github.workflow_sha }}
path: .artifacts/performance-publisher
sparse-checkout: |
scripts/lib/kova-report-publish-files.mjs
@@ -1246,7 +1309,7 @@ jobs:
- name: Create clawgrit reports app token
id: clawgrit_app_token
if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}
if: ${{ needs.resolve_target.outputs.secret_eligible == 'true' && steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
with:
@@ -1257,7 +1320,7 @@ jobs:
permission-contents: write
- name: Publish to clawgrit reports
if: ${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}
if: ${{ needs.resolve_target.outputs.secret_eligible == 'true' && steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
env:
CLAWGRIT_REPORTS_APP_TOKEN: ${{ steps.clawgrit_app_token.outputs.token }}
test/scripts/openclaw-performance-workflow.test.ts
+203 -9
View File
@@ -74,6 +74,46 @@ function kovaMatrixEntries(): Array<Record<string, string>> {
return readWorkflow().jobs?.kova?.strategy?.matrix?.include ?? [];
}
function runCandidateTrustClassification({
candidateSha,
eventName,
ref,
workflowSha,
}: {
candidateSha: string;
eventName: "schedule" | "workflow_dispatch";
ref: string;
workflowSha: string;
}) {
const step = findStep("Classify performance candidate trust", "resolve_target");
const root = tempDirs.make("openclaw-performance-candidate-trust-");
const output = join(root, "output");
const result = spawnSync("bash", ["-c", step.run ?? ""], {
encoding: "utf8",
env: {
...process.env,
CANDIDATE_SHA: candidateSha,
DEFAULT_BRANCH: "main",
GITHUB_EVENT_NAME: eventName,
GITHUB_OUTPUT: output,
GITHUB_REF: ref,
WORKFLOW_SHA: workflowSha,
},
});
const outputs = Object.fromEntries(
existsSync(output)
? readFileSync(output, "utf8")
.trim()
.split("\n")
.map((line) => {
const separator = line.indexOf("=");
return [line.slice(0, separator), line.slice(separator + 1)];
})
: [],
);
return { outputs, result };
}
describe("OpenClaw performance workflow", () => {
it("uses an optional dispatch identifier to name parent-owned runs", () => {
const workflow = readFileSync(WORKFLOW, "utf8");
@@ -195,13 +235,13 @@ describe("OpenClaw performance workflow", () => {
decideLane.run?.indexOf('echo "run=$run_lane"') ?? -1,
);
expect(configureLiveAuth.if).toBe(
"${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' }}",
"${{ steps.lane.outputs.run == 'true' && matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' }}",
);
expect(runKova.env?.OPENAI_API_KEY).toBe(
"${{ matrix.live == 'true' && secrets.OPENAI_API_KEY || '' }}",
"${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_API_KEY || '' }}",
);
expect(runKova.env?.OPENAI_BASE_URL).toBe(
"${{ matrix.live == 'true' && secrets.OPENAI_BASE_URL || '' }}",
"${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_BASE_URL || '' }}",
);
try {
@@ -212,6 +252,7 @@ describe("OpenClaw performance workflow", () => {
GITHUB_OUTPUT: output,
KOVA_REF_TRUSTED_FOR_LIVE: "false",
LANE_ID: "live-openai-candidate",
SECRET_ELIGIBLE: "true",
},
});
expect(rejected.status).toBe(1);
@@ -226,6 +267,7 @@ describe("OpenClaw performance workflow", () => {
GITHUB_OUTPUT: output,
KOVA_REF_TRUSTED_FOR_LIVE: "true",
LANE_ID: "live-openai-candidate",
SECRET_ELIGIBLE: "true",
},
});
expect(accepted.status).toBe(0);
@@ -235,6 +277,157 @@ describe("OpenClaw performance workflow", () => {
}
});
it("keeps arbitrary performance candidates secretless and cacheless", () => {
const workflow = readWorkflow();
const trust = findStep("Classify performance candidate trust", "resolve_target");
const decideLane = findStep("Decide lane");
const kovaHarness = findStep("Checkout performance workflow helpers");
const kovaStage = findStep("Stage trusted setup action graph");
const kovaSetup = findStep("Set up Node environment");
const sourceHarness = findStep("Checkout source performance helpers", "source_performance");
const sourceStage = findStep("Stage trusted source setup action graph", "source_performance");
const sourceSetup = findStep("Set up source performance environment", "source_performance");
const publisherHarness = findStep("Checkout performance publisher helper", "publish");
expect(workflow.jobs?.resolve_target?.outputs).toMatchObject({
secret_eligible: "${{ steps.candidate_trust.outputs.secret_eligible }}",
cache_write_allowed: "${{ steps.candidate_trust.outputs.cache_write_allowed }}",
});
expect(trust.env).toMatchObject({
CANDIDATE_SHA: "${{ steps.resolve.outputs.tested_sha }}",
DEFAULT_BRANCH: "${{ github.event.repository.default_branch }}",
WORKFLOW_SHA: "${{ github.workflow_sha }}",
});
expect(trust.run).toContain("secret_eligible=false");
expect(trust.run).toContain("cache_write_allowed=false");
expect(trust.run).toContain('"$GITHUB_REF" == "refs/heads/${DEFAULT_BRANCH}"');
expect(trust.run).toContain('"$CANDIDATE_SHA" == "$WORKFLOW_SHA"');
expect(trust.run).toContain("secret_eligible=true");
expect(trust.run).toContain("cache_write_allowed=true");
for (const harness of [kovaHarness, sourceHarness, publisherHarness]) {
expect(harness.with?.ref).toBe("${{ github.workflow_sha }}");
expect(harness.with?.["persist-credentials"]).toBe(false);
}
for (const setup of [kovaSetup, sourceSetup]) {
expect(setup.uses).toBe("./.artifacts/performance-workflow/.github/actions/setup-node-env");
expect(setup.with?.["cache-mode"]).toBe(
"${{ needs.resolve_target.outputs.cache_write_allowed == 'true' && 'restore' || 'off' }}",
);
}
expect(kovaStage.run).toBe(sourceStage.run);
for (const stage of [kovaStage, sourceStage]) {
expect(stage.run).toContain(
'trusted_action="$PERFORMANCE_HELPER_DIR/.github/actions/setup-pnpm-store-cache"',
);
expect(stage.run).toContain('rm -rf -- "$actions_dir/setup-pnpm-store-cache"');
expect(stage.run).toContain(
'cp -R -- "$trusted_action" "$actions_dir/setup-pnpm-store-cache"',
);
expect(stage.run).toContain(
'cmp "$trusted_action/action.yml" "$actions_dir/setup-pnpm-store-cache/action.yml"',
);
expect(stage.run).toContain(
'cmp "$trusted_action/ensure-node.sh" "$actions_dir/setup-pnpm-store-cache/ensure-node.sh"',
);
}
const kovaSteps = workflow.jobs?.kova?.steps ?? [];
const sourceSteps = workflow.jobs?.source_performance?.steps ?? [];
expect(kovaSteps.findIndex((step) => step.name === kovaStage.name)).toBeLessThan(
kovaSteps.findIndex((step) => step.name === kovaSetup.name),
);
expect(sourceSteps.findIndex((step) => step.name === sourceStage.name)).toBeLessThan(
sourceSteps.findIndex((step) => step.name === sourceSetup.name),
);
expect(decideLane.run).toContain(
'if [[ "$LANE_ID" == "live-openai-candidate" && "$run_lane" == "true" && "$SECRET_ELIGIBLE" != "true" ]]; then',
);
expect(decideLane.run).toContain('reason="candidate is not eligible for live credentials"');
const trustedSha = "a".repeat(40);
for (const eventName of ["schedule", "workflow_dispatch"] as const) {
const trusted = runCandidateTrustClassification({
candidateSha: trustedSha,
eventName,
ref: "refs/heads/main",
workflowSha: trustedSha,
});
expect(trusted.result.status, trusted.result.stderr).toBe(0);
expect(trusted.outputs).toEqual({
secret_eligible: "true",
cache_write_allowed: "true",
});
}
for (const candidate of [
{
candidateSha: "b".repeat(40),
eventName: "workflow_dispatch" as const,
ref: "refs/heads/main",
workflowSha: trustedSha,
},
{
candidateSha: trustedSha,
eventName: "workflow_dispatch" as const,
ref: "refs/heads/release/2026.8.1",
workflowSha: trustedSha,
},
]) {
const untrusted = runCandidateTrustClassification(candidate);
expect(untrusted.result.status, untrusted.result.stderr).toBe(0);
expect(untrusted.outputs).toEqual({
secret_eligible: "false",
cache_write_allowed: "false",
});
}
});
it("replaces candidate-owned nested setup actions with the trusted workflow copy", () => {
const stages = [
findStep("Stage trusted setup action graph"),
findStep("Stage trusted source setup action graph", "source_performance"),
];
for (const stage of stages) {
const root = tempDirs.make("openclaw-performance-action-graph-");
const workspace = join(root, "candidate");
const helper = join(root, "workflow");
const candidateAction = join(workspace, ".github/actions/setup-pnpm-store-cache/action.yml");
const candidateEnsureNode = join(
workspace,
".github/actions/setup-pnpm-store-cache/ensure-node.sh",
);
const trustedAction = join(helper, ".github/actions/setup-pnpm-store-cache/action.yml");
const trustedEnsureNode = join(
helper,
".github/actions/setup-pnpm-store-cache/ensure-node.sh",
);
mkdirSync(join(workspace, ".github/actions/setup-pnpm-store-cache"), {
recursive: true,
});
mkdirSync(join(helper, ".github/actions/setup-pnpm-store-cache"), {
recursive: true,
});
writeFileSync(candidateAction, "candidate action\n");
writeFileSync(candidateEnsureNode, "candidate script\n");
writeFileSync(trustedAction, "trusted action\n");
writeFileSync(trustedEnsureNode, "trusted script\n");
const result = spawnSync("bash", ["-c", stage.run ?? ""], {
encoding: "utf8",
env: {
...process.env,
GITHUB_WORKSPACE: workspace,
PERFORMANCE_HELPER_DIR: helper,
},
});
expect(result.status, result.stderr).toBe(0);
expect(readFileSync(candidateAction, "utf8")).toBe("trusted action\n");
expect(readFileSync(candidateEnsureNode, "utf8")).toBe("trusted script\n");
}
});
it("pins the OCM release archive and checksum", () => {
const workflow = readFileSync(WORKFLOW, "utf8");
const installRun = findStep("Install OCM and Kova").run ?? "";
@@ -425,7 +618,7 @@ describe("OpenClaw performance workflow", () => {
expect(publisher?.needs).toEqual(["resolve_target", "kova", "source_performance"]);
expect(publisher?.if).toBe(
"${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish_reports == true)) && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' && needs.source_performance.result != 'cancelled' }}",
"${{ always() && needs.resolve_target.outputs.secret_eligible == 'true' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish_reports == true)) && needs.resolve_target.result == 'success' && needs.kova.result != 'cancelled' && needs.source_performance.result != 'cancelled' }}",
);
expect(publisher?.["runs-on"]).toBe("ubuntu-24.04");
expect(publisher?.permissions?.actions).toBe("read");
@@ -488,7 +681,7 @@ describe("OpenClaw performance workflow", () => {
expect(appToken.id).toBe("clawgrit_app_token");
expect(appToken.if).toBe(
"${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}",
"${{ needs.resolve_target.outputs.secret_eligible == 'true' && steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}",
);
expect(appToken.uses).toBe(
"actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1",
@@ -506,7 +699,7 @@ describe("OpenClaw performance workflow", () => {
expect(workflowText.split(appTokenOutput)).toHaveLength(2);
expect(workflowText.split("${{ secrets.CLAWSWEEPER_APP_PRIVATE_KEY }}")).toHaveLength(2);
expect(publish.if).toBe(
"${{ steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}",
"${{ needs.resolve_target.outputs.secret_eligible == 'true' && steps.prepare.outputs.ready == 'true' && steps.prepare.outputs.already_published != 'true' }}",
);
expect(workflowText).not.toContain("CLAWGRIT_REPORTS_TOKEN");
expect(workflowText).not.toContain("secrets.GH_APP_PRIVATE_KEY");
@@ -622,7 +815,7 @@ describe("OpenClaw performance workflow", () => {
"scripts/lib/kova-report-selector.mjs",
);
expect(helper.with).toMatchObject({
ref: "${{ github.sha }}",
ref: "${{ github.workflow_sha }}",
path: ".artifacts/performance-publisher",
"sparse-checkout":
"scripts/lib/kova-report-publish-files.mjs\nscripts/lib/kova-report-selector.mjs\n",
@@ -1110,16 +1303,17 @@ esac
const runKova = findStep("Run Kova");
expect(configureAuth.if).toContain("matrix.live == 'true'");
expect(configureAuth.if).toContain("needs.resolve_target.outputs.secret_eligible == 'true'");
expect(configureAuth.env?.OPENAI_API_KEY).toBe("${{ secrets.OPENAI_API_KEY }}");
expect(configureAuth.run).toContain('if [[ -z "${OPENAI_API_KEY:-}" ]]; then');
expect(configureAuth.run).toContain("cannot run without live evidence");
expect(configureAuth.run).toContain("exit 1");
expect(configureAuth.run).not.toContain("will be skipped");
expect(runKova.env?.OPENAI_API_KEY).toBe(
"${{ matrix.live == 'true' && secrets.OPENAI_API_KEY || '' }}",
"${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_API_KEY || '' }}",
);
expect(runKova.env?.OPENAI_BASE_URL).toBe(
"${{ matrix.live == 'true' && secrets.OPENAI_BASE_URL || '' }}",
"${{ matrix.live == 'true' && needs.resolve_target.outputs.secret_eligible == 'true' && secrets.OPENAI_BASE_URL || '' }}",
);
expect(runKova.run).not.toContain('echo "skipped=true" >> "$GITHUB_OUTPUT"');
});