mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(ci): harden release performance evidence (#103790)
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
911f8ec68f
commit
366d09bcc7
@@ -33,7 +33,7 @@ on:
|
||||
default: false
|
||||
type: boolean
|
||||
live_openai_candidate:
|
||||
description: Run the live OpenAI GPT 5.5 agent-turn lane
|
||||
description: Run the live OpenAI GPT 5.6 agent-turn lane
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
@@ -70,7 +70,7 @@ env:
|
||||
OCM_VERSION: v0.2.15
|
||||
OCM_LINUX_X64_SHA256: b849b8de5d77e97e0df9319703254ae95e29d7f26a7552ea79bf173ff110ea0a
|
||||
KOVA_REPOSITORY: openclaw/Kova
|
||||
PERFORMANCE_MODEL_ID: gpt-5.5
|
||||
PERFORMANCE_MODEL_ID: gpt-5.6
|
||||
KOVA_SCENARIO_TIMEOUT_MS: "300000"
|
||||
|
||||
jobs:
|
||||
@@ -144,7 +144,7 @@ jobs:
|
||||
include_filters: "scenario:fresh-install,scenario:gateway-performance,scenario:agent-cold-warm-message"
|
||||
expected_release_entries: "fresh-install:fresh,fresh-install:onboarded-user,agent-cold-warm-message:mock-openai-provider,gateway-performance:many-bundled-plugins"
|
||||
- lane: live-openai-candidate
|
||||
title: Kova live OpenAI GPT 5.5 agent turn
|
||||
title: Kova live OpenAI GPT 5.6 agent turn
|
||||
auth: live
|
||||
repeat: "1"
|
||||
deep_profile: "false"
|
||||
@@ -352,7 +352,7 @@ jobs:
|
||||
echo "OPENCLAW_OCM_WORKSPACE_DEPENDENCY_DIRS=$workspace_dependency_dirs"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Pin Kova OpenAI model to GPT 5.5
|
||||
- name: Pin Kova OpenAI model to GPT 5.6
|
||||
if: steps.lane.outputs.run == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -443,7 +443,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ -z "${OPENAI_API_KEY:-}" ]]; then
|
||||
echo "OPENAI_API_KEY is not configured; live GPT 5.5 lane cannot run without live evidence." >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "OPENAI_API_KEY is not configured; live GPT 5.6 lane cannot run without live evidence." >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 1
|
||||
fi
|
||||
kova setup --ci --json
|
||||
@@ -502,7 +502,8 @@ jobs:
|
||||
--target "local-build:${GITHUB_WORKSPACE}" \
|
||||
--repeat "$repeat" \
|
||||
--include "$INCLUDE_FILTERS" \
|
||||
--auth "$AUTH_MODE"
|
||||
--auth "$AUTH_MODE" \
|
||||
--model "$PERFORMANCE_MODEL_ID"
|
||||
|
||||
effective_status="$status"
|
||||
if [[ "$FAIL_ON_REGRESSION" == "true" && "$status" != "0" ]]; then
|
||||
@@ -770,6 +771,8 @@ jobs:
|
||||
title: live OpenAI candidate
|
||||
env:
|
||||
LANE_ID: ${{ matrix.lane }}
|
||||
PERFORMANCE_PUBLISHER_HELPER: ${{ github.workspace }}/.artifacts/performance-publisher/scripts/lib/kova-report-publish-files.mjs
|
||||
PUBLISHED_REPORT_MAX_FILE_BYTES: "50000000"
|
||||
REPORT_PUBLISH_REQUIRED: ${{ github.event_name == 'schedule' || inputs.profile == 'release' }}
|
||||
steps:
|
||||
- name: Decide report publication lane
|
||||
@@ -786,6 +789,17 @@ jobs:
|
||||
fi
|
||||
echo "run=$run_lane" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout performance publisher helper
|
||||
if: steps.lane.outputs.run == 'true'
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
path: .artifacts/performance-publisher
|
||||
sparse-checkout: scripts/lib/kova-report-publish-files.mjs
|
||||
sparse-checkout-cone-mode: false
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve Kova artifact
|
||||
id: artifact
|
||||
if: steps.lane.outputs.run == 'true'
|
||||
@@ -847,6 +861,7 @@ jobs:
|
||||
if: steps.download.outcome == 'success'
|
||||
continue-on-error: ${{ env.REPORT_PUBLISH_REQUIRED != 'true' }}
|
||||
env:
|
||||
ARTIFACT_ID: ${{ steps.artifact.outputs.id }}
|
||||
INPUT_ROOT: ${{ steps.paths.outputs.input_root }}
|
||||
REPORTS_ROOT: ${{ steps.paths.outputs.reports_root }}
|
||||
TESTED_REF: ${{ needs.resolve_target.outputs.tested_ref }}
|
||||
@@ -863,7 +878,7 @@ jobs:
|
||||
echo "::${annotation}::Publisher paths escaped RUNNER_TEMP."
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! "$PRODUCER_ATTEMPT" =~ ^[0-9]+$ || "$PRODUCER_ATTEMPT" -gt "$GITHUB_RUN_ATTEMPT" ]]; then
|
||||
if [[ ! "$ARTIFACT_ID" =~ ^[0-9]+$ || ! "$PRODUCER_ATTEMPT" =~ ^[0-9]+$ || "$PRODUCER_ATTEMPT" -gt "$GITHUB_RUN_ATTEMPT" ]]; then
|
||||
echo "::${annotation}::Kova artifact attempt metadata is invalid."
|
||||
exit 1
|
||||
fi
|
||||
@@ -958,8 +973,6 @@ jobs:
|
||||
install -m 0644 "$report_md" "$dest/report.md"
|
||||
fi
|
||||
install -m 0644 "$summary" "$dest/index.md"
|
||||
mkdir -p "$dest/bundles"
|
||||
cp -R "$bundle"/. "$dest/bundles/"
|
||||
if [[ -n "$source_dir" ]]; then
|
||||
mkdir -p "$dest/source"
|
||||
cp -R "$source_dir"/. "$dest/source/"
|
||||
@@ -973,6 +986,14 @@ jobs:
|
||||
fi
|
||||
fi
|
||||
|
||||
artifact_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts/${ARTIFACT_ID}"
|
||||
{
|
||||
echo
|
||||
echo "## Full diagnostic artifact"
|
||||
echo
|
||||
echo "The complete Kova bundle remains in [Actions artifact ${ARTIFACT_ID}](${artifact_url}); its checksum is published under the bundles directory."
|
||||
} >> "$dest/index.md"
|
||||
|
||||
jq -n \
|
||||
--arg repository "$GITHUB_REPOSITORY" \
|
||||
--arg ref "$TESTED_REF" \
|
||||
@@ -987,6 +1008,12 @@ jobs:
|
||||
'{repository:$repository,ref:$ref,sha:$sha,tested_ref:$ref,tested_sha:$sha,workflow_ref:$workflow_ref,workflow_sha:$workflow_sha,workflow:$workflow,run_id:$run_id,run_attempt:$run_attempt,lane:$lane,path:$path}' \
|
||||
> "${reports_root}/openclaw-performance/${ref_slug}/latest-${LANE_ID}.json"
|
||||
|
||||
node "$PERFORMANCE_PUBLISHER_HELPER" \
|
||||
--bundle-dir "$bundle" \
|
||||
--bundle-destination "$dest/bundles" \
|
||||
--publish-root "$dest" \
|
||||
--max-file-bytes "$PUBLISHED_REPORT_MAX_FILE_BYTES"
|
||||
|
||||
git -C "$reports_root" config user.name "openclaw-performance[bot]"
|
||||
git -C "$reports_root" config user.email "openclaw-performance[bot]@users.noreply.github.com"
|
||||
git -C "$reports_root" add -- openclaw-performance
|
||||
|
||||
+3
-3
@@ -50,7 +50,7 @@ dispatch.
|
||||
| `ios-build` | Xcode project generation plus the iOS app simulator build | iOS app, shared app kit, or Swabble changes |
|
||||
| `android` | Android unit tests for both flavors plus one debug APK build | Android-relevant changes |
|
||||
| `test-performance-agent` | Separate workflow: daily Codex slow-test optimization after trusted activity | Main CI success or manual dispatch |
|
||||
| `openclaw-performance` | Separate workflow: daily/on-demand Kova runtime performance reports with mock-provider, deep-profile, and GPT 5.5 live lanes | Scheduled and manual dispatch |
|
||||
| `openclaw-performance` | Separate workflow: daily/on-demand Kova runtime performance reports with mock-provider, deep-profile, and GPT 5.6 live lanes | Scheduled and manual dispatch |
|
||||
|
||||
## Fail-fast order
|
||||
|
||||
@@ -230,11 +230,11 @@ The workflow installs OCM from a pinned release and Kova from `openclaw/Kova` at
|
||||
|
||||
- `mock-provider`: Kova diagnostic scenarios against a local-build runtime with deterministic fake OpenAI-compatible auth.
|
||||
- `mock-deep-profile`: CPU/heap/trace profiling for startup, gateway, and agent-turn hotspots. Runs on schedule, or on dispatch with `deep_profile=true`.
|
||||
- `live-openai-candidate`: a real OpenAI `openai/gpt-5.5` agent turn, skipped when `OPENAI_API_KEY` is unavailable. Runs on schedule, or on dispatch with `live_openai_candidate=true`.
|
||||
- `live-openai-candidate`: a real OpenAI `openai/gpt-5.6` agent turn, skipped when `OPENAI_API_KEY` is unavailable. Runs on schedule, or on dispatch with `live_openai_candidate=true`.
|
||||
|
||||
The mock-provider lane also runs OpenClaw-native source probes after the Kova pass: gateway boot timing and memory across default, skipped-channel, internal-hook, and fifty-plugin startup cases; bundled plugin import RSS, repeated mock-OpenAI `channel-chat-baseline` hello loops, CLI startup commands against the booted gateway, and the SQLite state smoke performance probe. When the previous published mock-provider source report is available for the tested ref, the source summary compares current RSS and heap values against that baseline and marks large RSS increases as `watch`. The source probe Markdown summary lives at `source/index.md` in the report bundle, with raw JSON beside it.
|
||||
|
||||
Every lane uploads GitHub artifacts. A separate publisher job downloads and validates those artifacts, then mints a short-lived ClawSweeper GitHub App token scoped only to `openclaw/clawgrit-reports` contents and passes it only to the Git push step. It commits `report.json`, `report.md`, bundles, `index.md`, and source-probe artifacts under `openclaw-performance/<tested-ref>/<run-id>-<attempt>/<lane>/`; the current tested-ref pointer is `openclaw-performance/<tested-ref>/latest-<lane>.json`. Scheduled runs and `profile=release` dispatches fail if app-token creation or report publication fails. Manual non-release dispatches keep publication advisory and retain the GitHub artifacts when authentication or publishing fails. The previous source baseline is fetched anonymously from the public reports repository, so a successful baseline fetch does not prove publisher authentication.
|
||||
Every lane uploads its complete GitHub artifact, including CPU, heap, trace, and compressed diagnostic bundles. A separate publisher job downloads and validates those artifacts, then mints a short-lived ClawSweeper GitHub App token scoped only to `openclaw/clawgrit-reports` contents and passes it only to the Git push step. It commits `report.json`, `report.md`, `index.md`, source-probe artifacts, and bundle metadata/checksums under `openclaw-performance/<tested-ref>/<run-id>-<attempt>/<lane>/`; the full diagnostic archive stays in the linked Actions artifact. The publisher rejects any report file over 50 MB before attempting a push. The current tested-ref pointer is `openclaw-performance/<tested-ref>/latest-<lane>.json`. Scheduled runs and `profile=release` dispatches fail if app-token creation or report publication fails. Manual non-release dispatches keep publication advisory and retain the GitHub artifacts when authentication or publishing fails. The previous source baseline is fetched anonymously from the public reports repository, so a successful baseline fetch does not prove publisher authentication.
|
||||
|
||||
## Full Release Validation
|
||||
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { chmodSync, copyFileSync, mkdirSync, readdirSync, statSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
function check(condition, message) {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
function positiveInteger(value, label) {
|
||||
const parsed = Number(value);
|
||||
check(Number.isSafeInteger(parsed) && parsed > 0, `${label} must be a positive integer`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function regularFiles(root) {
|
||||
const files = [];
|
||||
const visit = (dir, prefix) => {
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const relative = path.join(prefix, entry.name);
|
||||
const absolute = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
visit(absolute, relative);
|
||||
continue;
|
||||
}
|
||||
check(entry.isFile(), `publish tree contains non-regular file: ${relative}`);
|
||||
files.push({ absolute, relative });
|
||||
}
|
||||
};
|
||||
visit(root, "");
|
||||
return files;
|
||||
}
|
||||
|
||||
export function copyBundleMetadata({ bundleDir, destinationDir }) {
|
||||
const entries = readdirSync(bundleDir, { withFileTypes: true });
|
||||
const metadata = entries
|
||||
.filter(
|
||||
(entry) => entry.isFile() && (entry.name === "bundle.json" || entry.name.endsWith(".sha256")),
|
||||
)
|
||||
.map((entry) => entry.name)
|
||||
.toSorted();
|
||||
check(metadata.includes("bundle.json"), "Kova bundle metadata is missing bundle.json");
|
||||
check(
|
||||
metadata.some((name) => name.endsWith(".sha256")),
|
||||
"Kova bundle metadata is missing a checksum",
|
||||
);
|
||||
|
||||
mkdirSync(destinationDir, { recursive: true });
|
||||
for (const name of metadata) {
|
||||
const source = path.join(bundleDir, name);
|
||||
const destination = path.join(destinationDir, name);
|
||||
copyFileSync(source, destination);
|
||||
chmodSync(destination, 0o644);
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
|
||||
export function assertPublishedFileSizeLimit({ publishRoot, maxFileBytes }) {
|
||||
const limit = positiveInteger(maxFileBytes, "max file bytes");
|
||||
const files = regularFiles(publishRoot);
|
||||
for (const file of files) {
|
||||
const size = statSync(file.absolute).size;
|
||||
check(
|
||||
size <= limit,
|
||||
`refusing to publish oversized file ${file.relative}: ${size} bytes exceeds ${limit}`,
|
||||
);
|
||||
}
|
||||
return files.length;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const values = {};
|
||||
for (let index = 0; index < argv.length; index += 2) {
|
||||
const flag = argv[index];
|
||||
const value = argv[index + 1];
|
||||
check(flag?.startsWith("--") && value, "invalid CLI arguments");
|
||||
const key = flag.slice(2);
|
||||
check(
|
||||
["bundle-dir", "bundle-destination", "publish-root", "max-file-bytes"].includes(key),
|
||||
`unknown --${key}`,
|
||||
);
|
||||
check(values[key] === undefined, `duplicate --${key}`);
|
||||
values[key] = value;
|
||||
}
|
||||
for (const key of ["bundle-dir", "bundle-destination", "publish-root", "max-file-bytes"]) {
|
||||
check(values[key], `missing --${key}`);
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
function runCli() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const bundleFiles = copyBundleMetadata({
|
||||
bundleDir: args["bundle-dir"],
|
||||
destinationDir: args["bundle-destination"],
|
||||
});
|
||||
const publishedFileCount = assertPublishedFileSizeLimit({
|
||||
publishRoot: args["publish-root"],
|
||||
maxFileBytes: args["max-file-bytes"],
|
||||
});
|
||||
console.log(JSON.stringify({ bundleFiles, publishedFileCount }));
|
||||
}
|
||||
|
||||
if (path.resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) {
|
||||
try {
|
||||
runCli();
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,16 @@ import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const AUTH_MODES = new Set(["live", "mock"]);
|
||||
const CLI_KEYS = new Set(["auth", "include", "plan", "profile", "repeat", "report", "target"]);
|
||||
const CLI_KEYS = new Set([
|
||||
"auth",
|
||||
"include",
|
||||
"model",
|
||||
"plan",
|
||||
"profile",
|
||||
"repeat",
|
||||
"report",
|
||||
"target",
|
||||
]);
|
||||
|
||||
function check(condition, reason) {
|
||||
if (!condition) {
|
||||
@@ -55,7 +64,7 @@ function repeatIndexesFor(selectedPairs) {
|
||||
return new Map([...selectedPairs].map((key) => [key, new Set()]));
|
||||
}
|
||||
|
||||
function validateLiveRecord(record, key) {
|
||||
function validateLiveRecord(record, key, expectedModel) {
|
||||
const auth = object(record.auth, `record ${displayPair(key)} auth`);
|
||||
const provider = object(record.providerEvidence, `record ${displayPair(key)} provider evidence`);
|
||||
check(
|
||||
@@ -75,7 +84,28 @@ function validateLiveRecord(record, key) {
|
||||
provider.available === true,
|
||||
`live record ${displayPair(key)} provider evidence was unavailable`,
|
||||
);
|
||||
positiveInteger(provider.requestCount, `live record ${displayPair(key)} provider request count`);
|
||||
const requestCount = positiveInteger(
|
||||
provider.requestCount,
|
||||
`live record ${displayPair(key)} provider request count`,
|
||||
);
|
||||
const models = array(provider.models, `live record ${displayPair(key)} provider models`);
|
||||
check(
|
||||
models.length === 1,
|
||||
`live record ${displayPair(key)} provider model evidence was not exact`,
|
||||
);
|
||||
const model = object(models[0], `live record ${displayPair(key)} provider model`);
|
||||
check(
|
||||
text(model.value, `live record ${displayPair(key)} provider model value`) === expectedModel,
|
||||
`live record ${displayPair(key)} provider model did not match ${expectedModel}`,
|
||||
);
|
||||
const modelCount = positiveInteger(
|
||||
model.count,
|
||||
`live record ${displayPair(key)} provider model count`,
|
||||
);
|
||||
check(
|
||||
modelCount === requestCount,
|
||||
`live record ${displayPair(key)} provider model count did not match request count`,
|
||||
);
|
||||
}
|
||||
|
||||
export function validateKovaWorkflowEvidence({
|
||||
@@ -86,6 +116,7 @@ export function validateKovaWorkflowEvidence({
|
||||
repeat,
|
||||
includeFilters,
|
||||
authMode,
|
||||
expectedModel,
|
||||
}) {
|
||||
const expectedProfile = text(profile, "expected profile");
|
||||
const expectedTarget = text(target, "expected target");
|
||||
@@ -94,6 +125,7 @@ export function validateKovaWorkflowEvidence({
|
||||
text(value, `expected include filter ${index}`),
|
||||
);
|
||||
const expectedAuth = text(authMode, "expected auth mode");
|
||||
const expectedProviderModel = text(expectedModel, "expected model");
|
||||
check(AUTH_MODES.has(expectedAuth), `unsupported expected auth mode ${expectedAuth}`);
|
||||
check(expectedFilters.length > 0, "expected include filters were empty");
|
||||
|
||||
@@ -180,7 +212,7 @@ export function validateKovaWorkflowEvidence({
|
||||
const recordAuth = object(record.auth, `record ${displayPair(key)} auth`);
|
||||
check(recordAuth.mode === expectedAuth, `record ${displayPair(key)} auth mode did not match`);
|
||||
if (expectedAuth === "live") {
|
||||
validateLiveRecord(record, key);
|
||||
validateLiveRecord(record, key, expectedProviderModel);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,7 +249,7 @@ function parseCliArgs(argv) {
|
||||
check(!Object.hasOwn(flags, key), `duplicate --${key}`);
|
||||
flags[key] = value;
|
||||
}
|
||||
for (const key of ["plan", "report", "profile", "target", "repeat", "include", "auth"]) {
|
||||
for (const key of ["plan", "report", "profile", "target", "repeat", "include", "auth", "model"]) {
|
||||
text(flags[key], `--${key}`);
|
||||
}
|
||||
return flags;
|
||||
@@ -248,6 +280,7 @@ function runCli() {
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean),
|
||||
authMode: flags.auth,
|
||||
expectedModel: flags.model,
|
||||
});
|
||||
console.log(
|
||||
`Kova plan/report evidence validated: ${result.pairCount} scenario/state pairs x ${result.repeat} repeats (${result.authMode})`,
|
||||
|
||||
@@ -1201,6 +1201,8 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([
|
||||
["scripts/lib/live-docker-stage.sh", ["test/scripts/live-docker-stage.test.ts"]],
|
||||
["scripts/lib/local-heavy-check-runtime.mjs", ["test/scripts/local-heavy-check-runtime.test.ts"]],
|
||||
["scripts/lib/kova-report-gate.mjs", ["test/scripts/kova-report-gate.test.ts"]],
|
||||
["scripts/lib/kova-report-publish-files.mjs", ["test/scripts/kova-report-publish-files.test.ts"]],
|
||||
["scripts/lib/kova-workflow-evidence.mjs", ["test/scripts/kova-workflow-evidence.test.ts"]],
|
||||
["scripts/lib/managed-child-process.mjs", ["test/scripts/managed-child-process.test.ts"]],
|
||||
[
|
||||
"scripts/lib/windows-taskkill.mjs",
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
assertPublishedFileSizeLimit,
|
||||
copyBundleMetadata,
|
||||
} from "../../scripts/lib/kova-report-publish-files.mjs";
|
||||
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
|
||||
const tempRoots = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
describe("Kova report publish files", () => {
|
||||
it("publishes bundle metadata while leaving the full archive in the Actions artifact", () => {
|
||||
const root = tempRoots.make("openclaw-kova-publish-");
|
||||
const bundleDir = join(root, "artifact", "bundle");
|
||||
const destinationDir = join(root, "report", "bundles");
|
||||
mkdirSync(bundleDir, { recursive: true });
|
||||
writeFileSync(join(bundleDir, "bundle.json"), '{"schemaVersion":"kova.bundle.v1"}\n');
|
||||
writeFileSync(join(bundleDir, "bundle.tar.gz"), Buffer.alloc(200));
|
||||
writeFileSync(join(bundleDir, "bundle.tar.gz.sha256"), "abc bundle.tar.gz\n");
|
||||
|
||||
expect(copyBundleMetadata({ bundleDir, destinationDir })).toEqual([
|
||||
"bundle.json",
|
||||
"bundle.tar.gz.sha256",
|
||||
]);
|
||||
expect(readdirSync(destinationDir).sort()).toEqual(["bundle.json", "bundle.tar.gz.sha256"]);
|
||||
expect(readFileSync(join(destinationDir, "bundle.tar.gz.sha256"), "utf8")).toBe(
|
||||
"abc bundle.tar.gz\n",
|
||||
);
|
||||
});
|
||||
|
||||
it("requires bundle metadata and a checksum", () => {
|
||||
const root = tempRoots.make("openclaw-kova-publish-");
|
||||
const bundleDir = join(root, "bundle");
|
||||
mkdirSync(bundleDir);
|
||||
writeFileSync(join(bundleDir, "bundle.tar.gz"), Buffer.alloc(1));
|
||||
|
||||
expect(() =>
|
||||
copyBundleMetadata({ bundleDir, destinationDir: join(root, "destination") }),
|
||||
).toThrow("Kova bundle metadata is missing bundle.json");
|
||||
|
||||
writeFileSync(join(bundleDir, "bundle.json"), "{}\n");
|
||||
expect(() =>
|
||||
copyBundleMetadata({ bundleDir, destinationDir: join(root, "destination") }),
|
||||
).toThrow("Kova bundle metadata is missing a checksum");
|
||||
});
|
||||
|
||||
it("accepts the size boundary and rejects the first oversized published file", () => {
|
||||
const root = tempRoots.make("openclaw-kova-publish-");
|
||||
writeFileSync(join(root, "at-limit.json"), Buffer.alloc(100));
|
||||
|
||||
expect(assertPublishedFileSizeLimit({ publishRoot: root, maxFileBytes: 100 })).toBe(1);
|
||||
|
||||
writeFileSync(join(root, "oversized.json"), Buffer.alloc(101));
|
||||
expect(() => assertPublishedFileSizeLimit({ publishRoot: root, maxFileBytes: 100 })).toThrow(
|
||||
"oversized.json: 101 bytes exceeds 100",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -13,6 +13,7 @@ type Pair = {
|
||||
|
||||
const PROFILE = "release";
|
||||
const TARGET = "local-build:/work/openclaw";
|
||||
const MODEL = "gpt-5.6";
|
||||
const INCLUDE_FILTERS = ["scenario:scenario-a", "scenario:scenario-b"];
|
||||
const FIRST_PAIR: Pair = { scenario: "scenario-a", state: "state-a" };
|
||||
const PAIRS: Pair[] = [FIRST_PAIR, { scenario: "scenario-b", state: "state-b" }];
|
||||
@@ -60,6 +61,7 @@ function record(pair: Pair, repeatIndex: number, repeat: number, authMode: "live
|
||||
available: true,
|
||||
environmentDependent: true,
|
||||
requestCount: 1,
|
||||
models: [{ value: MODEL, count: 1 }],
|
||||
source: "openclaw-timeline",
|
||||
}
|
||||
: {
|
||||
@@ -107,7 +109,12 @@ function report({
|
||||
function validate(
|
||||
lanePlan: JsonObject,
|
||||
laneReport: JsonObject,
|
||||
options: { authMode?: "live" | "mock"; includeFilters?: string[]; repeat?: number } = {},
|
||||
options: {
|
||||
authMode?: "live" | "mock";
|
||||
expectedModel?: string;
|
||||
includeFilters?: string[];
|
||||
repeat?: number;
|
||||
} = {},
|
||||
) {
|
||||
return validateKovaWorkflowEvidence({
|
||||
plan: lanePlan,
|
||||
@@ -117,6 +124,7 @@ function validate(
|
||||
repeat: options.repeat ?? 2,
|
||||
includeFilters: options.includeFilters ?? INCLUDE_FILTERS,
|
||||
authMode: options.authMode ?? "mock",
|
||||
expectedModel: options.expectedModel ?? MODEL,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -162,6 +170,8 @@ function runCli({
|
||||
repeat,
|
||||
"--include",
|
||||
INCLUDE_FILTERS.join(","),
|
||||
"--model",
|
||||
MODEL,
|
||||
];
|
||||
if (includeAuth) {
|
||||
args.push("--auth", "mock");
|
||||
@@ -246,6 +256,75 @@ describe("Kova workflow evidence", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects live evidence for a different provider model", () => {
|
||||
const liveFilters = ["scenario:scenario-a"];
|
||||
const lanePlan = plan([FIRST_PAIR], 1, liveFilters);
|
||||
const laneReport = report({
|
||||
authMode: "live",
|
||||
includeFilters: liveFilters,
|
||||
pairs: [FIRST_PAIR],
|
||||
repeat: 1,
|
||||
});
|
||||
(firstRecordOf(laneReport).providerEvidence as JsonObject).models = [
|
||||
{ value: "gpt-5.5", count: 1 },
|
||||
];
|
||||
|
||||
expect(() =>
|
||||
validate(lanePlan, laneReport, {
|
||||
authMode: "live",
|
||||
includeFilters: liveFilters,
|
||||
repeat: 1,
|
||||
}),
|
||||
).toThrow("live record scenario-a/state-a provider model did not match gpt-5.6");
|
||||
});
|
||||
|
||||
it("rejects mixed live provider models", () => {
|
||||
const liveFilters = ["scenario:scenario-a"];
|
||||
const lanePlan = plan([FIRST_PAIR], 1, liveFilters);
|
||||
const laneReport = report({
|
||||
authMode: "live",
|
||||
includeFilters: liveFilters,
|
||||
pairs: [FIRST_PAIR],
|
||||
repeat: 1,
|
||||
});
|
||||
const providerEvidence = firstRecordOf(laneReport).providerEvidence as JsonObject;
|
||||
providerEvidence.requestCount = 2;
|
||||
providerEvidence.models = [
|
||||
{ value: MODEL, count: 1 },
|
||||
{ value: "gpt-5.5", count: 1 },
|
||||
];
|
||||
|
||||
expect(() =>
|
||||
validate(lanePlan, laneReport, {
|
||||
authMode: "live",
|
||||
includeFilters: liveFilters,
|
||||
repeat: 1,
|
||||
}),
|
||||
).toThrow("live record scenario-a/state-a provider model evidence was not exact");
|
||||
});
|
||||
|
||||
it("rejects live provider model count drift", () => {
|
||||
const liveFilters = ["scenario:scenario-a"];
|
||||
const lanePlan = plan([FIRST_PAIR], 1, liveFilters);
|
||||
const laneReport = report({
|
||||
authMode: "live",
|
||||
includeFilters: liveFilters,
|
||||
pairs: [FIRST_PAIR],
|
||||
repeat: 1,
|
||||
});
|
||||
(firstRecordOf(laneReport).providerEvidence as JsonObject).models = [
|
||||
{ value: MODEL, count: 2 },
|
||||
];
|
||||
|
||||
expect(() =>
|
||||
validate(lanePlan, laneReport, {
|
||||
authMode: "live",
|
||||
includeFilters: liveFilters,
|
||||
repeat: 1,
|
||||
}),
|
||||
).toThrow("live record scenario-a/state-a provider model count did not match request count");
|
||||
});
|
||||
|
||||
it("rejects plan and report schema drift", () => {
|
||||
const badPlan = plan();
|
||||
badPlan.schemaVersion = "kova.matrix.plan.v0";
|
||||
|
||||
@@ -96,6 +96,8 @@ describe("OpenClaw performance workflow", () => {
|
||||
expect(
|
||||
installRun.indexOf('npm --prefix "$KOVA_SRC" ci --ignore-scripts --no-audit --no-fund'),
|
||||
).toBeLessThan(installRun.indexOf('cat > "$HOME/.local/bin/kova"'));
|
||||
expect(workflow).toContain("PERFORMANCE_MODEL_ID: gpt-5.6");
|
||||
expect(workflow).toContain("Kova live OpenAI GPT 5.6 agent turn");
|
||||
});
|
||||
|
||||
it("resolves each target once before benchmark and publication fan out", () => {
|
||||
@@ -319,6 +321,33 @@ describe("OpenClaw performance workflow", () => {
|
||||
expect(publish.run).not.toContain("rebase FETCH_HEAD");
|
||||
});
|
||||
|
||||
it("publishes bounded bundle metadata while retaining full diagnostics as an artifact", () => {
|
||||
const workflow = readWorkflow();
|
||||
const publisher = workflow.jobs?.publish;
|
||||
const helper = findStep("Checkout performance publisher helper", "publish");
|
||||
const prepare = findStep("Prepare clawgrit report commit", "publish");
|
||||
const upload = findStep("Upload Kova artifacts");
|
||||
|
||||
expect(publisher?.env?.PUBLISHED_REPORT_MAX_FILE_BYTES).toBe("50000000");
|
||||
expect(publisher?.env?.PERFORMANCE_PUBLISHER_HELPER).toContain(
|
||||
"scripts/lib/kova-report-publish-files.mjs",
|
||||
);
|
||||
expect(helper.with).toMatchObject({
|
||||
ref: "${{ github.sha }}",
|
||||
path: ".artifacts/performance-publisher",
|
||||
"sparse-checkout": "scripts/lib/kova-report-publish-files.mjs",
|
||||
"sparse-checkout-cone-mode": false,
|
||||
"persist-credentials": false,
|
||||
});
|
||||
expect(upload.with?.path).toContain(".artifacts/kova/bundles/${{ matrix.lane }}");
|
||||
expect(prepare.env?.ARTIFACT_ID).toBe("${{ steps.artifact.outputs.id }}");
|
||||
expect(prepare.run).toContain('node "$PERFORMANCE_PUBLISHER_HELPER"');
|
||||
expect(prepare.run).toContain('--bundle-destination "$dest/bundles"');
|
||||
expect(prepare.run).toContain('--max-file-bytes "$PUBLISHED_REPORT_MAX_FILE_BYTES"');
|
||||
expect(prepare.run).toContain("The complete Kova bundle remains in [Actions artifact");
|
||||
expect(prepare.run).not.toContain('cp -R "$bundle"/. "$dest/bundles/"');
|
||||
});
|
||||
|
||||
it("reuses the producing artifact when only publisher jobs rerun", () => {
|
||||
const artifact = findStep("Resolve Kova artifact", "publish");
|
||||
const root = mkdtempSync(join(realpathSync(tmpdir()), "openclaw-artifact-resolver-"));
|
||||
@@ -653,6 +682,7 @@ esac
|
||||
expect(run).toContain('--repeat "$repeat"');
|
||||
expect(run).toContain('--include "$INCLUDE_FILTERS"');
|
||||
expect(run).toContain('--auth "$AUTH_MODE"');
|
||||
expect(run).toContain('--model "$PERFORMANCE_MODEL_ID"');
|
||||
});
|
||||
|
||||
it("installs local workspace packages beside the OCM root tarball", () => {
|
||||
|
||||
Reference in New Issue
Block a user