mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-25 20:05:46 -06:00
improve(ci): shard iOS screenshot capture (#128460)
* perf(ci): shard iOS screenshot capture * fix(ci): bind iOS screenshot shard evidence * fix(ci): preserve screenshot capture attempts * fix(ci): align screenshot shards with Xcode 26.6 * fix(ci): provision screenshot shard Fastlane * fix(ci): normalize Fastlane provenance version * fix(ci): select installed Fastlane version
This commit is contained in:
Vincent Koc
@@ -3126,7 +3126,7 @@ NODE
|
||||
expect(codeqlSelect.run).toContain('if [[ "$xcode_version" != 26.6* ]]; then');
|
||||
|
||||
for (const [workflowPath, selectorCount] of [
|
||||
[".github/workflows/ci.yml", 1],
|
||||
[".github/workflows/ci.yml", 2],
|
||||
[".github/workflows/ios-periphery.yml", 1],
|
||||
[".github/workflows/macos-periphery.yml", 1],
|
||||
[".github/workflows/shared-openclawkit-periphery.yml", 2],
|
||||
@@ -3320,6 +3320,7 @@ NODE
|
||||
"control-ui-i18n": "ubuntu-24.04",
|
||||
"docker-seed-e2e": "ubuntu-24.04",
|
||||
"ios-build": "macos-26",
|
||||
"ios-screenshot-shard": "macos-26",
|
||||
"macos-node": "macos-15",
|
||||
"macos-swift": "macos-26",
|
||||
"native-i18n": "ubuntu-24.04",
|
||||
@@ -3346,6 +3347,7 @@ NODE
|
||||
"macos-node": "blacksmith-6vcpu-macos-15",
|
||||
"macos-swift": "blacksmith-12vcpu-macos-26",
|
||||
"ios-build": "blacksmith-12vcpu-macos-26",
|
||||
"ios-screenshot-shard": "blacksmith-12vcpu-macos-26",
|
||||
"check-test-types-hosted-core-shard": "blacksmith-8vcpu-ubuntu-2404",
|
||||
"checks-ui": "blacksmith-8vcpu-ubuntu-2404",
|
||||
"checks-windows": "blacksmith-8vcpu-windows-2025",
|
||||
@@ -5805,10 +5807,16 @@ server.listen(0, "127.0.0.1", () => writeFileSync(readyPath, String(server.addre
|
||||
const workflow = readCiWorkflow();
|
||||
|
||||
expect(source.match(/&platform_checkout_step/gu) ?? []).toHaveLength(1);
|
||||
expect(source.match(/\*platform_checkout_step/gu) ?? []).toHaveLength(3);
|
||||
expect(source.match(/\*platform_checkout_step/gu) ?? []).toHaveLength(4);
|
||||
expect(source.match(/fetch_checkout_ref_once\(\)/gu) ?? []).toHaveLength(1);
|
||||
|
||||
for (const jobName of ["checks-windows", "macos-node", "macos-swift", "ios-build"]) {
|
||||
for (const jobName of [
|
||||
"checks-windows",
|
||||
"macos-node",
|
||||
"macos-swift",
|
||||
"ios-build",
|
||||
"ios-screenshot-shard",
|
||||
]) {
|
||||
const checkoutStep = workflow.jobs[jobName].steps.find(
|
||||
(step: WorkflowStep) => step.name === "Checkout",
|
||||
);
|
||||
@@ -7786,6 +7794,8 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}"
|
||||
"macos-node",
|
||||
"macos-swift",
|
||||
"ios-build",
|
||||
"ios-screenshot-shard",
|
||||
"ios-screenshot-evidence",
|
||||
"android",
|
||||
"docker-seed-e2e",
|
||||
];
|
||||
|
||||
@@ -127,11 +127,13 @@ function swiftFunctionBody(source: string, name: string): string {
|
||||
}
|
||||
|
||||
describe("iOS Fastlane release upload gates", () => {
|
||||
it("pins the CI Ruby and Fastlane toolchain", () => {
|
||||
it("pins the CI Ruby and Fastlane toolchain on the Fastlane-owning screenshot shards", () => {
|
||||
const workflow = readFileSync(ciWorkflowPath, "utf8");
|
||||
const iosJobStart = workflow.indexOf("\n ios-build:\n");
|
||||
const iosJobEnd = workflow.indexOf("\n android:\n", iosJobStart);
|
||||
const iosJobEnd = workflow.indexOf("\n ios-screenshot-shard:\n", iosJobStart);
|
||||
const iosJob = workflow.slice(iosJobStart, iosJobEnd);
|
||||
const shardJobEnd = workflow.indexOf("\n ios-screenshot-evidence:\n", iosJobEnd);
|
||||
const shardJob = workflow.slice(iosJobEnd, shardJobEnd);
|
||||
const gemfile = readFileSync(gemfilePath, "utf8");
|
||||
const lockfile = readFileSync(gemfileLockPath, "utf8");
|
||||
|
||||
@@ -144,16 +146,22 @@ describe("iOS Fastlane release upload gates", () => {
|
||||
expect(lockfile).toContain("CHECKSUMS");
|
||||
expect(lockfile).toContain("RUBY VERSION\n ruby 3.4.10");
|
||||
expect(lockfile).toContain("BUNDLED WITH\n 2.6.9");
|
||||
expect(iosJob).toContain('BUNDLE_DEPLOYMENT: "true"');
|
||||
expect(iosJob).toContain("BUNDLE_GEMFILE: ${{ github.workspace }}/apps/ios/Gemfile");
|
||||
expect(iosJob).toContain("ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b");
|
||||
expect(iosJob).toContain('ruby-version: "3.4.10"');
|
||||
expect(iosJob).toContain('bundler: "2.6.9"');
|
||||
expect(iosJob).toContain("bundler-cache: false");
|
||||
expect(iosJob).toContain("working-directory: apps/ios");
|
||||
expect(iosJob).toContain("bundle _2.6.9_ install --jobs 4 --retry 3");
|
||||
expect(iosJob).toContain("bundle _2.6.9_ check");
|
||||
expect(iosJob).toContain("bundle _2.6.9_ exec fastlane --version");
|
||||
expect(iosJob).not.toContain("BUNDLE_DEPLOYMENT");
|
||||
expect(iosJob).not.toContain("BUNDLE_GEMFILE");
|
||||
expect(iosJob).not.toContain("ruby/setup-ruby@");
|
||||
expect(iosJob).not.toContain("Install locked Fastlane bundle");
|
||||
expect(shardJob).toContain('BUNDLE_DEPLOYMENT: "true"');
|
||||
expect(shardJob).toContain("BUNDLE_GEMFILE: ${{ github.workspace }}/apps/ios/Gemfile");
|
||||
expect(shardJob).toContain("ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b");
|
||||
expect(shardJob).toContain('ruby-version: "3.4.10"');
|
||||
expect(shardJob).toContain('bundler: "2.6.9"');
|
||||
expect(shardJob).toContain("bundler-cache: false");
|
||||
expect(shardJob).toContain("working-directory: apps/ios");
|
||||
expect(shardJob).toContain("bundle _2.6.9_ install --jobs 4 --retry 3");
|
||||
expect(shardJob).toContain("bundle _2.6.9_ check");
|
||||
expect(shardJob).toContain("bundle _2.6.9_ exec fastlane --version");
|
||||
expect(workflow.match(/ruby\/setup-ruby@/gu)).toHaveLength(1);
|
||||
expect(workflow.match(/name: Install locked Fastlane bundle/gu)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("documents every iOS Fastlane command through the pinned bundle", () => {
|
||||
@@ -446,6 +454,8 @@ describe("iOS Fastlane release upload gates", () => {
|
||||
const screenshots = laneBody(fastfile, "screenshots");
|
||||
const capture = functionBody(fastfile, "capture_release_ios_screenshot!");
|
||||
const archive = functionBody(fastfile, "archive_snapshot_test_result!");
|
||||
const attemptRecorder = functionBody(fastfile, "record_release_ios_screenshot_attempt!");
|
||||
const attemptWriter = functionBody(fastfile, "write_release_ios_screenshot_attempts!");
|
||||
const verifier = functionBody(fastfile, "verify_snapshot_test_result!");
|
||||
|
||||
expect(screenshots).toContain("devices = snapshot_devices");
|
||||
@@ -465,11 +475,22 @@ describe("iOS Fastlane release upload gates", () => {
|
||||
expect(capture).toContain("stop_after_first_error: true");
|
||||
expect(capture).toContain("retrying once in a fresh simulator session");
|
||||
expect(capture).toContain("verify_snapshot_test_result!");
|
||||
expect(capture).toContain('capture_outcome: "failed"');
|
||||
expect(capture).toContain('capture_outcome: "succeeded"');
|
||||
expect(capture.indexOf('capture_outcome: "failed"')).toBeLessThan(
|
||||
capture.indexOf("raise if attempt == 2"),
|
||||
);
|
||||
expect(attemptRecorder).toContain('"captureOutcome" => capture_outcome');
|
||||
expect(attemptRecorder).toContain("write_release_ios_screenshot_attempts!(");
|
||||
expect(attemptWriter).toContain('"schemaVersion" => 1');
|
||||
expect(archive).toContain('"#{device}-#{screenshot_name}-attempt-#{attempt}.xcresult"');
|
||||
expect(screenshots).toContain("verify_release_ios_screenshot_manifest!(");
|
||||
expect(screenshots).toContain(
|
||||
'result_bundle_archive_directory = File.join(ios_root, "build", "SnapshotTestResults")',
|
||||
);
|
||||
expect(screenshots).toContain(
|
||||
'capture_attempts_path = File.join(result_bundle_archive_directory, "capture-attempts.json")',
|
||||
);
|
||||
expect(screenshots.indexOf("capture_release_ios_screenshot!")).toBeLessThan(
|
||||
screenshots.indexOf('FileUtils.rm_rf(File.join(output_directory, "test_output"))'),
|
||||
);
|
||||
@@ -515,9 +536,17 @@ describe("iOS Fastlane release upload gates", () => {
|
||||
it("requires the exact nonempty PNG manifest before Watch capture", () => {
|
||||
const fastfile = readFastfile();
|
||||
const screenshots = laneBody(fastfile, "screenshots");
|
||||
const snapshotDevices = functionBody(fastfile, "snapshot_devices");
|
||||
const defaultSnapshotDevices = functionBody(fastfile, "default_snapshot_devices");
|
||||
const verifier = functionBody(fastfile, "verify_release_ios_screenshot_manifest!");
|
||||
|
||||
expect(fastfile).toContain("REQUIRED_IOS_SCREENSHOT_NAMES");
|
||||
expect(snapshotDevices).toContain('ENV["OPENCLAW_SNAPSHOT_DEVICES"]');
|
||||
expect(snapshotDevices).toContain("return default_snapshot_devices if raw.empty?");
|
||||
expect(defaultSnapshotDevices).toContain("available_simulator_devices");
|
||||
expect(defaultSnapshotDevices).toContain('ENV["OPENCLAW_SNAPSHOT_DEVICE_FAMILY"]');
|
||||
expect(defaultSnapshotDevices).toContain("families = DEFAULT_SNAPSHOT_DEVICE_FAMILIES");
|
||||
expect(defaultSnapshotDevices).toContain("families = [family]");
|
||||
expect(verifier).toContain("expected_names - actual_names");
|
||||
expect(verifier).toContain("actual_names - expected_names");
|
||||
expect(verifier).toContain("File.size?(path)");
|
||||
@@ -528,25 +557,76 @@ describe("iOS Fastlane release upload gates", () => {
|
||||
expect(screenshots.indexOf("verify_release_ios_screenshot_manifest!")).toBeLessThan(
|
||||
screenshots.indexOf("watch_screenshot("),
|
||||
);
|
||||
expect(screenshots).toContain('ENV["OPENCLAW_SNAPSHOT_SKIP_WATCH"] == "1"');
|
||||
});
|
||||
|
||||
it("runs the exact screenshot lane during native Apple, manual, and full release CI", () => {
|
||||
it("shards exact screenshot evidence without changing runner authorization", () => {
|
||||
const workflow = readFileSync(ciWorkflowPath, "utf8");
|
||||
const iosJobStart = workflow.indexOf("\n ios-build:\n");
|
||||
const iosJobEnd = workflow.indexOf("\n android:\n", iosJobStart);
|
||||
const iosJobEnd = workflow.indexOf("\n ios-screenshot-shard:\n", iosJobStart);
|
||||
const iosJob = workflow.slice(iosJobStart, iosJobEnd);
|
||||
const shardJobStart = iosJobEnd;
|
||||
const shardJobEnd = workflow.indexOf("\n ios-screenshot-evidence:\n", shardJobStart);
|
||||
const shardJob = workflow.slice(shardJobStart, shardJobEnd);
|
||||
const reducerJobStart = shardJobEnd;
|
||||
const reducerJobEnd = workflow.indexOf("\n android:\n", reducerJobStart);
|
||||
const reducerJob = workflow.slice(reducerJobStart, reducerJobEnd);
|
||||
|
||||
expect(workflow).toContain('IOS_SCREENSHOT_NODE_VERSION: "24.16.0"');
|
||||
expect(workflow).toContain('IOS_SCREENSHOT_XCODE_VERSION: "Xcode 26.6 Build version 17F113"');
|
||||
expect(iosJob).toContain("timeout-minutes: 150");
|
||||
expect(iosJob).toContain("Capture iOS release screenshots");
|
||||
expect(iosJob).toContain("github.event_name == 'workflow_dispatch'");
|
||||
expect(iosJob).toContain("github.event_name == 'pull_request'");
|
||||
expect(iosJob).toContain("inputs.release_gate");
|
||||
expect(iosJob).toContain("needs.preflight.outputs.run_ios_screenshots == 'true'");
|
||||
expect(iosJob).not.toContain("needs.preflight.outputs.run_macos == 'true'");
|
||||
expect(iosJob).toContain("run: pnpm ios:screenshots");
|
||||
expect(iosJob).toContain("Upload iOS release screenshot evidence");
|
||||
expect(iosJob).toContain("apps/ios/build/SnapshotTestResults/*.xcresult");
|
||||
expect(iosJob).toContain("if-no-files-found: error");
|
||||
expect(iosJob).not.toContain("Capture iOS release screenshots");
|
||||
expect(shardJob).toContain("needs: [preflight, ios-build]");
|
||||
expect(shardJob).toContain("max-parallel: 2");
|
||||
expect(shardJob).toContain("device_family: [iphone, ipad-13]");
|
||||
expect(shardJob).toContain('OPENCLAW_SNAPSHOT_SKIP_WATCH: "1"');
|
||||
expect(shardJob).toContain("if: matrix.device_family == 'iphone'");
|
||||
expect(shardJob).toContain("run_ios_fastlane ios watch_screenshot");
|
||||
expect(shardJob).toContain("run: pnpm ios:screenshots");
|
||||
expect(shardJob).toContain("id: package_screenshot_evidence");
|
||||
expect(shardJob).toContain("steps.package_screenshot_evidence.outcome == 'failure'");
|
||||
expect(shardJob).toContain("apps/ios/build/SnapshotTestResults/capture-attempts.json");
|
||||
expect(shardJob).not.toContain("IOS_SCREENSHOT_FASTLANE_VERSION");
|
||||
expect(shardJob).toContain("IOS_SCREENSHOT_NODE_VERSION");
|
||||
expect(shardJob).toContain("IOS_SCREENSHOT_XCODE_VERSION");
|
||||
expect(shardJob).not.toContain('test "$fastlane_version" = "$IOS_SCREENSHOT_FASTLANE_VERSION"');
|
||||
expect(shardJob).toContain("node-version: ${{ env.IOS_SCREENSHOT_NODE_VERSION }}");
|
||||
expect(shardJob).not.toContain("SnapshotDerivedData");
|
||||
expect(shardJob.match(/contents: read/g)).toHaveLength(1);
|
||||
expect(reducerJob).toContain("needs: [preflight, ios-screenshot-shard]");
|
||||
expect(reducerJob).toContain("merge-multiple: false");
|
||||
expect(reducerJob).toContain("Setup screenshot evidence Node");
|
||||
expect(reducerJob).toContain("node-version: ${{ env.IOS_SCREENSHOT_NODE_VERSION }}");
|
||||
expect(reducerJob).toContain("id: reduce_screenshot_evidence");
|
||||
expect(reducerJob).toContain("scripts/ios-screenshot-evidence.mjs reduce");
|
||||
expect(reducerJob).toContain('--workflow-sha "$WORKFLOW_SHA"');
|
||||
expect(reducerJob).toContain('--run-id "$RUN_ID"');
|
||||
expect(reducerJob).toContain('--run-attempt "$RUN_ATTEMPT"');
|
||||
expect(reducerJob).toContain('--xcode-version "$IOS_SCREENSHOT_XCODE_VERSION"');
|
||||
expect(reducerJob).toContain('--fastlane-version "$IOS_SCREENSHOT_FASTLANE_VERSION"');
|
||||
expect(reducerJob).toContain('--node-version "$(node --version)"');
|
||||
expect(reducerJob).toContain("steps.reduce_screenshot_evidence.outcome == 'failure'");
|
||||
expect(reducerJob).toContain("apps/ios/build/ScreenshotEvidenceInputs/**/xcresults/*.xcresult");
|
||||
expect(reducerJob).toContain(
|
||||
"name: ios-release-screenshots-${{ needs.preflight.outputs.checkout_revision }}",
|
||||
);
|
||||
expect(reducerJob).toContain("apps/ios/build/ScreenshotEvidence/manifest.json");
|
||||
expect(reducerJob).not.toContain("SnapshotDerivedData");
|
||||
});
|
||||
|
||||
it("records the installed Fastlane semver before the update notice", () => {
|
||||
const workflow = readFileSync(ciWorkflowPath, "utf8");
|
||||
const parser = workflow.match(/run_ios_fastlane --version 2>&1 \| awk '([^']+)'/u)?.[1];
|
||||
|
||||
expect(parser).toBeDefined();
|
||||
const result = spawnSync("awk", [parser!], {
|
||||
encoding: "utf8",
|
||||
input:
|
||||
"fastlane installation at path:\nfastlane 2.236.1\n# fastlane 2.238.0 is available. You are on 2.236.1.\n",
|
||||
});
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toBe("2.236.1\n");
|
||||
});
|
||||
|
||||
it("preserves caller-pinned Swift tools in archive build PATH", () => {
|
||||
|
||||
@@ -0,0 +1,563 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
collectIosScreenshotEvidence,
|
||||
reduceIosScreenshotEvidence,
|
||||
} from "../../scripts/ios-screenshot-evidence.mjs";
|
||||
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
const TARGET_SHA = "a".repeat(40);
|
||||
const WORKFLOW_SHA = "b".repeat(40);
|
||||
const PNG = Buffer.concat([
|
||||
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
|
||||
Buffer.from("fixture"),
|
||||
]);
|
||||
const SCREENSHOTS = [
|
||||
"01-control-connected",
|
||||
"02-chat-connected",
|
||||
"03-agent-connected",
|
||||
"04-settings-connected",
|
||||
];
|
||||
const ATTEMPT_MODEL = {
|
||||
owner: "openclaw",
|
||||
unit: "capture_ios_screenshots invocation",
|
||||
maxAttempts: 2,
|
||||
fastlaneInternalRetries: "workflow-log",
|
||||
};
|
||||
type Family = "iphone" | "ipad-13" | "watch";
|
||||
|
||||
function provenance(targetSha = TARGET_SHA) {
|
||||
return {
|
||||
targetSha,
|
||||
workflowSha: WORKFLOW_SHA,
|
||||
runId: "12345",
|
||||
runAttempt: 2,
|
||||
tooling: {
|
||||
xcode: "Xcode 26.6 Build version 17F113",
|
||||
fastlane: "2.236.1",
|
||||
node: "v24.16.0",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function writeFamilySource(
|
||||
root: string,
|
||||
family: Family,
|
||||
options: {
|
||||
retry?: string;
|
||||
retryTestResult?: "fail" | "pass";
|
||||
retryWithoutXcresult?: boolean;
|
||||
} = {},
|
||||
) {
|
||||
const screenshots = path.join(root, family, "screenshots");
|
||||
const xcresults = path.join(root, family, "xcresults");
|
||||
fs.mkdirSync(screenshots, { recursive: true });
|
||||
fs.mkdirSync(xcresults, { recursive: true });
|
||||
const device =
|
||||
family === "iphone"
|
||||
? "iPhone 17 Pro Max"
|
||||
: family === "ipad-13"
|
||||
? "iPad Pro 13-inch (M5)"
|
||||
: "Apple Watch Ultra 3 (49mm)";
|
||||
const names = family === "watch" ? ["01-now-face"] : SCREENSHOTS;
|
||||
const captureAttempts = [];
|
||||
for (const name of names) {
|
||||
fs.writeFileSync(path.join(screenshots, `${device}-${name}.png`), PNG);
|
||||
if (family !== "watch") {
|
||||
const retried = options.retry === name;
|
||||
captureAttempts.push({
|
||||
deviceName: device,
|
||||
screenshotName: name,
|
||||
attempt: 1,
|
||||
captureOutcome: retried ? "failed" : "succeeded",
|
||||
});
|
||||
if (!(retried && options.retryWithoutXcresult)) {
|
||||
const attemptOne = path.join(xcresults, `${device}-${name}-attempt-1.xcresult`);
|
||||
fs.mkdirSync(attemptOne, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(attemptOne, "summary.txt"),
|
||||
retried ? (options.retryTestResult ?? "pass") : "pass",
|
||||
);
|
||||
}
|
||||
if (options.retry === name) {
|
||||
captureAttempts.push({
|
||||
deviceName: device,
|
||||
screenshotName: name,
|
||||
attempt: 2,
|
||||
captureOutcome: "succeeded",
|
||||
});
|
||||
const attemptTwo = path.join(xcresults, `${device}-${name}-attempt-2.xcresult`);
|
||||
fs.mkdirSync(attemptTwo, { recursive: true });
|
||||
fs.writeFileSync(path.join(attemptTwo, "summary.txt"), "pass");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (family !== "watch") {
|
||||
fs.writeFileSync(
|
||||
path.join(xcresults, "capture-attempts.json"),
|
||||
JSON.stringify({ schemaVersion: 1, attempts: captureAttempts }),
|
||||
);
|
||||
}
|
||||
return { device, screenshots, xcresults };
|
||||
}
|
||||
|
||||
function containerName(family: Family, targetSha = TARGET_SHA) {
|
||||
const shard = family === "watch" ? "iphone" : family;
|
||||
return `ios-release-screenshot-shard-${shard}-${targetSha}`;
|
||||
}
|
||||
|
||||
function familyDirectory(input: string, family: Family, targetSha = TARGET_SHA) {
|
||||
return path.join(input, containerName(family, targetSha), family);
|
||||
}
|
||||
|
||||
function manifestPath(input: string, family: Family, targetSha = TARGET_SHA) {
|
||||
return path.join(familyDirectory(input, family, targetSha), "manifest.json");
|
||||
}
|
||||
|
||||
function collectAll(
|
||||
root: string,
|
||||
targetSha = TARGET_SHA,
|
||||
options: { retryWithoutXcresult?: boolean } = {},
|
||||
) {
|
||||
const output = path.join(root, "collected");
|
||||
for (const family of ["iphone", "ipad-13", "watch"] as const) {
|
||||
const source = writeFamilySource(root, family, {
|
||||
retry: family === "iphone" ? "02-chat-connected" : undefined,
|
||||
retryWithoutXcresult: family === "iphone" && options.retryWithoutXcresult,
|
||||
});
|
||||
collectIosScreenshotEvidence({
|
||||
family,
|
||||
screenshotDirectory: source.screenshots,
|
||||
xcresultDirectory: source.xcresults,
|
||||
outputDirectory: path.join(output, containerName(family, targetSha)),
|
||||
provenance: provenance(targetSha),
|
||||
readXcresultSummary: (resultPath) => {
|
||||
const result = fs.readFileSync(path.join(resultPath, "summary.txt"), "utf8");
|
||||
return result === "pass"
|
||||
? { testResult: "Passed", failedTests: 0 }
|
||||
: { testResult: "Failed", failedTests: 1 };
|
||||
},
|
||||
});
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function reduceAll(input: string, outputRoot: string, expected = provenance()) {
|
||||
return reduceIosScreenshotEvidence({
|
||||
inputDirectory: input,
|
||||
outputRoot,
|
||||
expectedProvenance: expected,
|
||||
});
|
||||
}
|
||||
|
||||
function updateManifest(
|
||||
input: string,
|
||||
family: Family,
|
||||
mutate: (manifest: Record<string, any>) => void,
|
||||
) {
|
||||
const filePath = manifestPath(input, family);
|
||||
const manifest = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
||||
mutate(manifest);
|
||||
fs.writeFileSync(filePath, JSON.stringify(manifest));
|
||||
}
|
||||
|
||||
function updateAllManifests(input: string, mutate: (manifest: Record<string, any>) => void) {
|
||||
for (const family of ["iphone", "ipad-13", "watch"] as const) {
|
||||
updateManifest(input, family, mutate);
|
||||
}
|
||||
}
|
||||
|
||||
describe("iOS screenshot evidence", () => {
|
||||
it("reduces the exact device union and models passed retry xcresults by capture outcome", () => {
|
||||
const root = tempDirs.make("ios-screenshot-evidence-");
|
||||
const input = collectAll(root);
|
||||
const output = path.join(root, "reduced");
|
||||
|
||||
const manifest = reduceAll(input, output);
|
||||
const iphoneManifest = JSON.parse(fs.readFileSync(manifestPath(input, "iphone"), "utf8"));
|
||||
const retryAttempts = iphoneManifest.captureAttempts.filter(
|
||||
(entry: { screenshotName: string }) => entry.screenshotName === "02-chat-connected",
|
||||
);
|
||||
|
||||
expect(manifest.targetSha).toBe(TARGET_SHA);
|
||||
expect(manifest.attemptModel).toEqual(ATTEMPT_MODEL);
|
||||
expect(retryAttempts.map((entry: { captureOutcome: string }) => entry.captureOutcome)).toEqual([
|
||||
"failed",
|
||||
"succeeded",
|
||||
]);
|
||||
expect(retryAttempts.map((entry: { testResult: string }) => entry.testResult)).toEqual([
|
||||
"Passed",
|
||||
"Passed",
|
||||
]);
|
||||
expect(fs.readdirSync(path.join(output, "apps/ios/fastlane/screenshots/en-US"))).toHaveLength(
|
||||
9,
|
||||
);
|
||||
expect(
|
||||
fs.existsSync(
|
||||
path.join(
|
||||
output,
|
||||
"apps/ios/build/SnapshotTestResults",
|
||||
"iPhone 17 Pro Max-02-chat-connected-attempt-1.xcresult",
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
fs.existsSync(
|
||||
path.join(
|
||||
output,
|
||||
"apps/ios/build/SnapshotTestResults",
|
||||
"iPhone 17 Pro Max-02-chat-connected-attempt-2.xcresult",
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("accepts a failed first invocation without an xcresult before a passing retry", () => {
|
||||
const root = tempDirs.make("ios-screenshot-missing-retry-xcresult-");
|
||||
const input = collectAll(root, TARGET_SHA, { retryWithoutXcresult: true });
|
||||
const output = path.join(root, "reduced");
|
||||
|
||||
reduceAll(input, output);
|
||||
const iphoneManifest = JSON.parse(fs.readFileSync(manifestPath(input, "iphone"), "utf8"));
|
||||
const retryAttempts = iphoneManifest.captureAttempts.filter(
|
||||
(entry: { screenshotName: string }) => entry.screenshotName === "02-chat-connected",
|
||||
);
|
||||
|
||||
expect(
|
||||
retryAttempts.map(
|
||||
(entry: { attempt: number; captureOutcome: string; artifactPath: string | null }) => ({
|
||||
attempt: entry.attempt,
|
||||
captureOutcome: entry.captureOutcome,
|
||||
artifactPath: entry.artifactPath,
|
||||
}),
|
||||
),
|
||||
).toEqual([
|
||||
{ attempt: 1, captureOutcome: "failed", artifactPath: null },
|
||||
{
|
||||
attempt: 2,
|
||||
captureOutcome: "succeeded",
|
||||
artifactPath: "xcresults/iPhone 17 Pro Max-02-chat-connected-attempt-2.xcresult",
|
||||
},
|
||||
]);
|
||||
expect(
|
||||
fs.existsSync(
|
||||
path.join(
|
||||
output,
|
||||
"apps/ios/build/SnapshotTestResults",
|
||||
"iPhone 17 Pro Max-02-chat-connected-attempt-1.xcresult",
|
||||
),
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
fs.existsSync(
|
||||
path.join(
|
||||
output,
|
||||
"apps/ios/build/SnapshotTestResults",
|
||||
"iPhone 17 Pro Max-02-chat-connected-attempt-2.xcresult",
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("requires the successful final invocation to have a passing xcresult", () => {
|
||||
const root = tempDirs.make("ios-screenshot-missing-final-xcresult-");
|
||||
const source = writeFamilySource(root, "iphone", { retry: "02-chat-connected" });
|
||||
fs.rmSync(
|
||||
path.join(source.xcresults, `${source.device}-02-chat-connected-attempt-2.xcresult`),
|
||||
{ recursive: true },
|
||||
);
|
||||
|
||||
expect(() =>
|
||||
collectIosScreenshotEvidence({
|
||||
family: "iphone",
|
||||
screenshotDirectory: source.screenshots,
|
||||
xcresultDirectory: source.xcresults,
|
||||
outputDirectory: path.join(root, "collected"),
|
||||
provenance: provenance(),
|
||||
readXcresultSummary: () => ({ testResult: "Passed", failedTests: 0 }),
|
||||
}),
|
||||
).toThrow("is missing for the successful final capture attempt");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "extra",
|
||||
mutate: (input: string) =>
|
||||
fs.cpSync(
|
||||
path.join(input, containerName("ipad-13")),
|
||||
path.join(input, `ios-release-screenshot-shard-extra-${TARGET_SHA}`),
|
||||
{ recursive: true },
|
||||
),
|
||||
},
|
||||
{
|
||||
label: "renamed",
|
||||
mutate: (input: string) =>
|
||||
fs.renameSync(
|
||||
path.join(input, containerName("iphone")),
|
||||
path.join(input, `ios-release-screenshot-shard-renamed-${TARGET_SHA}`),
|
||||
),
|
||||
},
|
||||
{
|
||||
label: "missing",
|
||||
mutate: (input: string) =>
|
||||
fs.rmSync(path.join(input, containerName("ipad-13")), { recursive: true }),
|
||||
},
|
||||
{
|
||||
label: "swapped",
|
||||
mutate: (input: string) => {
|
||||
const iphone = path.join(input, containerName("iphone"));
|
||||
const ipad = path.join(input, containerName("ipad-13"));
|
||||
const temporary = path.join(input, "temporary-container");
|
||||
fs.renameSync(iphone, temporary);
|
||||
fs.renameSync(ipad, iphone);
|
||||
fs.renameSync(temporary, ipad);
|
||||
},
|
||||
},
|
||||
])("rejects $label artifact container topology", ({ mutate }) => {
|
||||
const root = tempDirs.make("ios-screenshot-topology-");
|
||||
const input = collectAll(root);
|
||||
mutate(input);
|
||||
|
||||
expect(() => reduceAll(input, path.join(root, "reduced"))).toThrow(/topology mismatch/u);
|
||||
});
|
||||
|
||||
it("rejects cross-SHA shard evidence", () => {
|
||||
const root = tempDirs.make("ios-screenshot-cross-sha-");
|
||||
const input = collectAll(root);
|
||||
updateManifest(input, "ipad-13", (manifest) => {
|
||||
manifest.targetSha = "c".repeat(40);
|
||||
});
|
||||
|
||||
expect(() => reduceAll(input, path.join(root, "reduced"))).toThrow(
|
||||
"cross-SHA screenshot evidence",
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "workflow SHA",
|
||||
mutate: (manifest: Record<string, any>) => {
|
||||
manifest.workflowSha = "c".repeat(40);
|
||||
},
|
||||
error: "workflow SHA",
|
||||
},
|
||||
{
|
||||
label: "run id",
|
||||
mutate: (manifest: Record<string, any>) => {
|
||||
manifest.runId = "99999";
|
||||
},
|
||||
error: "workflow run id",
|
||||
},
|
||||
{
|
||||
label: "run attempt",
|
||||
mutate: (manifest: Record<string, any>) => {
|
||||
manifest.runAttempt = 3;
|
||||
},
|
||||
error: "workflow run attempt",
|
||||
},
|
||||
{
|
||||
label: "Xcode version",
|
||||
mutate: (manifest: Record<string, any>) => {
|
||||
manifest.tooling.xcode = "Xcode 26.6 Build version forged";
|
||||
},
|
||||
error: "xcode version",
|
||||
},
|
||||
{
|
||||
label: "Fastlane version",
|
||||
mutate: (manifest: Record<string, any>) => {
|
||||
manifest.tooling.fastlane = "2.236.0";
|
||||
},
|
||||
error: "fastlane version",
|
||||
},
|
||||
{
|
||||
label: "Node version",
|
||||
mutate: (manifest: Record<string, any>) => {
|
||||
manifest.tooling.node = "v24.0.0";
|
||||
},
|
||||
error: "node version",
|
||||
},
|
||||
])("rejects self-consistent forged $label", ({ mutate, error }) => {
|
||||
const root = tempDirs.make("ios-screenshot-forged-provenance-");
|
||||
const input = collectAll(root);
|
||||
updateAllManifests(input, mutate);
|
||||
|
||||
expect(() => reduceAll(input, path.join(root, "reduced"))).toThrow(error);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "missing",
|
||||
mutate: (manifest: Record<string, any>) => {
|
||||
delete manifest.attemptModel;
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "owner",
|
||||
mutate: (manifest: Record<string, any>) => {
|
||||
manifest.attemptModel.owner = "fastlane";
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "unit",
|
||||
mutate: (manifest: Record<string, any>) => {
|
||||
manifest.attemptModel.unit = "launch retry";
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "maximum",
|
||||
mutate: (manifest: Record<string, any>) => {
|
||||
manifest.attemptModel.maxAttempts = 3;
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Fastlane retry ownership",
|
||||
mutate: (manifest: Record<string, any>) => {
|
||||
manifest.attemptModel.fastlaneInternalRetries = "xcresult";
|
||||
},
|
||||
},
|
||||
])("rejects $label attempt model changes", ({ mutate }) => {
|
||||
const root = tempDirs.make("ios-screenshot-attempt-model-");
|
||||
const input = collectAll(root);
|
||||
updateManifest(input, "iphone", mutate);
|
||||
|
||||
expect(() => reduceAll(input, path.join(root, "reduced"))).toThrow("unexpected attempt model");
|
||||
});
|
||||
|
||||
it("rejects changed PNG bytes after collection", () => {
|
||||
const root = tempDirs.make("ios-screenshot-digest-");
|
||||
const input = collectAll(root);
|
||||
const screenshot = path.join(
|
||||
familyDirectory(input, "watch"),
|
||||
"screenshots",
|
||||
"Apple Watch Ultra 3 (49mm)-01-now-face.png",
|
||||
);
|
||||
fs.appendFileSync(screenshot, "changed");
|
||||
|
||||
expect(() => reduceAll(input, path.join(root, "reduced"))).toThrow(
|
||||
"screenshot digest mismatch",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects changed xcresult bytes after collection", () => {
|
||||
const root = tempDirs.make("ios-screenshot-xcresult-digest-");
|
||||
const input = collectAll(root);
|
||||
fs.appendFileSync(
|
||||
path.join(
|
||||
familyDirectory(input, "iphone"),
|
||||
"xcresults",
|
||||
"iPhone 17 Pro Max-01-control-connected-attempt-1.xcresult",
|
||||
"summary.txt",
|
||||
),
|
||||
"changed",
|
||||
);
|
||||
|
||||
expect(() => reduceAll(input, path.join(root, "reduced"))).toThrow("xcresult digest mismatch");
|
||||
});
|
||||
|
||||
it("rejects a non-passing final capture attempt", () => {
|
||||
const root = tempDirs.make("ios-screenshot-final-failure-");
|
||||
const input = collectAll(root);
|
||||
updateManifest(input, "ipad-13", (manifest) => {
|
||||
const final = manifest.captureAttempts.find(
|
||||
(entry: { screenshotName: string }) => entry.screenshotName === "01-control-connected",
|
||||
);
|
||||
final.testResult = "Failed";
|
||||
final.failedTests = 1;
|
||||
});
|
||||
|
||||
expect(() => reduceAll(input, path.join(root, "reduced"))).toThrow(
|
||||
"final xcresult is not passing",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a successful capture predecessor before attempt two", () => {
|
||||
const root = tempDirs.make("ios-screenshot-predecessor-");
|
||||
const input = collectAll(root);
|
||||
updateManifest(input, "iphone", (manifest) => {
|
||||
const predecessor = manifest.captureAttempts.find(
|
||||
(entry: { attempt: number; screenshotName: string }) =>
|
||||
entry.screenshotName === "02-chat-connected" && entry.attempt === 1,
|
||||
);
|
||||
predecessor.captureOutcome = "succeeded";
|
||||
});
|
||||
|
||||
expect(() => reduceAll(input, path.join(root, "reduced"))).toThrow(
|
||||
"unexpected capture outcome sequence",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an invalid PNG signature before collection", () => {
|
||||
const root = tempDirs.make("ios-screenshot-signature-");
|
||||
const source = writeFamilySource(root, "watch");
|
||||
fs.writeFileSync(
|
||||
path.join(source.screenshots, `${source.device}-01-now-face.png`),
|
||||
"not a png",
|
||||
);
|
||||
|
||||
expect(() =>
|
||||
collectIosScreenshotEvidence({
|
||||
family: "watch",
|
||||
screenshotDirectory: source.screenshots,
|
||||
xcresultDirectory: source.xcresults,
|
||||
outputDirectory: path.join(root, "collected"),
|
||||
provenance: provenance(),
|
||||
}),
|
||||
).toThrow("invalid PNG signature");
|
||||
});
|
||||
|
||||
it("rejects unexpected screenshots in a device shard", () => {
|
||||
const root = tempDirs.make("ios-screenshot-unexpected-");
|
||||
const source = writeFamilySource(root, "iphone");
|
||||
fs.writeFileSync(path.join(source.screenshots, `${source.device}-99-unexpected.png`), PNG);
|
||||
|
||||
expect(() =>
|
||||
collectIosScreenshotEvidence({
|
||||
family: "iphone",
|
||||
screenshotDirectory: source.screenshots,
|
||||
xcresultDirectory: source.xcresults,
|
||||
outputDirectory: path.join(root, "collected"),
|
||||
provenance: provenance(),
|
||||
readXcresultSummary: () => ({ testResult: "Passed", failedTests: 0 }),
|
||||
}),
|
||||
).toThrow("PNG union mismatch");
|
||||
});
|
||||
|
||||
it("rejects unexpected capture attempt entries in a shard manifest", () => {
|
||||
const root = tempDirs.make("ios-screenshot-unexpected-xcresult-");
|
||||
const input = collectAll(root);
|
||||
updateManifest(input, "iphone", (manifest) => {
|
||||
manifest.captureAttempts.push({
|
||||
...manifest.captureAttempts[0],
|
||||
screenshotName: "99-unexpected",
|
||||
});
|
||||
});
|
||||
|
||||
expect(() => reduceAll(input, path.join(root, "reduced"))).toThrow(
|
||||
"capture attempt union contains an unexpected screenshot",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects unexpected files in a collected shard", () => {
|
||||
const root = tempDirs.make("ios-screenshot-unexpected-file-");
|
||||
const input = collectAll(root);
|
||||
fs.writeFileSync(path.join(familyDirectory(input, "watch"), "unexpected.txt"), "unexpected");
|
||||
|
||||
expect(() => reduceAll(input, path.join(root, "reduced"))).toThrow(
|
||||
"shard contains unexpected evidence",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects artifact paths outside the declared family directory", () => {
|
||||
const root = tempDirs.make("ios-screenshot-artifact-path-");
|
||||
const input = collectAll(root);
|
||||
updateManifest(input, "watch", (manifest) => {
|
||||
manifest.screenshots[0].artifactPath = "../iphone/manifest.json";
|
||||
});
|
||||
|
||||
expect(() => reduceAll(input, path.join(root, "reduced"))).toThrow(
|
||||
"unexpected screenshot artifact path",
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user