ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(ios): isolate release screenshot captures

Browse Source
This commit is contained in:
joshavant
2026-07-23 19:40:51 -05:00
committed by Josh Avant
parent 6885d5b943
commit ee317497d9
4 changed files with 227 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+2 -2
View File
@@ -3228,11 +3228,11 @@ jobs:
run: pnpm ios:build
- name: Capture iOS release screenshots
if: ${{ github.event_name == 'workflow_dispatch' && env.HISTORICAL_TARGET != 'true' }}
if: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request') && env.HISTORICAL_TARGET != 'true' }}
run: pnpm ios:screenshots
- name: Upload iOS release screenshot evidence
if: ${{ always() && github.event_name == 'workflow_dispatch' && env.HISTORICAL_TARGET != 'true' }}
if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request') && env.HISTORICAL_TARGET != 'true' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: ios-release-screenshots-${{ needs.preflight.outputs.checkout_revision }}
apps/ios/UITests/OpenClawSnapshotUITests.swift
+56 -116
View File
@@ -9,15 +9,23 @@ final class OpenClawSnapshotUITests: XCTestCase {
let name: String
}
private static let screenshotTargets = [
ScreenshotTarget(initialTab: "control", initialDestination: "overview", name: "01-control-connected"),
ScreenshotTarget(initialTab: "chat", initialDestination: "chat", name: "02-chat-connected"),
ScreenshotTarget(initialTab: "agent", initialDestination: "agents", name: "03-agent-connected"),
ScreenshotTarget(initialTab: "settings", initialDestination: "settings", name: "04-settings-connected"),
]
private static let controlScreenshotTarget = ScreenshotTarget(
initialTab: "control",
initialDestination: "overview",
name: "01-control-connected")
private static let chatScreenshotTarget = ScreenshotTarget(
initialTab: "chat",
initialDestination: "chat",
name: "02-chat-connected")
private static let agentScreenshotTarget = ScreenshotTarget(
initialTab: "agent",
initialDestination: "agents",
name: "03-agent-connected")
private static let settingsScreenshotTarget = ScreenshotTarget(
initialTab: "settings",
initialDestination: "settings",
name: "04-settings-connected")
private static let appReadinessAccessibilityIdentifier = "RootTabs.Ready"
private static let releaseScreenshotLaunchArguments = ["-sidebar.pinnedPages", "overview,agents"]
private static let screenshotLaunchRetryThreshold: TimeInterval = 30
private var app: XCUIApplication?
@@ -31,19 +39,41 @@ final class OpenClawSnapshotUITests: XCTestCase {
try super.tearDownWithError()
}
func testConnectedGatewayTabs() throws {
let initialTarget = try XCTUnwrap(Self.screenshotTargets.first)
self.launchApp(
for: initialTarget,
additionalArguments: Self.releaseScreenshotLaunchArguments)
func testReleaseControlScreenshot() {
self.captureReleaseScreenshot(Self.controlScreenshotTarget)
}
for (index, target) in Self.screenshotTargets.enumerated() {
if index > 0 {
try self.selectReleaseScreenshotDestination(target)
func testReleaseChatScreenshot() {
self.captureReleaseScreenshot(Self.chatScreenshotTarget)
}
func testReleaseAgentScreenshot() {
self.captureReleaseScreenshot(Self.agentScreenshotTarget)
}
func testReleaseSettingsScreenshot() {
self.captureReleaseScreenshot(Self.settingsScreenshotTarget)
}
func testAgentsNavigateToSettingsThroughSidebar() throws {
try XCTSkipIf(UIDevice.current.userInterfaceIdiom != .phone, "Phone sidebar navigation only")
self.launchApp(for: Self.agentScreenshotTarget)
XCTAssertTrue(self.app?.buttons["agent-status-filter-menu"].waitForExistence(timeout: 8) == true)
let options = XCTExpectedFailure.Options()
options.isStrict = false
XCTExpectFailure(
"Agents-to-Settings navigation can wedge XCTest quiescence; keep it separate from release capture.",
options: options)
{
do {
try self.selectSidebarDestination("Settings")
} catch {
XCTFail("Agents-to-Settings navigation setup failed: \(error)")
}
self.waitForReleaseScreenshotTarget(target)
snapshot(target.name, timeWaitingForIdle: 5)
self.attachScreenshot(named: target.name)
XCTAssertTrue(
self.app?.descendants(matching: .any)["settings-system-agent-row"]
.waitForExistence(timeout: 8) == true)
}
}
@@ -929,40 +959,12 @@ final class OpenClawSnapshotUITests: XCTestCase {
{
self.terminateCurrentApp()
var app = self.configuredApp(
let app = self.configuredApp(
for: target,
appearance: appearance,
screenshotMode: screenshotMode,
additionalArguments: additionalArguments)
let launchDuration = self.launchDuration(app)
if screenshotMode,
launchDuration >= Self.screenshotLaunchRetryThreshold || app.state != .runningForeground
{
// A slow XCUIApplication launch can return with the scene inactive and its
// accessibility tree wedged. Recover before making any element query.
self.attachStalledLaunchScreenshot(duration: launchDuration)
app.terminate()
guard app.wait(for: .notRunning, timeout: 5) else {
self.app = app
XCTFail("OpenClaw did not terminate after a stalled screenshot launch")
return
}
app = self.configuredApp(
for: target,
appearance: appearance,
screenshotMode: screenshotMode,
additionalArguments: additionalArguments)
let retryDuration = self.launchDuration(app)
guard retryDuration < Self.screenshotLaunchRetryThreshold,
app.state == .runningForeground
else {
self.app = app
XCTFail("Screenshot app launch stalled again after one recovery attempt")
return
}
}
app.launch()
self.app = app
XCTAssertTrue(app.wait(for: .runningForeground, timeout: 8))
let readiness = app.descendants(matching: .any)[Self.appReadinessAccessibilityIdentifier]
@@ -999,19 +1001,11 @@ final class OpenClawSnapshotUITests: XCTestCase {
return app
}
private func launchDuration(_ app: XCUIApplication) -> TimeInterval {
let startedAt = Date()
app.launch()
return Date().timeIntervalSince(startedAt)
}
private func attachStalledLaunchScreenshot(duration: TimeInterval) {
XCTContext.runActivity(named: "Recover stalled screenshot launch") { activity in
let attachment = XCTAttachment(screenshot: XCUIScreen.main.screenshot())
attachment.name = "stalled-screenshot-launch-\(Int(duration.rounded()))s"
attachment.lifetime = .keepAlways
activity.add(attachment)
}
private func captureReleaseScreenshot(_ target: ScreenshotTarget) {
self.launchApp(for: target)
self.waitForReleaseScreenshotTarget(target)
snapshot(target.name, timeWaitingForIdle: 5)
self.attachScreenshot(named: target.name)
}
private func waitForReleaseScreenshotTarget(_ target: ScreenshotTarget) {
@@ -1034,60 +1028,6 @@ final class OpenClawSnapshotUITests: XCTestCase {
"Screenshot target \(target.name) did not render its readiness anchor")
}
private func selectReleaseScreenshotDestination(
_ target: ScreenshotTarget,
file: StaticString = #filePath,
line: UInt = #line) throws
{
let app = try XCTUnwrap(self.app, file: file, line: line)
let destination = target.initialDestination
let destinationButton = app.buttons["RootTabs.Sidebar.Destination.\(destination)"]
let showSidebar = app.buttons["RootTabs.Sidebar.Show"]
let hideSidebar = app.buttons["RootTabs.Sidebar.Hide"]
if showSidebar.isHittable {
showSidebar.tap()
self.waitForHittable(true, of: hideSidebar)
} else if !destinationButton.isHittable {
XCTAssertTrue(showSidebar.waitForExistence(timeout: 5), file: file, line: line)
XCTAssertTrue(showSidebar.isHittable, file: file, line: line)
showSidebar.tap()
self.waitForHittable(true, of: hideSidebar)
}
XCTAssertTrue(destinationButton.waitForExistence(timeout: 5), file: file, line: line)
XCTAssertTrue(destinationButton.isHittable, file: file, line: line)
let transitionStartedAt = Date()
destinationButton.tap()
let transitionDuration = Date().timeIntervalSince(transitionStartedAt)
let readiness = app.descendants(matching: .any)[Self.appReadinessAccessibilityIdentifier]
let reachedDestination = transitionDuration < Self.screenshotLaunchRetryThreshold &&
app.state == .runningForeground &&
self.element(readiness, hasValue: "ready:\(destination)", timeout: 8)
if !reachedDestination {
// XCTest can spend its full idle timeout after a synthesized navigation
// event. Relaunch only the affected destination, before another AX query.
self.attachStalledTransitionScreenshot(target: target, duration: transitionDuration)
self.launchApp(for: target, additionalArguments: Self.releaseScreenshotLaunchArguments)
return
}
// Drawer layouts collapse after selection. Split layouts do not, so close
// them explicitly to keep every App Store capture sidebar-free.
if hideSidebar.isHittable {
hideSidebar.tap()
}
self.waitForHittable(true, of: showSidebar)
}
private func attachStalledTransitionScreenshot(target: ScreenshotTarget, duration: TimeInterval) {
XCTContext.runActivity(named: "Recover stalled screenshot transition") { activity in
let attachment = XCTAttachment(screenshot: XCUIScreen.main.screenshot())
attachment.name = "stalled-\(target.initialDestination)-transition-\(Int(duration.rounded()))s"
attachment.lifetime = .keepAlways
activity.add(attachment)
}
}
private func terminateCurrentApp(
file: StaticString = #filePath,
line: UInt = #line)
apps/ios/fastlane/Fastfile
+123 -33
View File
@@ -39,12 +39,20 @@ REQUIRED_SCREENSHOT_FAMILIES = {
"iPhone" => /iPhone/,
"13-inch iPad" => /iPad (Air|Pro) 13-inch/
}.freeze
REQUIRED_IOS_SCREENSHOT_NAMES = [
"01-control-connected",
"02-chat-connected",
"03-agent-connected",
"04-settings-connected"
RELEASE_IOS_SCREENSHOT_TESTS = [
{ test: "testReleaseControlScreenshot", name: "01-control-connected" },
{ test: "testReleaseChatScreenshot", name: "02-chat-connected" },
{ test: "testReleaseAgentScreenshot", name: "03-agent-connected" },
{ test: "testReleaseSettingsScreenshot", name: "04-settings-connected" }
].freeze
REQUIRED_IOS_SCREENSHOT_NAMES = RELEASE_IOS_SCREENSHOT_TESTS.map { |entry| entry.fetch(:name) }.freeze
IOS_SCREENSHOT_TEST_TIMEOUT_SECONDS = 180
IOS_SCREENSHOT_XCARGS = [
"-allowProvisioningUpdates",
"-test-timeouts-enabled YES",
"-default-test-execution-time-allowance #{IOS_SCREENSHOT_TEST_TIMEOUT_SECONDS}",
"-maximum-test-execution-time-allowance #{IOS_SCREENSHOT_TEST_TIMEOUT_SECONDS}"
].join(" ").freeze
PNG_SIGNATURE = "\x89PNG\r\n\x1A\n".b.freeze
PUBLIC_METADATA_FILENAMES = [
"description.txt",
@@ -1611,6 +1619,88 @@ rescue JSON::ParserError, KeyError => e
UI.user_error!("Invalid screenshot test result summary for #{device}: #{e.message}")
end
def archive_snapshot_test_result!(result_bundle_path:, archive_directory:, device:, screenshot_name:, attempt:)
return unless File.directory?(result_bundle_path)
archive_path = File.join(
archive_directory,
"#{device}-#{screenshot_name}-attempt-#{attempt}.xcresult"
)
FileUtils.rm_rf(archive_path)
FileUtils.mv(result_bundle_path, archive_path)
end
def capture_release_ios_screenshot!(
project:,
device:,
screenshot:,
output_directory:,
result_bundle_path:,
result_bundle_archive_directory:,
derived_data_path:,
clear_previous_screenshots:
)
test_name = screenshot.fetch(:test)
screenshot_name = screenshot.fetch(:name)
expected_screenshot_path = File.join(output_directory, "en-US", "#{device}-#{screenshot_name}.png")
1.upto(2) do |attempt|
FileUtils.rm_f(expected_screenshot_path)
FileUtils.rm_rf(result_bundle_path)
begin
# Keep retry ownership outside Snapshot. Every action call creates a new
# xcodebuild/TestManager session and reboots the selected simulator.
capture_ios_screenshots(
project: project,
scheme: "OpenClawUITests",
configuration: "Debug",
devices: [device],
languages: ["en-US"],
only_testing: ["OpenClawUITests/OpenClawSnapshotUITests/#{test_name}"],
launch_arguments: ["--openclaw-screenshot-mode"],
output_directory: output_directory,
clear_previous_screenshots: clear_previous_screenshots && attempt == 1,
derived_data_path: derived_data_path,
test_without_building: true,
result_bundle: true,
number_of_retries: 0,
stop_after_first_error: true,
reinstall_app: true,
concurrent_simulators: false,
override_status_bar: true,
override_status_bar_arguments: SNAPSHOT_STATUS_BAR_ARGUMENTS,
skip_open_summary: true,
xcargs: IOS_SCREENSHOT_XCARGS
)
verify_snapshot_test_result!(result_bundle_path, "#{device} #{screenshot_name} attempt #{attempt}")
rescue StandardError => error
archive_snapshot_test_result!(
result_bundle_path: result_bundle_path,
archive_directory: result_bundle_archive_directory,
device: device,
screenshot_name: screenshot_name,
attempt: attempt
)
raise if attempt == 2
UI.important(
"Screenshot #{screenshot_name} failed on #{device}; " \
"retrying once in a fresh simulator session (#{error.message})."
)
next
end
archive_snapshot_test_result!(
result_bundle_path: result_bundle_path,
archive_directory: result_bundle_archive_directory,
device: device,
screenshot_name: screenshot_name,
attempt: attempt
)
return
end
end
def build_app_store_release(context)
version = context[:version]
project_path = File.join(ios_root, "OpenClaw.xcodeproj")
@@ -2040,34 +2130,34 @@ platform :ios do
FileUtils.mkdir_p(result_bundle_archive_directory)
FileUtils.rm_rf(derived_data_path)
devices = snapshot_devices
devices.each_with_index do |device, index|
capture_ios_screenshots(
project: File.join(ios_root, "OpenClaw.xcodeproj"),
scheme: "OpenClawUITests",
configuration: "Debug",
devices: [device],
languages: ["en-US"],
# App Store capture is isolated from functional UI checks; this lane only
# needs the deterministic four-screenshot release flow.
only_testing: ["OpenClawUITests/OpenClawSnapshotUITests/testConnectedGatewayTabs"],
launch_arguments: ["--openclaw-screenshot-mode"],
output_directory: output_directory,
clear_previous_screenshots: index.zero?,
derived_data_path: derived_data_path,
result_bundle: true,
number_of_retries: 0,
stop_after_first_error: true,
reinstall_app: true,
concurrent_simulators: false,
override_status_bar: true,
override_status_bar_arguments: SNAPSHOT_STATUS_BAR_ARGUMENTS,
skip_open_summary: true,
xcargs: "-allowProvisioningUpdates"
)
verify_snapshot_test_result!(result_bundle_path, device)
archived_result_bundle_path = File.join(result_bundle_archive_directory, "#{device}.xcresult")
FileUtils.rm_rf(archived_result_bundle_path)
FileUtils.mv(result_bundle_path, archived_result_bundle_path)
project = File.join(ios_root, "OpenClaw.xcodeproj")
run_tests(
project: project,
scheme: "OpenClawUITests",
configuration: "Debug",
devices: [devices.first],
derived_data_path: derived_data_path,
build_for_testing: true,
clean: true,
number_of_retries: 0,
xcargs: "-allowProvisioningUpdates"
)
capture_index = 0
devices.each do |device|
RELEASE_IOS_SCREENSHOT_TESTS.each do |screenshot|
capture_release_ios_screenshot!(
project: project,
device: device,
screenshot: screenshot,
output_directory: output_directory,
result_bundle_path: result_bundle_path,
result_bundle_archive_directory: result_bundle_archive_directory,
derived_data_path: derived_data_path,
clear_previous_screenshots: capture_index.zero?
)
capture_index += 1
end
end
verify_release_ios_screenshot_manifest!(
output_directory: output_directory,
test/scripts/ios-release-fastlane-gates.test.ts
+46 -31
View File
@@ -14,7 +14,6 @@ const snapshotUITestPath = path.join(
"UITests",
"OpenClawSnapshotUITests.swift",
);
const rootSidebarPath = path.join(process.cwd(), "apps", "ios", "Sources", "RootSidebar.swift");
const rootTabsPath = path.join(process.cwd(), "apps", "ios", "Sources", "RootTabs.swift");
const ciWorkflowPath = path.join(process.cwd(), ".github", "workflows", "ci.yml");
@@ -220,27 +219,33 @@ describe("iOS Fastlane release upload gates", () => {
it("fails from authoritative Xcode results and keeps successful bundles outside screenshots", () => {
const fastfile = readFastfile();
const screenshots = laneBody(fastfile, "screenshots");
const capture = functionBody(fastfile, "capture_release_ios_screenshot!");
const archive = functionBody(fastfile, "archive_snapshot_test_result!");
const verifier = functionBody(fastfile, "verify_snapshot_test_result!");
expect(screenshots).toContain("devices = snapshot_devices");
expect(screenshots).toContain("devices.each_with_index");
expect(screenshots).toContain("build_for_testing: true");
expect(screenshots).toContain("RELEASE_IOS_SCREENSHOT_TESTS.each");
expect(screenshots).toContain("capture_release_ios_screenshot!(");
expect(capture).toContain("1.upto(2)");
expect(screenshots).toContain(
'only_testing: ["OpenClawUITests/OpenClawSnapshotUITests/testConnectedGatewayTabs"]',
"result_bundle_archive_directory: result_bundle_archive_directory",
);
expect(screenshots).toContain("result_bundle: true");
expect(screenshots).toContain("number_of_retries: 0");
expect(screenshots).toContain("stop_after_first_error: true");
expect(capture).toContain(
'only_testing: ["OpenClawUITests/OpenClawSnapshotUITests/#{test_name}"]',
);
expect(capture).toContain("test_without_building: true");
expect(capture).toContain("result_bundle: true");
expect(capture).toContain("number_of_retries: 0");
expect(capture).toContain("stop_after_first_error: true");
expect(capture).toContain("retrying once in a fresh simulator session");
expect(capture).toContain("verify_snapshot_test_result!");
expect(archive).toContain('"#{device}-#{screenshot_name}-attempt-#{attempt}.xcresult"');
expect(screenshots).toContain("verify_release_ios_screenshot_manifest!(");
expect(screenshots).toContain("verify_snapshot_test_result!(result_bundle_path, device)");
expect(screenshots).toContain(
'result_bundle_archive_directory = File.join(ios_root, "build", "SnapshotTestResults")',
);
expect(screenshots.indexOf("verify_snapshot_test_result!")).toBeLessThan(
screenshots.indexOf("FileUtils.mv(result_bundle_path, archived_result_bundle_path)"),
);
expect(
screenshots.indexOf("FileUtils.mv(result_bundle_path, archived_result_bundle_path)"),
).toBeLessThan(
expect(screenshots.indexOf("capture_release_ios_screenshot!")).toBeLessThan(
screenshots.indexOf('FileUtils.rm_rf(File.join(output_directory, "test_output"))'),
);
expect(verifier).toContain('"xcresulttool"');
@@ -248,29 +253,38 @@ describe("iOS Fastlane release upload gates", () => {
expect(verifier).toContain("UI.test_failure!");
});
it("captures all release screens from one app launch with targeted launch recovery", () => {
it("captures each release screen from an independent direct launch", () => {
const snapshotUITest = readFileSync(snapshotUITestPath, "utf8");
const releaseTest = swiftFunctionBody(snapshotUITest, "testConnectedGatewayTabs");
const releaseTests = [
["testReleaseControlScreenshot", "controlScreenshotTarget"],
["testReleaseChatScreenshot", "chatScreenshotTarget"],
["testReleaseAgentScreenshot", "agentScreenshotTarget"],
["testReleaseSettingsScreenshot", "settingsScreenshotTarget"],
] as const;
const captureHelper = swiftFunctionBody(snapshotUITest, "captureReleaseScreenshot");
const launchHelper = swiftFunctionBody(snapshotUITest, "launchApp");
const rootSidebar = readFileSync(rootSidebarPath, "utf8");
const navigationTest = swiftFunctionBody(
snapshotUITest,
"testAgentsNavigateToSettingsThroughSidebar",
);
const rootTabs = readFileSync(rootTabsPath, "utf8");
expect(releaseTest.match(/self\.launchApp\(/g)).toHaveLength(1);
expect(snapshotUITest).toContain(
'releaseScreenshotLaunchArguments = ["-sidebar.pinnedPages", "overview,agents"]',
);
expect(releaseTest).toContain("selectReleaseScreenshotDestination");
expect(releaseTest).toContain("waitForReleaseScreenshotTarget");
expect(launchHelper).toContain("screenshotLaunchRetryThreshold");
expect(launchHelper).toContain("Recover before making any element query");
expect(launchHelper).toContain("app.wait(for: .notRunning, timeout: 5)");
expect(snapshotUITest).toContain("Recover stalled screenshot transition");
expect(snapshotUITest).toContain("self.launchApp(for: target");
for (const [testName, targetName] of releaseTests) {
const releaseTest = swiftFunctionBody(snapshotUITest, testName);
expect(releaseTest).toContain(`self.captureReleaseScreenshot(Self.${targetName})`);
}
expect(captureHelper.match(/self\.launchApp\(/g)).toHaveLength(1);
expect(captureHelper).toContain("waitForReleaseScreenshotTarget");
expect(launchHelper).toContain("app.launch()");
expect(snapshotUITest).not.toContain("screenshotLaunchRetryThreshold");
expect(snapshotUITest).not.toContain("selectReleaseScreenshotDestination");
expect(navigationTest).toContain("self.launchApp(for: Self.agentScreenshotTarget)");
expect(navigationTest).toContain('self.selectSidebarDestination("Settings")');
expect(navigationTest).toContain('"settings-system-agent-row"');
expect(navigationTest).toContain("XCTExpectedFailure.Options()");
expect(navigationTest).toContain("options.isStrict = false");
expect(rootTabs).toContain("self.scenePhase == .active");
expect(rootTabs).toContain("self.selectedSidebarDestination.rawValue");
expect(rootSidebar).toContain('"RootTabs.Sidebar.Destination.chat"');
expect(rootSidebar).toContain('"RootTabs.Sidebar.Destination.settings"');
expect(rootSidebar).toContain('"RootTabs.Sidebar.Destination.\\(destination.rawValue)"');
});
it("requires the exact nonempty PNG manifest before Watch capture", () => {
@@ -284,7 +298,7 @@ describe("iOS Fastlane release upload gates", () => {
expect(verifier).toContain("File.size?(path)");
expect(verifier).toContain("PNG_SIGNATURE");
expect(screenshots.indexOf("verify_release_ios_screenshot_manifest!")).toBeGreaterThan(
screenshots.indexOf("devices.each_with_index"),
screenshots.indexOf("RELEASE_IOS_SCREENSHOT_TESTS.each"),
);
expect(screenshots.indexOf("verify_release_ios_screenshot_manifest!")).toBeLessThan(
screenshots.indexOf("watch_screenshot("),
@@ -300,6 +314,7 @@ describe("iOS Fastlane release upload gates", () => {
expect(iosJob).toContain("timeout-minutes: 75");
expect(iosJob).toContain("Capture iOS release screenshots");
expect(iosJob).toContain("github.event_name == 'workflow_dispatch'");
expect(iosJob).toContain("github.event_name == 'pull_request'");
expect(iosJob).toContain("run: pnpm ios:screenshots");
expect(iosJob).toContain("Upload iOS release screenshot evidence");
expect(iosJob).toContain("apps/ios/build/SnapshotTestResults/*.xcresult");