openclaw/scripts/capture-model-picker-proof.mts

#!/usr/bin/env node
// Captures chat model-picker proof shots: the open picker inheriting the agent
// default, the same picker with a session override, and the picker after a
// failed catalog refresh. Also reports the x offset between the provider
// heading label and the model row name (the row-alignment change).
import { mkdir } from "node:fs/promises";
import path from "node:path";
import { chromium } from "playwright";
import {
  canRunPlaywrightChromium,
  installMockGateway,
  resolvePlaywrightChromiumExecutablePath,
  startControlUiE2eServer,
} from "../ui/src/test-helpers/control-ui-e2e.ts";

function readOption(name: string): string | undefined {
  const prefix = `--${name}=`;
  const inline = process.argv.slice(2).find((arg) => arg.startsWith(prefix));
  if (inline) {
    return inline.slice(prefix.length);
  }
  const index = process.argv.indexOf(`--${name}`);
  return index >= 0 ? process.argv[index + 1] : undefined;
}

const outputDir = path.resolve(
  readOption("output-dir") ?? ".artifacts/control-ui-e2e/model-picker-proof",
);
const label = readOption("label") ?? "after";
const executablePath = resolvePlaywrightChromiumExecutablePath(chromium.executablePath());
if (!canRunPlaywrightChromium(executablePath)) {
  throw new Error(`Playwright Chromium is unavailable at ${executablePath}`);
}

const models = [
  { id: "gpt-5.5", name: "GPT-5.5", provider: "openai", contextWindow: 400_000 },
  { id: "gpt-5.6-luna", name: "GPT-5.6 Luna", provider: "openai", contextWindow: 1_000_000 },
  { id: "gpt-5.6-sol", name: "GPT-5.6 Sol", provider: "openai", contextWindow: 1_000_000 },
  { id: "gpt-5.6-terra", name: "GPT-5.6 Terra", provider: "openai", contextWindow: 1_000_000 },
  {
    id: "claude-sonnet-4-6",
    name: "Claude Sonnet 4.6",
    provider: "anthropic",
    contextWindow: 200_000,
  },
  { id: "claude-opus-4-6", name: "Claude Opus 4.6", provider: "anthropic", contextWindow: 200_000 },
];

await mkdir(outputDir, { recursive: true });
const server = await startControlUiE2eServer(undefined, { source: true });
const browser = await chromium.launch({ executablePath });
const context = await browser.newContext({
  colorScheme: "dark",
  viewport: { width: 1280, height: 900 },
});
const page = await context.newPage();
page.setDefaultTimeout(30_000);

try {
  const gateway = await installMockGateway(page, { agentModel: "openai/gpt-5.5", models });
  await page.goto(`${server.baseUrl}chat`);
  await gateway.waitForRequest("chat.startup");

  const composer = page.locator(".agent-chat__input");
  await composer.waitFor({ state: "visible" });
  const trigger = composer.locator('[data-chat-model-select="true"]');
  const menu = composer.locator(".chat-controls__model-menu");

  const openPicker = async () => {
    await trigger.click();
    await menu.waitFor({ state: "visible" });
    await page.waitForTimeout(500);
  };
  const closePicker = async () => {
    await page.keyboard.press("Escape");
    await page.waitForTimeout(250);
  };
  const shoot = async (name: string) => {
    const box = await menu.boundingBox();
    if (!box) {
      throw new Error(`model menu has no layout box for ${name}`);
    }
    await page.screenshot({
      path: path.join(outputDir, `${label}-${name}.png`),
      clip: {
        x: Math.max(0, box.x - 8),
        y: Math.max(0, box.y - 8),
        width: box.width + 16,
        height: box.height + 16,
      },
    });
  };

  await openPicker();
  await shoot("default");

  const headingLabel = menu.locator(".chat-controls__provider-heading span").last();
  const rowName = menu.locator(".chat-controls__model-option-name").first();
  const headingBox = await headingLabel.boundingBox();
  const rowBox = await rowName.boundingBox();

  const search = menu.locator("[data-chat-model-search]");
  await search.fill("default");
  await page.waitForTimeout(400);
  await shoot("search-default");
  const searchMatches = await menu
    .locator("[data-chat-model-option]:not([hidden])")
    .evaluateAll((rows) => rows.map((row) => row.textContent?.replace(/\s+/gu, " ").trim() ?? ""));
  await search.fill("");
  await page.waitForTimeout(250);

  const override = menu.locator('[data-chat-model-option="openai/gpt-5.6-terra"]');
  await override.click();
  await page.waitForTimeout(600);
  await openPicker();
  await shoot("override");
  await closePicker();

  // A fresh page drops the per-client catalog cache, so the picker's own load
  // reaches the mock gateway and fails there.
  await gateway.setMethodResponse("models.list", {
    __mockError: { code: "UNAVAILABLE", message: "mock catalog refresh failed" },
  });
  await page.reload();
  await composer.waitFor({ state: "visible" });
  await page.waitForTimeout(1000);
  await openPicker();
  await page.waitForTimeout(1500);
  await shoot("refresh-failure");
  const catalogStateText = await menu.locator("[data-chat-model-catalog-state]").allTextContents();
  const rowsAfterFailure = await menu.locator("[data-chat-model-option]").count();
  await closePicker();

  console.log(
    JSON.stringify(
      {
        label,
        outputDir,
        providerHeadingLabelX: headingBox?.x ?? null,
        modelRowNameX: rowBox?.x ?? null,
        alignmentDeltaPx: headingBox && rowBox ? rowBox.x - headingBox.x : null,
        searchDefaultMatches: searchMatches,
        catalogStateAfterRefreshFailure: catalogStateText,
        modelRowsAfterRefreshFailure: rowsAfterFailure,
      },
      null,
      2,
    ),
  );
} finally {
  await context.close();
  await browser.close();
  await server.close();
}