openclaw/qa/scenarios/runtime/long-context-progress-watchdog.yaml

title: Long-context progress watchdog

scenario:
  id: long-context-progress-watchdog
  surface: runtime
  coverage:
    primary:
      - session-memory.codex-progress-sentinel
    secondary:
      - session-memory.token-pressure-long-context
      - openai.codex-harness
  objective: Fail live proof when long-context activity triggers Codex app-server timeout or stalled-progress sentinels.
  successCriteria:
    - Gateway config routes the selected QA model through the Codex app-server runtime.
    - Agent reads through the seeded long-context fixture and replies with the marker found at the tail.
    - Gateway logs since the scenario cursor contain no app-server timeout or stalled-progress sentinel.
  docsRefs:
    - docs/concepts/qa-e2e-automation.md
    - qa/scenarios/index.yaml
  codeRefs:
    - extensions/qa-lab/src/gateway-log-sentinel.ts
    - extensions/codex/src/app-server
  execution:
    kind: flow
    runtime: codex
    summary: Seed a large workspace fixture, complete a read turn, and scan for Codex app-server progress failures.
    config:
      requiredProviderMode: live-frontier
      harnessRuntime: codex
      fixtureFile: LONG_CONTEXT_SENTINEL_FIXTURE.txt
      expectedMarker: LONG-CONTEXT-WATCHDOG-OK
      repeatCount: 2000

flow:
  steps:
    - name: catches app-server timeout or stalled progress during long-context activity
      actions:
        - call: waitForGatewayHealthy
          args:
            - ref: env
            - 60000
        - call: waitForQaChannelReady
          args:
            - ref: env
            - 60000
        - assert:
            expr: "env.runtimeId === config.harnessRuntime"
            message:
              expr: "`expected suite runtime ${config.harnessRuntime}, got ${env.runtimeId}`"
        - call: reset
        - set: logCursor
          value:
            expr: markGatewayLogCursor()
        - set: fixturePath
          value:
            expr: "path.join(env.gateway.workspaceDir, config.fixtureFile)"
        - call: fs.writeFile
          args:
            - ref: fixturePath
            - expr: "`START LONG-CONTEXT-WATCHDOG\\n${Array.from({ length: config.repeatCount }, (_entry, index) => `context row ${index + 1}: alpha beta gamma`).join('\\n')}\\nTAIL ${config.expectedMarker}\\n`"
            - utf8
        - set: startIndex
          value:
            expr: state.getSnapshot().messages.length
        - call: runAgentPrompt
          args:
            - ref: env
            - sessionKey:
                expr: "`agent:qa:long-context-watchdog:${randomUUID().slice(0, 8)}`"
              message:
                expr: "`Read ${fixturePath}, find the marker on the TAIL line, and reply with that marker only.`"
              timeoutMs:
                expr: liveTurnTimeoutMs(env, 90000)
        - call: waitForOutboundMessage
          saveAs: outbound
          args:
            - ref: state
            - lambda:
                params: [candidate]
                expr: "candidate.conversation.id === 'qa-operator' && normalizeLowercaseStringOrEmpty(candidate.text).includes(normalizeLowercaseStringOrEmpty(config.expectedMarker))"
            - expr: liveTurnTimeoutMs(env, 45000)
            - sinceIndex:
                ref: startIndex
        - call: assertNoGatewayLogSentinels
          args:
            - since:
                ref: logCursor
              kinds:
                - codex-app-server-timeout
                - stalled-agent-run
      detailsExpr: outbound.text