openclaw/qa/scenarios/models/model-switch-tool-continuity.yaml

title: Model switch with tool continuity

scenario:
  id: model-switch-tool-continuity
  surface: models
  runtimePairLane: core
  coverage:
    primary:
      - agent-runtime.runtime-override-switching
    secondary:
      - agent-runtime.tool-continuity
  objective: Verify switching models preserves session context and tool use instead of dropping into plain-text only behavior.
  successCriteria:
    - Alternate model is actually requested.
    - A tool call still happens after the model switch.
    - Final answer acknowledges the handoff and reread QA mission.
  docsRefs:
    - docs/help/testing.md
    - docs/concepts/model-failover.md
  codeRefs:
    - extensions/qa-lab/src/suite.ts
    - extensions/qa-lab/src/providers/mock-openai/server.ts
  execution:
    kind: flow
    summary: Verify switching models preserves session context and tool use instead of dropping into plain-text only behavior.
    config:
      initialPrompt: "Read repo/qa/scenarios/index.yaml and summarize the QA scenario pack mission in one clause before any model switch."
      followupPrompt: "The harness has already requested the alternate model for this turn. Do not call session_status or change models yourself. Tool continuity check: use the read tool to reread repo/qa/scenarios/index.yaml, then mention the model handoff and QA mission in one short sentence."
      promptSnippet: "Tool continuity check"

flow:
  steps:
    - name: keeps using tools after switching models
      actions:
        - set: expectedAlternate
          value:
            expr: normalizeModelRef(env.alternateModel)
        - assert:
            expr: "normalizeModelRef(env.primaryModel) && expectedAlternate && (normalizeModelRef(env.primaryModel).provider !== expectedAlternate.provider || normalizeModelRef(env.primaryModel).model !== expectedAlternate.model)"
            message: primary and alternate models must normalize to different refs
        - call: waitForGatewayHealthy
          args:
            - ref: env
            - 60000
        - call: reset
        - call: runAgentPrompt
          saveAs: primaryRun
          args:
            - ref: env
            - sessionKey: agent:qa:model-switch-tools
              message:
                expr: config.initialPrompt
              timeoutMs:
                expr: liveTurnTimeoutMs(env, 30000)
        - assert:
            expr: "(() => { const expected = normalizeModelRef(env.primaryModel); const receipt = primaryRun?.waited?.terminalReceipt; return receipt?.runId === primaryRun?.started?.runId && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expected.provider && receipt.requested?.model === expected.model && receipt.successfulToolNames?.includes('read') && receipt.terminalDisposition === 'visible'; })()"
            message: default-model run did not return owned successful read evidence
        - assert:
            expr: "primaryRun?.waited?.terminalDelivery?.status === 'sent' && typeof primaryRun.waited.terminalDelivery.resultCount === 'number' && primaryRun.waited.terminalDelivery.resultCount > 0"
            message: default-model run did not return owned sent delivery evidence
        - call: runAgentPrompt
          saveAs: alternateRun
          args:
            - ref: env
            - sessionKey: agent:qa:model-switch-tools
              message:
                expr: config.followupPrompt
              provider:
                expr: expectedAlternate.provider
              model:
                expr: expectedAlternate.model
              timeoutMs:
                expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.alternateModel)
        - assert:
            expr: "(() => { const receipt = alternateRun?.waited?.terminalReceipt; return receipt?.runId === alternateRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expectedAlternate.provider && receipt.requested?.model === expectedAlternate.model && receipt.effective?.model === receipt.effective?.responseModel && receipt.successfulToolNames?.includes('read') && receipt.terminalDisposition === 'visible' && typeof receipt.rerouted === 'boolean' && `${receipt.effective?.provider}/${receipt.effective?.responseModel}` !== `${primaryRun.waited.terminalReceipt.effective?.provider}/${primaryRun.waited.terminalReceipt.effective?.responseModel}`; })()"
            message: alternate-model run did not return exact owned successful read evidence
        - assert:
            expr: "alternateRun?.waited?.terminalReply?.disposition === 'visible' && hasModelSwitchContinuitySignal(alternateRun.waited.terminalReply.text)"
            message: alternate-model terminal reply missed kickoff continuity
        - assert:
            expr: "alternateRun?.waited?.terminalDelivery?.status === 'sent' && typeof alternateRun.waited.terminalDelivery.resultCount === 'number' && alternateRun.waited.terminalDelivery.resultCount > 0"
            message: alternate-model run did not return owned sent delivery evidence
        - set: modelSwitchEvidence
          value:
            primary:
              ref: primaryRun.waited.terminalReceipt
            alternate:
              ref: alternateRun.waited.terminalReceipt
            terminalReply:
              ref: alternateRun.waited.terminalReply
            terminalDelivery:
              ref: alternateRun.waited.terminalDelivery
      detailsExpr: alternateRun.waited.terminalReply.text