mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(qa): keep live provider scenarios out of mock lanes (#119498)
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
57122c965c
commit
b3aab44f4b
@@ -25,6 +25,7 @@ scenario:
|
||||
kind: flow
|
||||
summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --model anthropic/claude-opus-5 --alt-model anthropic/claude-opus-5 --scenario anthropic-opus-api-key-smoke`.
|
||||
config:
|
||||
requiredProviderMode: live-frontier
|
||||
requiredProvider: anthropic
|
||||
requiredModel: claude-opus-5
|
||||
chatPrompt: "Anthropic Opus API key smoke. Reply exactly: ANTHROPIC-OPUS-API-KEY-OK"
|
||||
@@ -38,50 +39,49 @@ flow:
|
||||
value:
|
||||
expr: splitModelRef(env.primaryModel)
|
||||
- assert:
|
||||
expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
|
||||
expr: "env.providerMode === config.requiredProviderMode"
|
||||
message:
|
||||
expr: "`expected provider mode ${config.requiredProviderMode}, got ${env.providerMode}`"
|
||||
- assert:
|
||||
expr: "selected?.provider === config.requiredProvider"
|
||||
message:
|
||||
expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
|
||||
- assert:
|
||||
expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
|
||||
expr: "selected?.model === config.requiredModel"
|
||||
message:
|
||||
expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
|
||||
- assert:
|
||||
expr: "env.providerMode !== 'live-frontier' || Boolean(env.gateway.runtimeEnv.ANTHROPIC_API_KEY?.trim())"
|
||||
expr: "Boolean(env.gateway.runtimeEnv.ANTHROPIC_API_KEY?.trim())"
|
||||
message: expected ANTHROPIC_API_KEY to be available for API-key QA mode
|
||||
detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=env-api-key` : `mock-compatible provider=${selected?.provider}`"
|
||||
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} auth=env-api-key`"
|
||||
- name: talks through regular Anthropic Opus
|
||||
actions:
|
||||
- if:
|
||||
expr: "env.providerMode !== 'live-frontier'"
|
||||
then:
|
||||
- assert: "true"
|
||||
else:
|
||||
- call: reset
|
||||
- set: selected
|
||||
value:
|
||||
expr: splitModelRef(env.primaryModel)
|
||||
- call: runAgentPrompt
|
||||
args:
|
||||
- ref: env
|
||||
- sessionKey: agent:qa:anthropic-opus-api-key
|
||||
message:
|
||||
expr: config.chatPrompt
|
||||
provider:
|
||||
expr: selected?.provider
|
||||
model:
|
||||
expr: selected?.model
|
||||
timeoutMs:
|
||||
expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: chatOutbound
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.conversation.id === 'qa-operator'"
|
||||
- expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
|
||||
- assert:
|
||||
expr: "chatOutbound.text.includes(config.chatExpected)"
|
||||
message:
|
||||
expr: "`chat marker missing: ${chatOutbound.text}`"
|
||||
detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live Anthropic smoke' : chatOutbound.text"
|
||||
- call: reset
|
||||
- set: selected
|
||||
value:
|
||||
expr: splitModelRef(env.primaryModel)
|
||||
- call: runAgentPrompt
|
||||
args:
|
||||
- ref: env
|
||||
- sessionKey: agent:qa:anthropic-opus-api-key
|
||||
message:
|
||||
expr: config.chatPrompt
|
||||
provider:
|
||||
expr: selected?.provider
|
||||
model:
|
||||
expr: selected?.model
|
||||
timeoutMs:
|
||||
expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: chatOutbound
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.conversation.id === 'qa-operator'"
|
||||
- expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
|
||||
- assert:
|
||||
expr: "chatOutbound.text.includes(config.chatExpected)"
|
||||
message:
|
||||
expr: "`chat marker missing: ${chatOutbound.text}`"
|
||||
detailsExpr: chatOutbound.text
|
||||
|
||||
@@ -25,6 +25,7 @@ scenario:
|
||||
kind: flow
|
||||
summary: Run with `OPENCLAW_LIVE_SETUP_TOKEN_VALUE=<setup-token> pnpm openclaw qa suite --provider-mode live-frontier --model anthropic/claude-opus-5 --alt-model anthropic/claude-opus-5 --scenario anthropic-opus-setup-token-smoke`.
|
||||
config:
|
||||
requiredProviderMode: live-frontier
|
||||
requiredProvider: anthropic
|
||||
requiredModel: claude-opus-5
|
||||
profileId: "anthropic:qa-setup-token"
|
||||
@@ -39,54 +40,53 @@ flow:
|
||||
value:
|
||||
expr: splitModelRef(env.primaryModel)
|
||||
- assert:
|
||||
expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
|
||||
expr: "env.providerMode === config.requiredProviderMode"
|
||||
message:
|
||||
expr: "`expected provider mode ${config.requiredProviderMode}, got ${env.providerMode}`"
|
||||
- assert:
|
||||
expr: "selected?.provider === config.requiredProvider"
|
||||
message:
|
||||
expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
|
||||
- assert:
|
||||
expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
|
||||
expr: "selected?.model === config.requiredModel"
|
||||
message:
|
||||
expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
|
||||
- assert:
|
||||
expr: "env.providerMode !== 'live-frontier' || env.gateway.cfg.auth?.profiles?.[config.profileId]?.mode === 'token'"
|
||||
expr: "env.gateway.cfg.auth?.profiles?.[config.profileId]?.mode === 'token'"
|
||||
message:
|
||||
expr: "`expected token profile ${config.profileId} in QA config`"
|
||||
- assert:
|
||||
expr: "env.providerMode !== 'live-frontier' || !env.gateway.runtimeEnv.OPENCLAW_LIVE_SETUP_TOKEN_VALUE"
|
||||
expr: "!env.gateway.runtimeEnv.OPENCLAW_LIVE_SETUP_TOKEN_VALUE"
|
||||
message: setup-token value should not be passed to the gateway child env
|
||||
detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} auth=setup-token profile=${config.profileId}` : `mock-compatible provider=${selected?.provider}`"
|
||||
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} auth=setup-token profile=${config.profileId}`"
|
||||
- name: talks through regular Anthropic Opus
|
||||
actions:
|
||||
- if:
|
||||
expr: "env.providerMode !== 'live-frontier'"
|
||||
then:
|
||||
- assert: "true"
|
||||
else:
|
||||
- call: reset
|
||||
- set: selected
|
||||
value:
|
||||
expr: splitModelRef(env.primaryModel)
|
||||
- call: runAgentPrompt
|
||||
args:
|
||||
- ref: env
|
||||
- sessionKey: agent:qa:anthropic-opus-setup-token
|
||||
message:
|
||||
expr: config.chatPrompt
|
||||
provider:
|
||||
expr: selected?.provider
|
||||
model:
|
||||
expr: selected?.model
|
||||
timeoutMs:
|
||||
expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: chatOutbound
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.conversation.id === 'qa-operator'"
|
||||
- expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
|
||||
- assert:
|
||||
expr: "chatOutbound.text.includes(config.chatExpected)"
|
||||
message:
|
||||
expr: "`chat marker missing: ${chatOutbound.text}`"
|
||||
detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live Anthropic smoke' : chatOutbound.text"
|
||||
- call: reset
|
||||
- set: selected
|
||||
value:
|
||||
expr: splitModelRef(env.primaryModel)
|
||||
- call: runAgentPrompt
|
||||
args:
|
||||
- ref: env
|
||||
- sessionKey: agent:qa:anthropic-opus-setup-token
|
||||
message:
|
||||
expr: config.chatPrompt
|
||||
provider:
|
||||
expr: selected?.provider
|
||||
model:
|
||||
expr: selected?.model
|
||||
timeoutMs:
|
||||
expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: chatOutbound
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.conversation.id === 'qa-operator'"
|
||||
- expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.primaryModel)
|
||||
- assert:
|
||||
expr: "chatOutbound.text.includes(config.chatExpected)"
|
||||
message:
|
||||
expr: "`chat marker missing: ${chatOutbound.text}`"
|
||||
detailsExpr: chatOutbound.text
|
||||
|
||||
@@ -33,6 +33,7 @@ scenario:
|
||||
kind: flow
|
||||
summary: Run with `OPENCLAW_LIVE_OPENAI_KEY="${OPENAI_API_KEY}" pnpm openclaw qa suite --provider-mode live-frontier --model openai/gpt-5.6-luna --alt-model openai/gpt-5.6-luna --fast --thinking medium --scenario openai-native-web-search-live`.
|
||||
config:
|
||||
requiredProviderMode: live-frontier
|
||||
requiredProvider: openai
|
||||
requiredModel: gpt-5.6-luna
|
||||
expectedMarker: WEB-SEARCH-OK
|
||||
@@ -61,11 +62,15 @@ flow:
|
||||
value:
|
||||
expr: splitModelRef(env.primaryModel)
|
||||
- assert:
|
||||
expr: "env.providerMode !== 'live-frontier' || selected?.provider === config.requiredProvider"
|
||||
expr: "env.providerMode === config.requiredProviderMode"
|
||||
message:
|
||||
expr: "`expected provider mode ${config.requiredProviderMode}, got ${env.providerMode}`"
|
||||
- assert:
|
||||
expr: "selected?.provider === config.requiredProvider"
|
||||
message:
|
||||
expr: "`expected live primary provider ${config.requiredProvider}, got ${env.primaryModel}`"
|
||||
- assert:
|
||||
expr: "env.providerMode !== 'live-frontier' || selected?.model === config.requiredModel"
|
||||
expr: "selected?.model === config.requiredModel"
|
||||
message:
|
||||
expr: "`expected live primary model ${config.requiredModel}, got ${env.primaryModel}`"
|
||||
- call: readConfigSnapshot
|
||||
@@ -84,60 +89,55 @@ flow:
|
||||
message:
|
||||
expr: "`expected web search provider auto/openai/unset for native OpenAI search, got ${JSON.stringify(searchConfig)}`"
|
||||
- assert:
|
||||
expr: "env.providerMode !== 'live-frontier' || Boolean(env.gateway.runtimeEnv.OPENAI_API_KEY?.trim() || env.gateway.runtimeEnv.OPENCLAW_LIVE_OPENAI_KEY?.trim())"
|
||||
expr: "Boolean(env.gateway.runtimeEnv.OPENAI_API_KEY?.trim() || env.gateway.runtimeEnv.OPENCLAW_LIVE_OPENAI_KEY?.trim())"
|
||||
message: expected OPENAI_API_KEY or OPENCLAW_LIVE_OPENAI_KEY for live OpenAI QA
|
||||
detailsExpr: "env.providerMode === 'live-frontier' ? `provider=${selected?.provider} model=${selected?.model} webSearch=${JSON.stringify(searchConfig)}` : `mock-compatible provider=${selected?.provider}`"
|
||||
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} webSearch=${JSON.stringify(searchConfig)}`"
|
||||
- name: searches official OpenAI News through the live model
|
||||
actions:
|
||||
- if:
|
||||
expr: "env.providerMode !== 'live-frontier'"
|
||||
then:
|
||||
- assert: "true"
|
||||
else:
|
||||
- call: reset
|
||||
- set: selected
|
||||
value:
|
||||
expr: splitModelRef(env.primaryModel)
|
||||
- call: runAgentPrompt
|
||||
args:
|
||||
- ref: env
|
||||
- sessionKey: agent:qa:openai-native-web-search
|
||||
message:
|
||||
expr: config.searchPrompt
|
||||
provider:
|
||||
expr: selected?.provider
|
||||
model:
|
||||
expr: selected?.model
|
||||
timeoutMs:
|
||||
expr: resolveQaLiveTurnTimeoutMs(env, 180000, env.primaryModel)
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: searchOutbound
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.conversation.id === 'qa-operator'"
|
||||
- expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
|
||||
- set: searchText
|
||||
value:
|
||||
expr: searchOutbound.text
|
||||
- set: searchTextLower
|
||||
value:
|
||||
expr: normalizeLowercaseStringOrEmpty(searchText)
|
||||
- assert:
|
||||
expr: "searchText.includes(config.expectedMarker)"
|
||||
message:
|
||||
expr: "`missing ${config.expectedMarker}: ${searchText}`"
|
||||
- assert:
|
||||
expr: "!searchText.includes(config.failureMarker) && !/(web search is unavailable|unable to search|cannot search|can't search)/i.test(searchText)"
|
||||
message:
|
||||
expr: "`search looked unavailable: ${searchText}`"
|
||||
- assert:
|
||||
expr: "/URL:\\s*https?:\\/\\/[^\\s]*openai\\.com\\/news/i.test(searchText)"
|
||||
message:
|
||||
expr: "`missing official OpenAI News URL: ${searchText}`"
|
||||
- assert:
|
||||
expr: "/HEADLINE:\\s*\\S.{8,}/i.test(searchText)"
|
||||
message:
|
||||
expr: "`missing searched headline: ${searchText}`"
|
||||
detailsExpr: "env.providerMode !== 'live-frontier' ? 'mock mode: skipped live OpenAI web search probe' : searchText"
|
||||
- call: reset
|
||||
- set: selected
|
||||
value:
|
||||
expr: splitModelRef(env.primaryModel)
|
||||
- call: runAgentPrompt
|
||||
args:
|
||||
- ref: env
|
||||
- sessionKey: agent:qa:openai-native-web-search
|
||||
message:
|
||||
expr: config.searchPrompt
|
||||
provider:
|
||||
expr: selected?.provider
|
||||
model:
|
||||
expr: selected?.model
|
||||
timeoutMs:
|
||||
expr: resolveQaLiveTurnTimeoutMs(env, 180000, env.primaryModel)
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: searchOutbound
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.conversation.id === 'qa-operator'"
|
||||
- expr: resolveQaLiveTurnTimeoutMs(env, 60000, env.primaryModel)
|
||||
- set: searchText
|
||||
value:
|
||||
expr: searchOutbound.text
|
||||
- set: searchTextLower
|
||||
value:
|
||||
expr: normalizeLowercaseStringOrEmpty(searchText)
|
||||
- assert:
|
||||
expr: "searchText.includes(config.expectedMarker)"
|
||||
message:
|
||||
expr: "`missing ${config.expectedMarker}: ${searchText}`"
|
||||
- assert:
|
||||
expr: "!searchText.includes(config.failureMarker) && !/(web search is unavailable|unable to search|cannot search|can't search)/i.test(searchText)"
|
||||
message:
|
||||
expr: "`search looked unavailable: ${searchText}`"
|
||||
- assert:
|
||||
expr: "/URL:\\s*https?:\\/\\/[^\\s]*openai\\.com\\/news/i.test(searchText)"
|
||||
message:
|
||||
expr: "`missing official OpenAI News URL: ${searchText}`"
|
||||
- assert:
|
||||
expr: "/HEADLINE:\\s*\\S.{8,}/i.test(searchText)"
|
||||
message:
|
||||
expr: "`missing searched headline: ${searchText}`"
|
||||
detailsExpr: searchText
|
||||
|
||||
Reference in New Issue
Block a user