diff --git a/turnstone/eval/cli.py b/turnstone/eval/cli.py index e1f0db16..32e2526e 100644 --- a/turnstone/eval/cli.py +++ b/turnstone/eval/cli.py @@ -184,8 +184,12 @@ def main() -> None: parser.add_argument( "--temperature", type=float, - default=0.7, - help="Sampling temperature (default: 0.7)", + default=None, + help=( + "Sampling temperature. Omitted from the wire by default so " + "the alias / stored setting / serving default applies — the " + "same assignment scheme production runs" + ), ) parser.add_argument( "--max-tokens", @@ -195,9 +199,13 @@ def main() -> None: ) parser.add_argument( "--reasoning-effort", - default="medium", + default=None, choices=["low", "medium", "high"], - help="Reasoning effort (default: medium)", + help=( + "Reasoning effort. Omitted from the wire by default — a " + "code-chosen token is unvetted and can flip thinking on for " + "lanes the operator never engaged" + ), ) parser.add_argument( "--context-window", diff --git a/turnstone/eval/core.py b/turnstone/eval/core.py index 2018839d..de4b5bfe 100644 --- a/turnstone/eval/core.py +++ b/turnstone/eval/core.py @@ -238,10 +238,10 @@ class HeadlessSession(ChatSession): model: str, system_prompt_override: str | None = None, instructions: str | None = None, - temperature: float = 0.7, + temperature: float | None = None, max_tokens: int = 32768, tool_timeout: int = 30, - reasoning_effort: str = "medium", + reasoning_effort: str | None = None, context_window: int = 131072, compact_max_tokens: int = 32768, auto_compact_pct: float = 0.8, @@ -651,9 +651,9 @@ def _run_single_test( model: str, system_prompt: str, case: dict[str, Any], - temperature: float, + temperature: float | None, max_tokens: int, - reasoning_effort: str, + reasoning_effort: str | None, context_window: int, verbose: bool = False, log_prefix: str = "", @@ -1100,9 +1100,9 @@ def _run_iteration_parallel( system_prompt: str, cases: list[dict[str, Any]], n_runs: int, - temperature: float, + temperature: float | None, max_tokens: int, - reasoning_effort: str, + reasoning_effort: str | None, context_window: int, test_timeout: int, parallel: int, @@ -1251,9 +1251,9 @@ def _run_iteration( system_prompt: str, cases: list[dict[str, Any]], n_runs: int, - temperature: float, + temperature: float | None, max_tokens: int, - reasoning_effort: str, + reasoning_effort: str | None, context_window: int, verbose: bool = False, test_timeout: int = 300, @@ -1469,9 +1469,9 @@ def run_skill_adherence( model: str, cases: list[dict[str, Any]], n_runs: int, - temperature: float, + temperature: float | None, max_tokens: int, - reasoning_effort: str, + reasoning_effort: str | None, context_window: int, test_timeout: int = 300, parallel: int = 1, diff --git a/turnstone/eval/nudges.py b/turnstone/eval/nudges.py index 5b562b70..d7292ac1 100644 --- a/turnstone/eval/nudges.py +++ b/turnstone/eval/nudges.py @@ -517,9 +517,9 @@ class CoordinatorHeadlessSession(HeadlessSession): coord_client: CoordinatorClient, ws_id: str, user_id: str, - temperature: float, + temperature: float | None, max_tokens: int, - reasoning_effort: str, + reasoning_effort: str | None, context_window: int, ) -> None: super().__init__( @@ -981,9 +981,9 @@ def _run_single_nudge( model: str, case: dict[str, Any], arm: str, - temperature: float, + temperature: float | None, max_tokens: int, - reasoning_effort: str, + reasoning_effort: str | None, context_window: int, max_turns: int, test_timeout: int, @@ -2019,9 +2019,9 @@ def run_nudge_response( model: str, cells: list[dict[str, Any]], n_runs: int = 10, - temperature: float = 0.7, + temperature: float | None = None, max_tokens: int = 8192, - reasoning_effort: str = "medium", + reasoning_effort: str | None = None, context_window: int = 131072, max_turns: int = 8, test_timeout: int = 300,