fix(eval): stop pinning temperature and reasoning effort on the wire

The eval CLI defaulted temperature to 0.7 and reasoning effort to
medium, so every sweep sent code-chosen sampling knobs the house
assignment scheme forbids — the wire should omit the fields and let
the alias / stored setting / serving default apply, as production
does. Both flags now default to unset and the harnesses plumb None
through to model_turn, whose provider layer already omits absent
knobs. Absolute numbers from earlier sweeps were collected under the
pinned values; contrasts were at least uniform under the same pin.
This commit is contained in:
Patrick Buckley
2026-07-29 07:36:28 -07:00
parent 33c82962a2
commit 1bc2c39ca8
3 changed files with 28 additions and 20 deletions
+12 -4
View File
@@ -184,8 +184,12 @@ def main() -> None:
parser.add_argument(
"--temperature",
type=float,
default=0.7,
help="Sampling temperature (default: 0.7)",
default=None,
help=(
"Sampling temperature. Omitted from the wire by default so "
"the alias / stored setting / serving default applies — the "
"same assignment scheme production runs"
),
)
parser.add_argument(
"--max-tokens",
@@ -195,9 +199,13 @@ def main() -> None:
)
parser.add_argument(
"--reasoning-effort",
default="medium",
default=None,
choices=["low", "medium", "high"],
help="Reasoning effort (default: medium)",
help=(
"Reasoning effort. Omitted from the wire by default — a "
"code-chosen token is unvetted and can flip thinking on for "
"lanes the operator never engaged"
),
)
parser.add_argument(
"--context-window",
+10 -10
View File
@@ -238,10 +238,10 @@ class HeadlessSession(ChatSession):
model: str,
system_prompt_override: str | None = None,
instructions: str | None = None,
temperature: float = 0.7,
temperature: float | None = None,
max_tokens: int = 32768,
tool_timeout: int = 30,
reasoning_effort: str = "medium",
reasoning_effort: str | None = None,
context_window: int = 131072,
compact_max_tokens: int = 32768,
auto_compact_pct: float = 0.8,
@@ -651,9 +651,9 @@ def _run_single_test(
model: str,
system_prompt: str,
case: dict[str, Any],
temperature: float,
temperature: float | None,
max_tokens: int,
reasoning_effort: str,
reasoning_effort: str | None,
context_window: int,
verbose: bool = False,
log_prefix: str = "",
@@ -1100,9 +1100,9 @@ def _run_iteration_parallel(
system_prompt: str,
cases: list[dict[str, Any]],
n_runs: int,
temperature: float,
temperature: float | None,
max_tokens: int,
reasoning_effort: str,
reasoning_effort: str | None,
context_window: int,
test_timeout: int,
parallel: int,
@@ -1251,9 +1251,9 @@ def _run_iteration(
system_prompt: str,
cases: list[dict[str, Any]],
n_runs: int,
temperature: float,
temperature: float | None,
max_tokens: int,
reasoning_effort: str,
reasoning_effort: str | None,
context_window: int,
verbose: bool = False,
test_timeout: int = 300,
@@ -1469,9 +1469,9 @@ def run_skill_adherence(
model: str,
cases: list[dict[str, Any]],
n_runs: int,
temperature: float,
temperature: float | None,
max_tokens: int,
reasoning_effort: str,
reasoning_effort: str | None,
context_window: int,
test_timeout: int = 300,
parallel: int = 1,
+6 -6
View File
@@ -517,9 +517,9 @@ class CoordinatorHeadlessSession(HeadlessSession):
coord_client: CoordinatorClient,
ws_id: str,
user_id: str,
temperature: float,
temperature: float | None,
max_tokens: int,
reasoning_effort: str,
reasoning_effort: str | None,
context_window: int,
) -> None:
super().__init__(
@@ -981,9 +981,9 @@ def _run_single_nudge(
model: str,
case: dict[str, Any],
arm: str,
temperature: float,
temperature: float | None,
max_tokens: int,
reasoning_effort: str,
reasoning_effort: str | None,
context_window: int,
max_turns: int,
test_timeout: int,
@@ -2019,9 +2019,9 @@ def run_nudge_response(
model: str,
cells: list[dict[str, Any]],
n_runs: int = 10,
temperature: float = 0.7,
temperature: float | None = None,
max_tokens: int = 8192,
reasoning_effort: str = "medium",
reasoning_effort: str | None = None,
context_window: int = 131072,
max_turns: int = 8,
test_timeout: int = 300,