mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
fix(eval): stop pinning temperature and reasoning effort on the wire
The eval CLI defaulted temperature to 0.7 and reasoning effort to medium, so every sweep sent code-chosen sampling knobs the house assignment scheme forbids — the wire should omit the fields and let the alias / stored setting / serving default apply, as production does. Both flags now default to unset and the harnesses plumb None through to model_turn, whose provider layer already omits absent knobs. Absolute numbers from earlier sweeps were collected under the pinned values; contrasts were at least uniform under the same pin.
This commit is contained in:
+12
-4
@@ -184,8 +184,12 @@ def main() -> None:
|
||||
parser.add_argument(
|
||||
"--temperature",
|
||||
type=float,
|
||||
default=0.7,
|
||||
help="Sampling temperature (default: 0.7)",
|
||||
default=None,
|
||||
help=(
|
||||
"Sampling temperature. Omitted from the wire by default so "
|
||||
"the alias / stored setting / serving default applies — the "
|
||||
"same assignment scheme production runs"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-tokens",
|
||||
@@ -195,9 +199,13 @@ def main() -> None:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reasoning-effort",
|
||||
default="medium",
|
||||
default=None,
|
||||
choices=["low", "medium", "high"],
|
||||
help="Reasoning effort (default: medium)",
|
||||
help=(
|
||||
"Reasoning effort. Omitted from the wire by default — a "
|
||||
"code-chosen token is unvetted and can flip thinking on for "
|
||||
"lanes the operator never engaged"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--context-window",
|
||||
|
||||
+10
-10
@@ -238,10 +238,10 @@ class HeadlessSession(ChatSession):
|
||||
model: str,
|
||||
system_prompt_override: str | None = None,
|
||||
instructions: str | None = None,
|
||||
temperature: float = 0.7,
|
||||
temperature: float | None = None,
|
||||
max_tokens: int = 32768,
|
||||
tool_timeout: int = 30,
|
||||
reasoning_effort: str = "medium",
|
||||
reasoning_effort: str | None = None,
|
||||
context_window: int = 131072,
|
||||
compact_max_tokens: int = 32768,
|
||||
auto_compact_pct: float = 0.8,
|
||||
@@ -651,9 +651,9 @@ def _run_single_test(
|
||||
model: str,
|
||||
system_prompt: str,
|
||||
case: dict[str, Any],
|
||||
temperature: float,
|
||||
temperature: float | None,
|
||||
max_tokens: int,
|
||||
reasoning_effort: str,
|
||||
reasoning_effort: str | None,
|
||||
context_window: int,
|
||||
verbose: bool = False,
|
||||
log_prefix: str = "",
|
||||
@@ -1100,9 +1100,9 @@ def _run_iteration_parallel(
|
||||
system_prompt: str,
|
||||
cases: list[dict[str, Any]],
|
||||
n_runs: int,
|
||||
temperature: float,
|
||||
temperature: float | None,
|
||||
max_tokens: int,
|
||||
reasoning_effort: str,
|
||||
reasoning_effort: str | None,
|
||||
context_window: int,
|
||||
test_timeout: int,
|
||||
parallel: int,
|
||||
@@ -1251,9 +1251,9 @@ def _run_iteration(
|
||||
system_prompt: str,
|
||||
cases: list[dict[str, Any]],
|
||||
n_runs: int,
|
||||
temperature: float,
|
||||
temperature: float | None,
|
||||
max_tokens: int,
|
||||
reasoning_effort: str,
|
||||
reasoning_effort: str | None,
|
||||
context_window: int,
|
||||
verbose: bool = False,
|
||||
test_timeout: int = 300,
|
||||
@@ -1469,9 +1469,9 @@ def run_skill_adherence(
|
||||
model: str,
|
||||
cases: list[dict[str, Any]],
|
||||
n_runs: int,
|
||||
temperature: float,
|
||||
temperature: float | None,
|
||||
max_tokens: int,
|
||||
reasoning_effort: str,
|
||||
reasoning_effort: str | None,
|
||||
context_window: int,
|
||||
test_timeout: int = 300,
|
||||
parallel: int = 1,
|
||||
|
||||
@@ -517,9 +517,9 @@ class CoordinatorHeadlessSession(HeadlessSession):
|
||||
coord_client: CoordinatorClient,
|
||||
ws_id: str,
|
||||
user_id: str,
|
||||
temperature: float,
|
||||
temperature: float | None,
|
||||
max_tokens: int,
|
||||
reasoning_effort: str,
|
||||
reasoning_effort: str | None,
|
||||
context_window: int,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
@@ -981,9 +981,9 @@ def _run_single_nudge(
|
||||
model: str,
|
||||
case: dict[str, Any],
|
||||
arm: str,
|
||||
temperature: float,
|
||||
temperature: float | None,
|
||||
max_tokens: int,
|
||||
reasoning_effort: str,
|
||||
reasoning_effort: str | None,
|
||||
context_window: int,
|
||||
max_turns: int,
|
||||
test_timeout: int,
|
||||
@@ -2019,9 +2019,9 @@ def run_nudge_response(
|
||||
model: str,
|
||||
cells: list[dict[str, Any]],
|
||||
n_runs: int = 10,
|
||||
temperature: float = 0.7,
|
||||
temperature: float | None = None,
|
||||
max_tokens: int = 8192,
|
||||
reasoning_effort: str = "medium",
|
||||
reasoning_effort: str | None = None,
|
||||
context_window: int = 131072,
|
||||
max_turns: int = 8,
|
||||
test_timeout: int = 300,
|
||||
|
||||
Reference in New Issue
Block a user