mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Patrick Buckley
752fea0fdd
* feat(console): reactive node discovery via PG LISTEN/NOTIFY dispatcher Add a console-side `NotifyDispatcher` that holds a dedicated PostgreSQL `LISTEN` connection and fans wake-ups out to per-channel handlers on a separate dispatch thread. Cluster collector subscribes to a new `services` channel and runs node discovery reactively — new-node / graceful-deregister visibility drops from up-to-60 s to ~500 ms on Postgres, with the 60 s discovery loop retained as the backstop for crash-shaped node loss (NOTIFY only fires on real writes). Storage layer gains a uniform `notify` / `listen` API: - PostgreSQL: real `pg_notify` / `LISTEN` on a dedicated session-mode connection that bypasses pgbouncer (mandatory: pgbouncer is required in transaction-pool mode per docs, which is incompatible with LISTEN). - SQLite: in-process fan-out + synthetic-sweep fallback so consumer code is identical across backends. `TURNSTONE_DB_LISTEN_URL` (or `[database] listen_url` in config.toml) points the dispatcher's connection direct-to-Postgres. Defaults to the main DB URL when unset. Migration 053 installs the `services_notify` trigger; it filters heartbeat-only UPDATEs in-trigger so the 30 s × N-nodes heartbeat tick stays quiet, while INSERT, DELETE, and url/metadata-changing UPDATE still fire. Dispatcher detail: - Two threads: listener (drains stream → bounded queue) and dispatch (invokes handlers under exception suppression). Same-channel notifies coalesce per dispatch batch so an N-node deploy burst is one `_discover_nodes` per channel. - Reconnect uses exponential backoff (1 s → 30 s cap). After any successful reopen — whether the prior failure was a stream-poll error or a connect / initial-LISTEN error — one synthetic Notify with payload="reconcile" is enqueued per channel so handlers re-read on the same code path they use for real events. Future consumers (ConfigStore live reload, scheduler immediate dispatch, audit live-tail) plug in by adding their channel to the dispatcher's construction list. Tests: 22 dispatcher tests (incl. reconnect + coalescing under stub storage), 7 SQLite notify-stream tests, 4 PG-gated trigger-filter tests, 4 collector wire-in tests. All pass; ruff + mypy clean. * fix(notify): address Copilot review on #505 - _sqlite.py: SQLiteBackend.listen() now de-dupes channel names via dict.fromkeys before constructing the stream — duplicates would otherwise register the queue twice and double-deliver each notify. - _sqlite.py: SQLiteBackend.listen() gains a keyword-only sweep_interval parameter (defaults to _SQLITE_NOTIFY_SWEEP_INTERVAL) — matches what the comment at the constant already promised, and lets future consumers without their own polling timer pick a tighter cadence without reaching into private stream attributes. - _sqlite.py: documented the `except queue.Empty: pass` end-of-drain termination so it's not mistaken for swallowing an unexpected error. - _postgresql.py: docstring referenced :func:`_pg_listen_url` which was renamed to _resolve_pg_listen_url during PR development. - notify_dispatcher.py: module docstring referenced a non-existent _bootstrap_console_subsystem; wire-in is at console/server.py::main. Refuted (no change, false positives from github-code-quality bot): - 4× "Statement has no effect" on Protocol-method `...` ellipsis bodies (idiomatic Python Protocol declaration, not dead code). - 2× "Mixed import style" in tests — `import ... as nd_mod` is intentional to allow attribute assignment for monkey-patching the module's `_RECONNECT_BACKOFF_INITIAL` constant inside try/finally.
133 lines
5.3 KiB
TOML
133 lines
5.3 KiB
TOML
# turnstone.toml — shared bootstrap configuration
|
|
#
|
|
# This file is read once at startup. Values here are overridden by
|
|
# environment variables, which are in turn overridden by CLI flags.
|
|
#
|
|
# All sections are optional. Missing sections use binary defaults.
|
|
# Config file location precedence:
|
|
# 1. --config flag
|
|
# 2. $TURNSTONE_CONFIG env var
|
|
# 3. ~/.config/turnstone/config.toml
|
|
|
|
# --- LLM API (turnstone, node, eval) ---
|
|
|
|
[api]
|
|
# base_url = "" # API endpoint; empty = binary default
|
|
# api_key = "" # env: OPENAI_API_KEY or ANTHROPIC_API_KEY
|
|
|
|
# --- Default Model (turnstone, node, eval) ---
|
|
|
|
[model]
|
|
# name = "" # Model ID; empty = provider default (gpt-5 / claude-sonnet-4)
|
|
# temperature = 0.0 # 0 = provider default
|
|
# reasoning_effort = "" # "none", "minimal", "low", "medium", "high", "xhigh", "max"
|
|
# context_window = 0 # 0 = auto-detect from provider capabilities
|
|
# max_tokens = 0 # 0 = provider default
|
|
#
|
|
# Sub-agent routing (plan_agent, task_agent tools). Each falls back to
|
|
# agent_model when unset, then to the session model. Use this to point
|
|
# the rare-but-expensive plan agent at a stronger model than the
|
|
# frequent task agent.
|
|
# agent_model = "" # legacy single-knob: both plan and task share this
|
|
# plan_model = "" # plan_agent override (e.g. "claude" for a smart planner)
|
|
# task_model = "" # task_agent override (e.g. "local" for cheap subtasks)
|
|
# plan_effort = "" # reasoning effort for plan_agent (default: "high")
|
|
# task_effort = "" # reasoning effort for task_agent (default: inherit session)
|
|
#
|
|
# At call time, the calling LLM may also pass `model="<alias>"` to
|
|
# plan_agent / task_agent to override these per-invocation. Tool
|
|
# descriptions list available aliases dynamically; bad aliases return
|
|
# an error so the model retries with a valid choice.
|
|
|
|
# --- Named Models (turnstone, node, eval) ---
|
|
# Define model aliases with per-model overrides. Useful for local model
|
|
# servers or mixing providers. Reference by name with --model flag.
|
|
#
|
|
# [models.local]
|
|
# name = "llama-3-70b"
|
|
# provider = "openai"
|
|
# base_url = "http://localhost:8000/v1"
|
|
# context_window = 8192
|
|
#
|
|
# [models.local.capabilities]
|
|
# supports_vision = false
|
|
# supports_web_search = false
|
|
#
|
|
# [models.claude]
|
|
# name = "claude-opus-4-7"
|
|
# provider = "anthropic"
|
|
|
|
# --- Database (turnstone, node, console) ---
|
|
|
|
[database]
|
|
# url = "" # postgres://user:pass@host/db or /path/to.db
|
|
# env: TURNSTONE_DB_URL
|
|
# listen_url = "" # direct-to-postgres URL for the console's
|
|
# dedicated LISTEN connection. Set this when
|
|
# `url` points at pgbouncer in transaction
|
|
# pooling mode (LISTEN holds session state and
|
|
# is incompatible with transaction pooling —
|
|
# see docs/pgbouncer.md). Defaults to `url`
|
|
# when unset. env: TURNSTONE_DB_LISTEN_URL
|
|
# SSL params (passed through to SQLAlchemy connection):
|
|
# sslmode = "prefer" # disable, allow, prefer, require, verify-ca, verify-full
|
|
# sslrootcert = "" # path to CA cert for verify-ca/verify-full
|
|
# sslcert = "" # path to client cert (mTLS)
|
|
# sslkey = "" # path to client key (mTLS)
|
|
|
|
# --- Auth (node, console) ---
|
|
|
|
[auth]
|
|
# Auth is always enabled. JWT secret is required.
|
|
# jwt_secret = "" # HS256 signing secret (min 32 bytes recommended)
|
|
# env: TURNSTONE_JWT_SECRET
|
|
|
|
# --- Logging (turnstone, node, console) ---
|
|
|
|
[log]
|
|
# level = "" # "debug", "info", "warn", "error"
|
|
# empty = binary default (warn for CLI, info for servers)
|
|
# env: TURNSTONE_LOG_LEVEL
|
|
# json = false # JSON output; auto-enabled when stderr is not a TTY
|
|
|
|
# --- Session (turnstone, node) ---
|
|
|
|
[session]
|
|
# instructions = "" # Default system message
|
|
# compact_max_tokens = 32768 # Max tokens for context compaction summary
|
|
# auto_compact_pct = 0.8 # Trigger compaction at this % of context window
|
|
|
|
# --- Tools (turnstone, node) ---
|
|
|
|
[tools]
|
|
# timeout = 120 # Tool execution timeout in seconds
|
|
# skip_permissions = false # Auto-approve all tool calls
|
|
|
|
# --- Judge (turnstone, node) ---
|
|
|
|
[judge]
|
|
# enabled = true # Enable intent validation
|
|
# confidence_threshold = 0.7 # Minimum confidence for heuristic verdicts
|
|
# output_guard = true # Scan tool output for security signals
|
|
# redact_secrets = true # Redact detected credentials in output
|
|
|
|
# --- Memory (turnstone, node) ---
|
|
|
|
[memory]
|
|
# relevance_k = 5 # Top-K memories for context injection
|
|
# fetch_limit = 50 # Max memories to fetch for ranking
|
|
# max_content = 32768 # Max memory content size in chars
|
|
# nudge_cooldown = 300 # Min seconds between metacognitive nudges
|
|
# nudges = true # Enable memory nudges
|
|
|
|
# --- MCP (turnstone, node) ---
|
|
|
|
[mcp]
|
|
# config_path = "" # Path to MCP servers config file (JSON)
|
|
|
|
# --- Server (node, console) ---
|
|
|
|
[server]
|
|
# max_workstreams = 50 # Maximum concurrent workstreams per node
|
|
# env: TURNSTONE_MAX_WORKSTREAMS
|