diff --git a/Dockerfile b/Dockerfile index ade6b817..4267c46a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,9 +10,12 @@ LABEL org.opencontainers.image.title="turnstone" \ COPY --from=ghcr.io/astral-sh/uv:0.11.2 /uv /usr/local/bin/uv +# Remove the slim image's man page exclusion so man-db has actual content +RUN rm -f /etc/dpkg/dpkg.cfg.d/docker + # System dependencies: psycopg (libpq5), developer tooling for agent workflows RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ - libpq5 git curl jq man-db info \ + libpq5 git curl jq man-db manpages \ && rm -rf /var/lib/apt/lists/* # Non-root user diff --git a/docs/architecture.md b/docs/architecture.md index 6c93cfc0..b967b957 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -3,7 +3,7 @@ Turnstone is an AI orchestration platform with tool use, parallel workstreams, and persistent memory. It connects to any OpenAI-compatible API (local vLLM, OpenAI, etc.) or Anthropic's native Messages API via pluggable provider adapters, and gives the -model 17 built-in tools plus external tools via MCP (Model Context Protocol) for +model 19 built-in tools plus external tools via MCP (Model Context Protocol) for reading, writing, searching, planning, and executing code. The core design principle is a **UI-agnostic engine with pluggable frontends**. diff --git a/docs/diagrams/02-package-structure.puml b/docs/diagrams/02-package-structure.puml index 7df1ef9c..7f20462d 100644 --- a/docs/diagrams/02-package-structure.puml +++ b/docs/diagrams/02-package-structure.puml @@ -97,7 +97,7 @@ package "turnstone/sdk/" <> { ' Tool schemas package "turnstone/tools/" <> { - component [*.json\n18 tool schemas] as schemas <> + component [*.json\n19 tool schemas] as schemas <> } ' Entry point dependencies diff --git a/docs/diagrams/05-tool-pipeline.puml b/docs/diagrams/05-tool-pipeline.puml index a903b2bc..3b330887 100644 --- a/docs/diagrams/05-tool-pipeline.puml +++ b/docs/diagrams/05-tool-pipeline.puml @@ -24,7 +24,7 @@ partition "Phase 1: Prepare" #E8F5E9 { :Dispatch to _prepare_{func_name}(); note right - **Dispatch table (17 tools):** + **Dispatch table (19 built-in + tool_search):** ┌───────────────┬──────────────────┐ │ Tool │ Needs Approval? │ ├───────────────┼──────────────────┤ @@ -33,16 +33,19 @@ partition "Phase 1: Prepare" #E8F5E9 { │ write_file │ ✓ Yes │ │ edit_file │ ✓ Yes │ │ search │ ✗ Auto-approve │ + │ diff_file │ ✗ Auto-approve │ │ math │ ✗ Auto-approve │ │ man │ ✗ Auto-approve │ │ web_fetch │ ✗ Auto-approve │ │ web_search │ ✗ Auto-approve │ │ tool_search │ ✗ Auto-approve │ - │ task │ ✓ Yes │ - │ plan │ ✓ Yes │ + │ task_agent │ ✓ Yes │ + │ plan_agent │ ✓ Yes │ │ memory │ ✗ Auto-approve │ │ recall │ ✗ Auto-approve │ │ notify │ ✗ Auto-approve │ + │ watch │ ✓ create only │ + │ skill │ ✓ load only │ │ read_resource │ ✓ Yes │ │ use_prompt │ ✓ Yes │ ├───────────────┼──────────────────┤ diff --git a/docs/diagrams/png/02-package-structure.png b/docs/diagrams/png/02-package-structure.png index e83dc98b..66ce909a 100644 --- a/docs/diagrams/png/02-package-structure.png +++ b/docs/diagrams/png/02-package-structure.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:efcc7cbe8161a54b5ec24bdfd47e8a142f70029e6e66c707e811b99369f85ebf -size 310079 +oid sha256:2b3ea69f852e93dc1bc7943db0c71d0cdd1afcbcf470a8737189ae56e85b3206 +size 310075 diff --git a/docs/diagrams/png/05-tool-pipeline.png b/docs/diagrams/png/05-tool-pipeline.png index aadf5ca6..26bc06ca 100644 --- a/docs/diagrams/png/05-tool-pipeline.png +++ b/docs/diagrams/png/05-tool-pipeline.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:72b3932ce99a860f5069544cd8423d3cdae3a51f6566db262b19ced19c780eb0 -size 274374 +oid sha256:674712a0563f51837383184652efeb28b7bec13378be636e89d2959bfba39d1e +size 281519 diff --git a/docs/tools.md b/docs/tools.md index db1a1241..8d79c687 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -1,6 +1,6 @@ # Tools Reference -turnstone exposes 18 built-in tools plus any number of external MCP tools to the +turnstone exposes 19 built-in tools plus any number of external MCP tools to the LLM via the OpenAI function-calling interface. Built-in tools are defined as JSON files under `turnstone/tools/` and loaded at startup by `turnstone/core/tools.py`. MCP tools are discovered from configured MCP servers at startup by @@ -46,12 +46,12 @@ schema plus turnstone-specific metadata keys: | Name | Description | |---------------------|-------------| -| `TOOLS` | All 17 tool definitions (sent to the model). | +| `TOOLS` | All 19 tool definitions (sent to the model). | | `AGENT_TOOLS` | Tools with `agent: true` -- available to plan sub-agents. Read-only tools. | | `TASK_AGENT_TOOLS` | Tools with `task_agent: true` -- available to task sub-agents. Includes write operations. | | `AGENT_AUTO_TOOLS` | Set of tool names with `auto_approve: true` -- no user confirmation needed. | | `TASK_AUTO_TOOLS` | Same as `AGENT_AUTO_TOOLS` (identical filter). | -| `BUILTIN_TOOL_NAMES`| Frozenset of all 17 built-in tool names. Used by tool search to distinguish always-on tools from deferrable MCP tools. | +| `BUILTIN_TOOL_NAMES`| Frozenset of all 19 built-in tool names. Used by tool search to distinguish always-on tools from deferrable MCP tools. | | `PRIMARY_KEY_MAP` | Dict mapping tool name to its `primary_key` parameter name. | --- @@ -69,7 +69,7 @@ Tool execution follows a three-phase pipeline inside `ChatSession._execute_tools - Parses the JSON arguments (with fallback for malformed JSON). - If JSON parsing fails entirely, uses `PRIMARY_KEY_MAP` to map a bare string to the correct parameter. -- Dispatches to the matching `_prepare_{func_name}()` handler. There are 17 +- Dispatches to the matching `_prepare_{func_name}()` handler. There are 19 built-in tools plus `tool_search` (synthetic, client-side BM25 fallback) and the generic `_prepare_mcp_tool()` handler for MCP tools. - Validates arguments and builds a preview dict containing: @@ -188,8 +188,10 @@ Execute a bash command and return stdout + stderr. | Parameter | Type | Required | Description | |-----------|--------|----------|-------------| | `command` | string | yes | The bash command to execute. | +| `timeout` | integer | no | Timeout in seconds (1-600). Omit to use the global `tools.timeout` setting (typically 120s). | +| `stop_on_error` | boolean | no | Enable `set -e` so the script exits on the first command failure. Default false. | -- **What it does**: Runs the command in a subprocess with a configurable timeout (default 120s). Commands are sanitized and checked against a blocklist (e.g. `rm -rf /`). Environment variables containing secrets are scrubbed (`*_KEY`, `*_SECRET`, `*_TOKEN`, etc.). +- **What it does**: Runs the command in a subprocess with a configurable timeout. Commands are sanitized and checked against a blocklist (e.g. `rm -rf /`). Environment variables containing secrets are scrubbed (`*_KEY`, `*_SECRET`, `*_TOKEN`, etc.). - **Output format**: Stdout is returned directly. Stderr lines are prefixed with `[stderr]` so the model can distinguish them. When the command itself redirects stderr to stdout (`2>&1`), no prefix is added. Output exceeding 256KB is truncated (head + tail preserved, middle replaced with a truncation notice). - **Auto-approve**: No -- requires user confirmation. - **Agent availability**: `task_agent` only (not available to plan sub-agents). @@ -222,8 +224,9 @@ Write content to a file, creating it if needed. |-----------|--------|----------|-------------| | `path` | string | yes | Absolute or relative file path. | | `content` | string | yes | The full file content to write. | +| `mode` | string | no | `"overwrite"` (default) replaces the file. `"append"` adds content to the end. | -- **What it does**: Creates or overwrites the file at the given path. Parent directories are created as needed. +- **What it does**: Creates or overwrites (or appends to) the file at the given path. Parent directories are created as needed. - **Auto-approve**: No -- requires user confirmation. - **Agent availability**: `task_agent` only. @@ -239,17 +242,36 @@ Replace exact strings in a file, or apply multiple replacements atomically. | `old_string` | string | no* | The exact text to find and replace. | | `new_string` | string | no* | The replacement text. | | `near_line` | integer | no | Disambiguate when `old_string` matches multiple locations. | -| `edits` | array | no* | Multiple replacements to apply atomically (see below). | +| `edits` | array | no* | Multiple replacements to apply atomically (see below). | +| `replace_all` | boolean | no | Replace ALL occurrences of `old_string`. Cannot combine with `near_line` or `edits`. | \* Provide either `old_string`+`new_string` (single edit) or `edits` array (batch), not both. -- **What it does**: Finds `old_string` in the file and replaces it with `new_string`. Fails if the string is not found or matches multiple locations (unless `near_line` is provided to pick the nearest match). Requires a prior `read_file` call on the same path. +- **What it does**: Finds `old_string` in the file and replaces it with `new_string`. Fails if the string is not found or matches multiple locations (unless `near_line` or `replace_all` is provided). Requires a prior `read_file` or `diff_file` call on the same path. - **Batch mode**: The `edits` array accepts multiple `{old_string, new_string, near_line?}` entries applied atomically. All edits are validated before any are applied. Overlapping edits (two entries targeting the same text region) are rejected. Edits are applied in reverse file-position order so character offsets stay stable. +- **Replace-all mode**: When `replace_all` is true, all occurrences are replaced via `str.replace()`. The approval preview shows the occurrence count. - **Auto-approve**: No -- requires user confirmation. - **Agent availability**: `task_agent` only. --- +### diff_file + +Show a unified diff between two files, or between a file and a provided string. + +| Parameter | Type | Required | Description | +|-----------------|---------|----------|-------------| +| `path_a` | string | yes | Path to the first file. | +| `path_b` | string | no | Path to the second file. Mutually exclusive with `content_b`. | +| `content_b` | string | no | String content to compare against `path_a`. Mutually exclusive with `path_b`. | +| `context_lines` | integer | no | Number of context lines around changes (default 3, max 20). | + +- **What it does**: Returns unified diff output using Python's `difflib`. Binary files (containing null bytes) are rejected with a clear error. Files read through `diff_file` satisfy `edit_file`'s read guard — you can diff then edit without a separate `read_file` call. Large diffs are streamed with early cutoff at the tool truncation limit. +- **Auto-approve**: Yes (read-only). +- **Agent availability**: `agent` and `task_agent`. + +--- + ### search Search file contents for a regex pattern. @@ -607,7 +629,7 @@ CLI flags override the config file: directly. 2. **Partitioning**: When active, tools are split into two sets: - - **Always-on** -- the 17 built-in tools (members of `BUILTIN_TOOL_NAMES`). + - **Always-on** -- the 19 built-in tools (members of `BUILTIN_TOOL_NAMES`). These are always visible to the model. - **Deferred** -- all MCP tools. These are not sent in the tool list unless the model searches for them. @@ -650,7 +672,7 @@ MCP-compatible service. 3. **Schema conversion**: Each MCP tool's `inputSchema` is converted to OpenAI function-calling format. The tool name is prefixed: `mcp__{server}__{tool}`. -4. **Merging**: MCP tools are appended after the 17 built-in tools via +4. **Merging**: MCP tools are appended after the 19 built-in tools via `merge_mcp_tools()`. Built-in tools appear first, giving them natural LLM priority. When dynamic tool search is active, MCP tools are deferred rather than directly visible -- the model discovers them via search as needed (see diff --git a/turnstone/core/output_guard.py b/turnstone/core/output_guard.py index a1093ec1..caf29273 100644 --- a/turnstone/core/output_guard.py +++ b/turnstone/core/output_guard.py @@ -51,11 +51,13 @@ _RE_PRIVATE_KEY_BLOCK = re.compile( r"-----END\s+(?:RSA\s+|EC\s+|OPENSSH\s+|PGP\s+)?PRIVATE\s+KEY-----", ) _RE_CONNECTION_STRING = re.compile( - r"(?:postgresql|mysql|mongodb|redis|amqp)://[^:@\s]+:[^@\s]+@", + r"(?:postgresql\+?(?:psycopg)?|mysql|mongodb|redis|amqp|sqlite)://[^:@\s]+:[^@\s]+@", ) _RE_ENV_SECRET_LINE = re.compile(r"[A-Z][A-Z_0-9]+=\S+") _RE_ENV_SECRET_KEY = re.compile( - r"(?:^|_)(?:SECRET|TOKEN|PASSWORD|CREDENTIAL)(?:_|$)|(?:^|_)KEY(?:_|$)", + r"(?:^|_)(?:SECRET|TOKEN|PASSWORD|CREDENTIAL|DSN)(?:_|$)" + r"|(?:^|_)KEY(?:_|$)" + r"|^(?:DATABASE_URL|TURNSTONE_DB_URL|DB_URL)$", re.IGNORECASE, ) _RE_JSON_SECRET = re.compile(