mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-13 23:42:25 -06:00
Compare commits
49 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c2c6689a8e | |||
| f5af4875ba | |||
| 0d77d65266 | |||
| c11991819e | |||
| fc948d711d | |||
| d9722f3578 | |||
| f494553020 | |||
| c766c81f25 | |||
| 3b746fb28d | |||
| b82fa4923c | |||
| 4ac316dc0e | |||
| 387ef06da7 | |||
| e4e2200c33 | |||
| 8b94d553e4 | |||
| cf16724137 | |||
| 22402e89de | |||
| e7743fd079 | |||
| 27349e1c13 | |||
| 37a48bb30d | |||
| 730f5704ff | |||
| 9b3b1c1ddd | |||
| a9ed8a954b | |||
| 1efcbcf2ba | |||
| 1d36d80fe5 | |||
| e603a6a7d1 | |||
| 2e95f2ac73 | |||
| 5f27ed9fca | |||
| 20df7b3034 | |||
| 68c991fbdd | |||
| 376da3d084 | |||
| e2a199c9c3 | |||
| 4152ea2352 | |||
| 44cc14b46f | |||
| 83b0cde32f | |||
| 2ef8a8711b | |||
| 3658b77de8 | |||
| 83577739e0 | |||
| 71d13936fe | |||
| 0cd061196c | |||
| 19abc0cc65 | |||
| c5cdfc8f44 | |||
| 8895bf07eb | |||
| 101afd84da | |||
| efd98712e9 | |||
| 67f43a7ee0 | |||
| 2888e8ce0a | |||
| d1a248b413 | |||
| 723cad24bb | |||
| 73cacc8ad6 |
@@ -0,0 +1,208 @@
|
||||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"config:recommended",
|
||||
"helpers:pinGitHubActionDigests",
|
||||
":separateMajorReleases"
|
||||
],
|
||||
"labels": [
|
||||
"dependencies"
|
||||
],
|
||||
"prConcurrentLimit": 5,
|
||||
"prHourlyLimit": 2,
|
||||
"schedule": [
|
||||
"before 9am on Monday"
|
||||
],
|
||||
"timezone": "America/New_York",
|
||||
"lockFileMaintenance": {
|
||||
"enabled": true,
|
||||
"schedule": [
|
||||
"before 9am on Monday"
|
||||
]
|
||||
},
|
||||
"customManagers": [
|
||||
{
|
||||
"customType": "regex",
|
||||
"description": "Track vendored KaTeX version",
|
||||
"managerFilePatterns": [
|
||||
"/pyproject\\.toml$/"
|
||||
],
|
||||
"matchStrings": [
|
||||
"katex-(?<currentValue>[\\d.]+)/"
|
||||
],
|
||||
"depNameTemplate": "katex",
|
||||
"datasourceTemplate": "npm"
|
||||
},
|
||||
{
|
||||
"customType": "regex",
|
||||
"description": "Track vendored Highlight.js version",
|
||||
"managerFilePatterns": [
|
||||
"/pyproject\\.toml$/"
|
||||
],
|
||||
"matchStrings": [
|
||||
"hljs-(?<currentValue>[\\d.]+)/"
|
||||
],
|
||||
"depNameTemplate": "highlight.js",
|
||||
"datasourceTemplate": "npm"
|
||||
},
|
||||
{
|
||||
"customType": "regex",
|
||||
"description": "Track vendored Mermaid version",
|
||||
"managerFilePatterns": [
|
||||
"/pyproject\\.toml$/"
|
||||
],
|
||||
"matchStrings": [
|
||||
"mermaid-(?<currentValue>[\\d.]+)/"
|
||||
],
|
||||
"depNameTemplate": "mermaid",
|
||||
"datasourceTemplate": "npm"
|
||||
}
|
||||
],
|
||||
"packageRules": [
|
||||
{
|
||||
"description": "LLM SDKs — always review manually",
|
||||
"groupName": "LLM SDKs",
|
||||
"matchPackageNames": [
|
||||
"openai",
|
||||
"anthropic",
|
||||
"mcp"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on Monday"
|
||||
],
|
||||
"automerge": false
|
||||
},
|
||||
{
|
||||
"description": "Web framework stack",
|
||||
"groupName": "Web Framework",
|
||||
"matchPackageNames": [
|
||||
"starlette",
|
||||
"uvicorn",
|
||||
"sse-starlette",
|
||||
"httpx",
|
||||
"httpx-sse",
|
||||
"pydantic"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on Wednesday"
|
||||
],
|
||||
"automerge": true,
|
||||
"matchUpdateTypes": [
|
||||
"patch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Database layer",
|
||||
"groupName": "Database",
|
||||
"matchPackageNames": [
|
||||
"sqlalchemy",
|
||||
"alembic",
|
||||
"psycopg"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on Wednesday"
|
||||
],
|
||||
"automerge": true,
|
||||
"matchUpdateTypes": [
|
||||
"patch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Security-critical — always review manually",
|
||||
"groupName": "Security",
|
||||
"matchPackageNames": [
|
||||
"PyJWT",
|
||||
"pyjwt",
|
||||
"bcrypt"
|
||||
],
|
||||
"automerge": false
|
||||
},
|
||||
{
|
||||
"description": "Infrastructure dependencies",
|
||||
"groupName": "Infrastructure",
|
||||
"matchPackageNames": [
|
||||
"structlog",
|
||||
"redis",
|
||||
"croniter",
|
||||
"discord.py"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on the first day of the month"
|
||||
],
|
||||
"automerge": true,
|
||||
"matchUpdateTypes": [
|
||||
"patch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Vendored JS — requires manual file download after merge",
|
||||
"groupName": "Vendored JS",
|
||||
"matchPackageNames": [
|
||||
"katex",
|
||||
"highlight.js",
|
||||
"mermaid"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on the first day of the month"
|
||||
],
|
||||
"automerge": false,
|
||||
"prBodyNotes": [
|
||||
"This PR updates version references only.",
|
||||
"After merging, run `scripts/update-vendored-js.sh <lib> <version>` to download the actual files."
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Dev/test tooling",
|
||||
"groupName": "Tooling",
|
||||
"matchPackageNames": [
|
||||
"ruff",
|
||||
"mypy",
|
||||
"types-redis",
|
||||
"pytest",
|
||||
"pytest-cov",
|
||||
"pre-commit"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on the first day of the month"
|
||||
],
|
||||
"automerge": true,
|
||||
"matchUpdateTypes": [
|
||||
"patch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Docker base images",
|
||||
"groupName": "Docker Images",
|
||||
"matchManagers": [
|
||||
"dockerfile",
|
||||
"docker-compose"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on the first day of the month"
|
||||
],
|
||||
"automerge": false
|
||||
},
|
||||
{
|
||||
"description": "TypeScript SDK dev dependencies",
|
||||
"groupName": "TypeScript SDK",
|
||||
"matchFileNames": [
|
||||
"sdk/typescript/**"
|
||||
],
|
||||
"schedule": [
|
||||
"before 9am on the first day of the month"
|
||||
],
|
||||
"automerge": true,
|
||||
"matchUpdateTypes": [
|
||||
"patch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "GitHub Actions — group all action updates",
|
||||
"groupName": "GitHub Actions",
|
||||
"matchManagers": [
|
||||
"github-actions"
|
||||
],
|
||||
"automerge": false
|
||||
}
|
||||
]
|
||||
}
|
||||
+48
-12
@@ -10,21 +10,21 @@ jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- run: pip install ruff
|
||||
- run: ruff check turnstone/ tests/
|
||||
- run: ruff format --check turnstone/ tests/
|
||||
python-version: "3.14"
|
||||
- run: pip install pre-commit
|
||||
# mypy runs separately in typecheck job with full project deps
|
||||
- run: SKIP=mypy pre-commit run --all-files
|
||||
|
||||
typecheck:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
python-version: "3.14"
|
||||
- run: pip install mypy types-redis
|
||||
- run: pip install -e ".[mq]"
|
||||
- run: mypy turnstone/
|
||||
@@ -35,14 +35,50 @@ jobs:
|
||||
matrix:
|
||||
python-version: ["3.11", "3.12", "3.13"]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- run: pip install -e ".[test,mq]"
|
||||
- run: pytest tests/ -m "not live" --cov=turnstone --cov-report=term-missing --cov-report=xml -q
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
if: always()
|
||||
with:
|
||||
name: coverage-${{ matrix.python-version }}
|
||||
path: coverage.xml
|
||||
|
||||
lock-check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: astral-sh/setup-uv@e06108dd0aef18192324c70427afc47652e63a82 # v7
|
||||
with:
|
||||
uv-version: "0.9.18"
|
||||
- run: uv lock --check
|
||||
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: astral-sh/setup-uv@e06108dd0aef18192324c70427afc47652e63a82 # v7
|
||||
with:
|
||||
uv-version: "0.9.18"
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- run: uv sync --frozen --all-extras
|
||||
- run: uv pip install pip-audit
|
||||
- run: uv run pip-audit --strict --desc
|
||||
|
||||
security-ts:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: sdk/typescript
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
|
||||
with:
|
||||
node-version: "24"
|
||||
- run: npm ci
|
||||
- run: npm audit --audit-level=moderate
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
name: Docker Security Scan
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
schedule:
|
||||
- cron: "0 6 * * 1" # Weekly Monday 06:00 UTC
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- run: docker build -t turnstone:scan .
|
||||
- uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # 0.35.0
|
||||
with:
|
||||
image-ref: "turnstone:scan"
|
||||
severity: "HIGH,CRITICAL"
|
||||
exit-code: "1"
|
||||
@@ -13,16 +13,16 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
environment: pypi
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
python-version: "3.14"
|
||||
- run: pip install build
|
||||
- run: python -m build
|
||||
- uses: pypa/gh-action-pypi-publish@release/v1
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@b25b93d384199fc0fc8c2e126b2d937a0cbeb2ae # v2
|
||||
with:
|
||||
generate_release_notes: true
|
||||
draft: false
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
repos:
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.9.10
|
||||
rev: v0.15.6
|
||||
hooks:
|
||||
- id: ruff
|
||||
args: [--fix]
|
||||
- id: ruff-format
|
||||
|
||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
||||
rev: v1.14.1
|
||||
rev: v1.19.1
|
||||
hooks:
|
||||
- id: mypy
|
||||
additional_dependencies: [types-redis>=4.6, redis>=7.2]
|
||||
|
||||
+22
-24
@@ -1,41 +1,39 @@
|
||||
# =============================================================================
|
||||
# Turnstone — multi-stage Docker build
|
||||
# Turnstone — Docker build with uv for reproducible, locked installs
|
||||
# Single image for all services: server, bridge, console, sim, eval
|
||||
# =============================================================================
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Stage 1: Builder — build the wheel
|
||||
# ----------------------------------------------------------------------------
|
||||
FROM python:3.13-slim AS builder
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
RUN pip install --no-cache-dir hatchling
|
||||
|
||||
COPY pyproject.toml README.md LICENSE ./
|
||||
COPY turnstone/ turnstone/
|
||||
|
||||
RUN pip wheel --no-deps --wheel-dir /build/wheels .
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Stage 2: Runtime — slim image with the installed package
|
||||
# ----------------------------------------------------------------------------
|
||||
FROM python:3.13-slim
|
||||
FROM python:3.14-slim
|
||||
|
||||
LABEL org.opencontainers.image.title="turnstone" \
|
||||
org.opencontainers.image.description="Multi-node AI orchestration platform"
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.10.10 /uv /usr/local/bin/uv
|
||||
|
||||
# System dependencies for psycopg (PostgreSQL client library)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends libpq5 \
|
||||
RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends libpq5 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Non-root user
|
||||
RUN useradd --create-home --shell /bin/bash turnstone
|
||||
|
||||
# Install the wheel with all optional extras
|
||||
COPY --from=builder /build/wheels/*.whl /tmp/wheels/
|
||||
RUN pip install --no-cache-dir "$(ls /tmp/wheels/*.whl)[mq,console,sim,postgres,discord]" \
|
||||
&& rm -rf /tmp/wheels
|
||||
WORKDIR /app
|
||||
|
||||
# Compile bytecode for faster startup
|
||||
ENV UV_COMPILE_BYTECODE=1
|
||||
|
||||
# Install dependencies first (cached layer — only re-runs when deps change)
|
||||
COPY pyproject.toml uv.lock README.md LICENSE ./
|
||||
RUN uv sync --frozen --no-install-project --no-dev \
|
||||
--extra mq --extra console --extra sim --extra postgres --extra discord --extra anthropic
|
||||
|
||||
# Install the project itself
|
||||
COPY turnstone/ turnstone/
|
||||
RUN uv sync --frozen --no-dev \
|
||||
--extra mq --extra console --extra sim --extra postgres --extra discord --extra anthropic
|
||||
|
||||
# Add venv to PATH so entry points are found
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
|
||||
# Health check script (stdlib only, no pip deps needed)
|
||||
COPY docker/healthcheck.py /usr/local/bin/healthcheck.py
|
||||
|
||||
@@ -18,7 +18,7 @@ Turnstone gives LLMs tools — shell, files, search, web, planning — and orche
|
||||
- **Multi-node clusters** — generic work load-balances across nodes, directed work routes to a specific server
|
||||
- **Cluster dashboard** — real-time view of all nodes and workstreams, reverse proxy for server UIs
|
||||
- **Intent validation** — an LLM judge evaluates every tool call before approval, presenting risk assessments and evidence-based recommendations so users can make informed decisions instead of blindly approving raw tool calls
|
||||
- **Governance & compliance** — RBAC, tool policies, prompt templates, workstream templates, usage tracking, and append-only audit logs
|
||||
- **Governance & compliance** — RBAC, OIDC SSO (Okta, Azure AD, Google, Keycloak), tool policies, prompt templates, workstream templates, usage tracking, and append-only audit logs
|
||||
- **Cluster simulator** — test the stack at scale (up to 1000 nodes) without an LLM backend
|
||||
|
||||
Works with any OpenAI-compatible API (vLLM, llama.cpp, NVIDIA NIM) or Anthropic's native Messages API. Supports [MCP](https://modelcontextprotocol.io/) for external tool servers with native deferred tool loading on Anthropic and OpenAI APIs (BM25 fallback for local models).
|
||||
@@ -136,18 +136,20 @@ Detailed UML diagrams are available in [`docs/diagrams/`](docs/diagrams/):
|
||||
| [Governance Architecture](docs/diagrams/png/19-governance-architecture.png) | RBAC, policies, audit, usage enforcement flow |
|
||||
| [WS Template Architecture](docs/diagrams/png/21-ws-template-architecture.png) | Workstream template application and lifecycle |
|
||||
| [Judge Architecture](docs/diagrams/png/22-judge-architecture.png) | Intent validation two-tier evaluation pipeline |
|
||||
| [OIDC Architecture](docs/diagrams/png/25-oidc-architecture.png) | OIDC SSO authorization code flow with PKCE |
|
||||
|
||||
### Governance
|
||||
|
||||
Turnstone includes a built-in governance layer for enterprise deployments — manage who can do what, which tools run unattended, and where every token goes.
|
||||
|
||||
- **RBAC** — 15 granular permissions, 3 built-in roles (admin / operator / viewer), custom roles, privilege escalation prevention
|
||||
- **OIDC SSO** — single sign-on via any OpenID Connect provider (Okta, Azure AD, Google, Keycloak); Authorization Code Flow with PKCE, auto-provisioning, claim-based role mapping with demotion propagation; see [docs/oidc.md](docs/oidc.md)
|
||||
- **Tool policies** — glob-pattern rules (`allow` / `deny` / `ask`) with priority ordering; automate approvals or lock down dangerous tools
|
||||
- **Prompt templates** — reusable system messages with `{{variable}}` substitution and categories
|
||||
- **Usage tracking** — per-request token and tool metrics, aggregation by day / model / user, automatic 90-day pruning
|
||||
- **Audit logging** — append-only event trail for all admin mutations, IP-aware, 365-day retention
|
||||
|
||||
All governance features are managed through the console admin panel (10 tabs) and the full REST API. See [docs/governance.md](docs/governance.md) for setup and configuration.
|
||||
All governance features are managed through the console admin panel (13 tabs) and the full REST API. Runtime settings (model, tools, rate limiting, health, judge, memory) are configurable via the admin Settings tab — no config file edits or restarts needed for most changes. See [docs/governance.md](docs/governance.md) for setup and [docs/settings.md](docs/settings.md) for the settings reference.
|
||||
|
||||
### Intent Validation (LLM Judge)
|
||||
|
||||
@@ -193,7 +195,7 @@ Bridges BLPOP from their per-node queue (priority) then the shared queue. Direct
|
||||
|
||||
## Tools
|
||||
|
||||
16 built-in tools, 2 agent tools, plus external tools via MCP:
|
||||
15 built-in tools, 2 agent tools, plus external tools via MCP:
|
||||
|
||||
| Tool | Description | Auto-approved |
|
||||
|------|-------------|:---:|
|
||||
@@ -206,9 +208,8 @@ Bridges BLPOP from their per-node queue (priority) then the shared queue. Direct
|
||||
| `man` | Read man pages | yes |
|
||||
| `web_fetch` | Fetch URL content | |
|
||||
| `web_search` | Web search (provider-native or Tavily) | |
|
||||
| `remember` | Save persistent facts | yes |
|
||||
| `recall` | Search memories and history | yes |
|
||||
| `forget` | Remove a memory | yes |
|
||||
| `memory` | Structured persistent memory (save/search/delete/list) | yes |
|
||||
| `recall` | Search conversation history | yes |
|
||||
| `notify` | Send notifications to linked channels | yes |
|
||||
| `watch` | Periodic command polling with conditions | |
|
||||
| `task` | Spawn autonomous sub-agent | |
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
Turnstone — Third-Party Notices
|
||||
|
||||
This file contains the licenses and notices for third-party software bundled
|
||||
with Turnstone. Each bundled dependency retains its original license; the
|
||||
Turnstone BUSL-1.1 license does not apply to these components.
|
||||
|
||||
================================================================================
|
||||
|
||||
KaTeX 0.16.38
|
||||
https://katex.org/
|
||||
https://github.com/KaTeX/KaTeX
|
||||
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2013-2020 Khan Academy and other contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
================================================================================
|
||||
|
||||
highlight.js 11.11.1
|
||||
https://highlightjs.org/
|
||||
https://github.com/highlightjs/highlight.js
|
||||
|
||||
BSD 3-Clause License
|
||||
|
||||
Copyright (c) 2006, Ivan Sagalaev.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
================================================================================
|
||||
|
||||
Mermaid 11.13.0
|
||||
https://mermaid.js.org/
|
||||
https://github.com/mermaid-js/mermaid
|
||||
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2014-2022 Knut Sveidqvist
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
+5
-4
@@ -26,7 +26,7 @@ services:
|
||||
# PostgreSQL — production database (profile: production)
|
||||
# -------------------------------------------------------------------
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
image: pgautoupgrade/pgautoupgrade:18-alpine
|
||||
profiles:
|
||||
- production
|
||||
- cluster
|
||||
@@ -35,6 +35,7 @@ services:
|
||||
POSTGRES_DB: turnstone
|
||||
POSTGRES_USER: ${POSTGRES_USER:-turnstone}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required for production profile}
|
||||
PGDATA: /var/lib/postgresql/data
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
networks:
|
||||
@@ -44,7 +45,7 @@ services:
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 5s
|
||||
start_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
@@ -56,7 +57,7 @@ services:
|
||||
# Redis — message broker, pub/sub, node registry
|
||||
# -------------------------------------------------------------------
|
||||
redis:
|
||||
image: redis:7.4-alpine
|
||||
image: redis:8.6-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
@@ -257,7 +258,7 @@ services:
|
||||
# docker compose --profile ddgCluster up
|
||||
# -------------------------------------------------------------------
|
||||
ddg-search:
|
||||
image: python:3.13-slim
|
||||
image: python:3.14-slim
|
||||
profiles:
|
||||
- ddgCluster
|
||||
command:
|
||||
|
||||
@@ -7,10 +7,10 @@ appVersion: "0.3.0"
|
||||
|
||||
dependencies:
|
||||
- name: postgresql
|
||||
version: ~16.0
|
||||
version: ~18.5.0
|
||||
repository: https://charts.bitnami.com/bitnami
|
||||
condition: postgresql.enabled
|
||||
- name: redis
|
||||
version: ~20.0
|
||||
version: ~25.3.0
|
||||
repository: https://charts.bitnami.com/bitnami
|
||||
condition: redis.enabled
|
||||
|
||||
@@ -618,6 +618,67 @@ Each saved workstream object:
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/templates`
|
||||
|
||||
Returns a summary list of all available prompt templates. This is a read-only
|
||||
endpoint (requires `read` scope) that exposes template names and categories
|
||||
without revealing template content. Useful for populating template selectors
|
||||
in UIs or discovering available templates before creating a workstream.
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"templates": [
|
||||
{"name": "safety-guidelines", "category": "safety", "is_default": true, "origin": "manual"},
|
||||
{"name": "mcp__server__code", "category": "", "is_default": false, "origin": "mcp"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Each template summary:
|
||||
|
||||
| Field | Type | Description |
|
||||
|--------------|--------|------------------------------------------------------|
|
||||
| `name` | string | Template name (used in `template` field on creation) |
|
||||
| `category` | string | Template category |
|
||||
| `is_default` | bool | Whether template is auto-applied to all sessions |
|
||||
| `origin` | string | Template origin: `manual` or `mcp` |
|
||||
|
||||
> **Note:** For full template management (create, update, delete, view content),
|
||||
> use the admin endpoints at `GET /v1/api/admin/templates` (requires `admin.templates` permission).
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/ws-templates`
|
||||
|
||||
Returns a summary list of enabled workstream templates. This is a read-only
|
||||
endpoint (requires `read` scope) for populating template selectors in UIs.
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"ws_templates": [
|
||||
{"name": "code-review", "description": "Code review profile", "model": "gpt-5"},
|
||||
{"name": "ops-triage", "description": "On-call triage", "model": ""}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Each workstream template summary:
|
||||
|
||||
| Field | Type | Description |
|
||||
|---------------|--------|-------------------------------------------------|
|
||||
| `name` | string | Template name (used in `ws_template` on creation)|
|
||||
| `description` | string | Human-readable description |
|
||||
| `model` | string | Model alias override (empty = use default) |
|
||||
|
||||
> **Note:** For full workstream template management, use the admin endpoints at
|
||||
> `GET /v1/api/admin/ws-templates` (requires `admin.templates` permission).
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/send`
|
||||
|
||||
Sends a user message to a workstream. Spawns a daemon worker thread that calls
|
||||
@@ -940,6 +1001,270 @@ Status code: `403`
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/memories`
|
||||
|
||||
List structured memories with optional filters. Requires `read` scope.
|
||||
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Required | Default | Description |
|
||||
|------------|--------|----------|---------|------------------------------|
|
||||
| `type` | string | no | `""` | Filter by memory type (user, project, feedback, reference) |
|
||||
| `scope` | string | no | `""` | Filter by scope (global, workstream, user) |
|
||||
| `scope_id` | string | no | `""` | Scope qualifier. Auto-resolved for `scope=user` when auth is active. |
|
||||
| `limit` | int | no | `100` | Max results (capped at 200) |
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"memories": [
|
||||
{
|
||||
"memory_id": "a1b2c3d4-e5f6-...",
|
||||
"name": "project_architecture",
|
||||
"description": "Core architecture patterns",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"scope_id": "",
|
||||
"content": "The project uses a hexagonal architecture...",
|
||||
"created": "2026-03-10T10:00:00",
|
||||
"updated": "2026-03-12T14:30:00"
|
||||
}
|
||||
],
|
||||
"total": 1
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/memories`
|
||||
|
||||
Save or upsert a structured memory. Requires `write` scope. Returns `201` on
|
||||
create, `200` on update.
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "deployment_process",
|
||||
"content": "Deploy via GitHub Actions. Staging auto-deploys on push to main.",
|
||||
"description": "CI/CD deployment workflow",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"scope_id": ""
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Default | Description |
|
||||
|--------------|--------|----------|-------------|--------------------------------------|
|
||||
| `name` | string | yes | -- | Memory name (max 256 chars) |
|
||||
| `content` | string | yes | -- | Memory content (max 65536 chars) |
|
||||
| `description`| string | no | `""` | Short description for search ranking |
|
||||
| `type` | string | no | `"project"` | One of: user, project, feedback, reference |
|
||||
| `scope` | string | no | `"global"` | One of: global, workstream, user |
|
||||
| `scope_id` | string | no | `""` | Scope qualifier (auto-resolved for user scope) |
|
||||
|
||||
**Response (created):** `201`
|
||||
|
||||
```json
|
||||
{
|
||||
"memory_id": "a1b2c3d4-e5f6-...",
|
||||
"name": "deployment_process",
|
||||
"description": "CI/CD deployment workflow",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"scope_id": "",
|
||||
"content": "Deploy via GitHub Actions...",
|
||||
"created": "2026-03-14T10:00:00",
|
||||
"updated": "2026-03-14T10:00:00"
|
||||
}
|
||||
```
|
||||
|
||||
**Error responses:**
|
||||
|
||||
| Status | Condition |
|
||||
|--------|--------------------------------------------------------|
|
||||
| 400 | Missing name, empty content, invalid type/scope, name too long, content too long |
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/memories/search`
|
||||
|
||||
Search memories by query. Uses POST for the request body but is non-mutating
|
||||
(requires only `read` scope).
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{
|
||||
"query": "authentication",
|
||||
"type": "project",
|
||||
"scope": "",
|
||||
"limit": 20
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Default | Description |
|
||||
|------------|--------|----------|---------|--------------------------------|
|
||||
| `query` | string | yes | -- | Search query |
|
||||
| `type` | string | no | `""` | Filter by type |
|
||||
| `scope` | string | no | `""` | Filter by scope |
|
||||
| `scope_id` | string | no | `""` | Filter by scope ID |
|
||||
| `limit` | int | no | `20` | Max results (capped at 50) |
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"memories": [
|
||||
{
|
||||
"memory_id": "a1b2c3d4-e5f6-...",
|
||||
"name": "auth_patterns",
|
||||
"description": "Authentication architecture",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"scope_id": "",
|
||||
"content": "JWT tokens with HS256...",
|
||||
"created": "2026-03-10T10:00:00",
|
||||
"updated": "2026-03-12T14:30:00"
|
||||
}
|
||||
],
|
||||
"total": 1
|
||||
}
|
||||
```
|
||||
|
||||
**Error:** `400` with `{"error": "query is required"}` if `query` is empty.
|
||||
|
||||
---
|
||||
|
||||
### `DELETE /v1/api/memories/{name}`
|
||||
|
||||
Delete a memory by name and scope. Requires `write` scope.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|--------|----------------------|
|
||||
| `name` | string | Memory name |
|
||||
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Required | Default | Description |
|
||||
|------------|--------|----------|------------|---------------------|
|
||||
| `scope` | string | no | `"global"` | Scope of the memory |
|
||||
| `scope_id` | string | no | `""` | Scope qualifier |
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{"status": "ok", "name": "deployment_process"}
|
||||
```
|
||||
|
||||
**Error (not found):** `404`
|
||||
|
||||
```json
|
||||
{"error": "Memory 'deployment_process' not found"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/admin/memories` (Console)
|
||||
|
||||
List structured memories across all scopes. Requires `admin.memories`
|
||||
permission.
|
||||
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Required | Default | Description |
|
||||
|------------|--------|----------|---------|------------------------------|
|
||||
| `type` | string | no | `""` | Filter by type |
|
||||
| `scope` | string | no | `""` | Filter by scope |
|
||||
| `scope_id` | string | no | `""` | Filter by scope ID |
|
||||
| `limit` | int | no | `100` | Max results (capped at 200) |
|
||||
|
||||
**Response:** `200` -- same schema as `GET /v1/api/memories`.
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/admin/memories/search` (Console)
|
||||
|
||||
Search memories by query. Requires `admin.memories` permission.
|
||||
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Required | Default | Description |
|
||||
|------------|--------|----------|---------|-------------------------------|
|
||||
| `q` | string | yes | -- | Search query |
|
||||
| `type` | string | no | `""` | Filter by type |
|
||||
| `scope` | string | no | `""` | Filter by scope |
|
||||
| `scope_id` | string | no | `""` | Filter by scope ID |
|
||||
| `limit` | int | no | `20` | Max results (capped at 50) |
|
||||
|
||||
**Response:** `200` -- same schema as `GET /v1/api/memories`.
|
||||
|
||||
**Error:** `400` with `{"error": "q is required"}` if `q` is empty.
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/admin/memories/{memory_id}` (Console)
|
||||
|
||||
Get a single memory by ID. Requires `admin.memories` permission.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-------------|--------|------------------------|
|
||||
| `memory_id` | string | Memory UUID |
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"memory_id": "a1b2c3d4-e5f6-...",
|
||||
"name": "project_architecture",
|
||||
"description": "Core architecture patterns",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"scope_id": "",
|
||||
"content": "The project uses...",
|
||||
"created": "2026-03-10T10:00:00",
|
||||
"updated": "2026-03-12T14:30:00"
|
||||
}
|
||||
```
|
||||
|
||||
**Error (not found):** `404`
|
||||
|
||||
```json
|
||||
{"error": "Memory not found"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `DELETE /v1/api/admin/memories/{memory_id}` (Console)
|
||||
|
||||
Delete a memory by ID. Records an audit event (`memory.delete`). Requires
|
||||
`admin.memories` permission.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-------------|--------|------------------------|
|
||||
| `memory_id` | string | Memory UUID |
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{"status": "ok"}
|
||||
```
|
||||
|
||||
**Error (not found):** `404`
|
||||
|
||||
```json
|
||||
{"error": "Memory not found"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/admin/verdicts` (Console)
|
||||
|
||||
List intent validation verdicts from the `intent_verdicts` table. This endpoint
|
||||
@@ -986,6 +1311,165 @@ is on the **console** server and requires the `admin.judge` permission.
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/admin/settings` (Console)
|
||||
|
||||
List all settings with their effective values, defaults, and metadata. Requires
|
||||
the `admin.settings` permission.
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"settings": [
|
||||
{
|
||||
"key": "model.temperature",
|
||||
"value": 0.7,
|
||||
"source": "storage",
|
||||
"type": "float",
|
||||
"description": "Sampling temperature",
|
||||
"section": "model",
|
||||
"is_secret": false,
|
||||
"node_id": "",
|
||||
"changed_by": "admin",
|
||||
"updated": "2026-03-14T10:00:00",
|
||||
"restart_required": false
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/admin/settings/schema` (Console)
|
||||
|
||||
Return the full registry catalog (all defined settings with metadata). Requires
|
||||
the `admin.settings` permission. Useful for building dynamic admin UIs.
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"schema": [
|
||||
{
|
||||
"key": "model.temperature",
|
||||
"type": "float",
|
||||
"default": 0.5,
|
||||
"description": "Sampling temperature",
|
||||
"section": "model",
|
||||
"is_secret": false,
|
||||
"min_value": 0.0,
|
||||
"max_value": 2.0,
|
||||
"choices": null,
|
||||
"restart_required": false
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `PUT /v1/api/admin/settings/{key}` (Console)
|
||||
|
||||
Update a setting. Requires the `admin.settings` permission. The value is
|
||||
validated against the registry definition (type coercion, range checks, choices).
|
||||
Secret settings (`is_secret=true`) return `403`.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|--------|-------------|
|
||||
| `key` | string | Dotted setting key (e.g. `model.temperature`) |
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{
|
||||
"value": 0.7,
|
||||
"node_id": ""
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Default | Description |
|
||||
|-----------|--------|----------|---------|-------------|
|
||||
| `value` | any | yes | -- | New value (type-coerced against registry) |
|
||||
| `node_id` | string | no | `""` | Node ID for per-node override |
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"key": "model.temperature",
|
||||
"value": 0.7,
|
||||
"source": "storage",
|
||||
"type": "float",
|
||||
"description": "Sampling temperature",
|
||||
"section": "model",
|
||||
"is_secret": false,
|
||||
"node_id": "",
|
||||
"changed_by": "admin",
|
||||
"updated": "",
|
||||
"restart_required": false
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:**
|
||||
|
||||
| Status | Condition |
|
||||
|--------|-----------|
|
||||
| 400 | Unknown key, invalid value, type mismatch, out of range, missing `value` field |
|
||||
| 403 | Secret setting (must use config.toml or env) |
|
||||
|
||||
---
|
||||
|
||||
### `DELETE /v1/api/admin/settings/{key}` (Console)
|
||||
|
||||
Reset a setting to its registry default by removing it from storage. Requires
|
||||
the `admin.settings` permission.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|--------|-------------|
|
||||
| `key` | string | Dotted setting key |
|
||||
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Required | Default | Description |
|
||||
|-----------|--------|----------|---------|-------------|
|
||||
| `node_id` | string | no | `""` | Node ID (empty = global) |
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{"status": "ok", "key": "model.temperature", "default": 0.5}
|
||||
```
|
||||
|
||||
**Response (not found):** `404`
|
||||
|
||||
```json
|
||||
{"error": "Setting 'model.temperature' has no stored value"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### MCP Servers
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| GET | `/v1/api/admin/mcp-servers` | List all MCP server definitions with live node status. Query: `?reveal=true` to show env/header secrets. |
|
||||
| POST | `/v1/api/admin/mcp-servers` | Create an MCP server definition. Body: `{name, transport, command?, args?, url?, headers?, env?, auto_approve?, enabled?}` |
|
||||
| GET | `/v1/api/admin/mcp-servers/{server_id}` | Get a single MCP server with per-node connection status. |
|
||||
| PUT | `/v1/api/admin/mcp-servers/{server_id}` | Update an MCP server definition. Partial updates supported. |
|
||||
| DELETE | `/v1/api/admin/mcp-servers/{server_id}` | Delete an MCP server definition. |
|
||||
| POST | `/v1/api/admin/mcp-servers/reload` | Tell all cluster nodes to re-read the `mcp_servers` DB table and reconcile (add new, remove stale, reconnect changed). |
|
||||
| POST | `/v1/api/admin/mcp-servers/import` | Import servers from a pasted JSON config. Body: `{config: {mcpServers: {...}}}`. Skips existing names. |
|
||||
|
||||
Permission: `admin.mcp`
|
||||
|
||||
Secrets (`env`, `headers` fields) are masked with `***` by default. Use `?reveal=true` on GET endpoints to see actual values.
|
||||
|
||||
---
|
||||
|
||||
### `OPTIONS` (any path)
|
||||
|
||||
Handles CORS preflight requests.
|
||||
|
||||
+53
-19
@@ -3,7 +3,7 @@
|
||||
Turnstone is an AI orchestration platform with tool use, parallel workstreams, and persistent
|
||||
memory. It connects to any OpenAI-compatible API (local vLLM, OpenAI, etc.) or
|
||||
Anthropic's native Messages API via pluggable provider adapters, and gives the
|
||||
model 18 built-in tools plus external tools via MCP (Model Context Protocol) for
|
||||
model 17 built-in tools plus external tools via MCP (Model Context Protocol) for
|
||||
reading, writing, searching, planning, and executing code.
|
||||
|
||||
The core design principle is a **UI-agnostic engine with pluggable frontends**.
|
||||
@@ -47,7 +47,10 @@ turnstone/
|
||||
watch.py WatchRunner daemon — periodic command polling, condition DSL, result dispatch
|
||||
judge.py Intent validation — heuristic rules + LLM judge, advisory verdicts
|
||||
model_registry.py ModelRegistry — named model configs, lazy client creation, fallback routing
|
||||
memory.py Persistence facade (delegates to storage backend)
|
||||
memory.py Persistence facade + structured memory API (delegates to storage backend)
|
||||
config.py Config file loader (config.toml), apply_config(), warn_migrated_settings()
|
||||
config_store.py ConfigStore — database-backed settings with in-memory cache, thread-safe get/set
|
||||
settings_registry.py SettingDef catalog (~40 settings), validation, type coercion, serialization
|
||||
storage/ Pluggable storage: StorageBackend protocol, SQLite + PostgreSQL
|
||||
metrics.py Prometheus-compatible metrics collector (MetricsCollector)
|
||||
healthcheck.py BackendHealthMonitor — periodic probe + circuit breaker
|
||||
@@ -88,14 +91,16 @@ turnstone/
|
||||
_config.py Base ChannelConfig dataclass
|
||||
discord/ Discord adapter (bot, cog, views, streaming, config)
|
||||
shared_static/ Shared design system (base.css, auth.js, theme.js, toast.js, utils.js, kb.js)
|
||||
katex-0.16.38/ Vendored KaTeX math rendering library (MIT, woff2 fonts)
|
||||
ui/
|
||||
colors.py ANSI color constants with NO_COLOR support
|
||||
markdown.py Streaming terminal markdown renderer (line-buffered)
|
||||
spinner.py Braille character spinner (daemon thread)
|
||||
static/
|
||||
index.html Single-page app shell (links to CSS and JS)
|
||||
style.css Page-specific UI styles (dashboard layout, approval blocks)
|
||||
app.js Page-specific client-side JavaScript (SSE, workstreams, markdown)
|
||||
style.css Page-specific UI styles (dashboard, markdown elements, approval blocks)
|
||||
renderer.js Markdown + LaTeX renderer (tables, nested lists, blockquotes, KaTeX math)
|
||||
app.js Page-specific client-side JavaScript (SSE, workstreams, tool approval)
|
||||
tools/
|
||||
*.json 15 tool schemas (OpenAI function-calling format + turnstone metadata)
|
||||
```
|
||||
@@ -436,13 +441,13 @@ from each schema and builds:
|
||||
- `PRIMARY_KEY_MAP` -- `{name: primary_key}` for JSON fallback recovery
|
||||
- `merge_mcp_tools(builtin, mcp_tools)` -- merges built-in + MCP tools at session init
|
||||
|
||||
### 14 Tools by Category
|
||||
### 13 Tools by Category
|
||||
|
||||
**Read-only (auto-approve)**:
|
||||
- `read_file` -- read file contents with optional offset/limit
|
||||
- `search` -- ripgrep-based codebase search
|
||||
- `man` -- read man pages
|
||||
- `recall` -- retrieve stored memories
|
||||
- `recall` -- search conversation history
|
||||
|
||||
**Write (requires approval)**:
|
||||
- `bash` -- execute shell commands (with safety checks via `turnstone.core.safety`)
|
||||
@@ -456,9 +461,8 @@ from each schema and builds:
|
||||
- `task` -- delegate to a sub-agent with full tool access (`TASK_AGENT_TOOLS`)
|
||||
- `plan` -- explore codebase and write a structured plan (`AGENT_TOOLS`)
|
||||
|
||||
**Memory (persistent key-value store)**:
|
||||
- `remember` -- save a fact
|
||||
- `forget` -- delete a fact
|
||||
**Memory (structured persistent store)**:
|
||||
- `memory` -- save, search, delete, or list memories (typed and scoped)
|
||||
|
||||
### Prepare / Execute Pattern
|
||||
|
||||
@@ -504,8 +508,18 @@ independently, then returns the final content as the tool result.
|
||||
and exposes their tools alongside built-in tools. The MCP SDK is fully async; turnstone
|
||||
bridges this with a background asyncio event loop in a daemon thread.
|
||||
|
||||
**Configuration sources:** MCP servers can be defined in config files (TOML/JSON)
|
||||
or in the database via the admin UI. Database-backed definitions are managed
|
||||
through the console admin panel's MCP Servers tab and stored in the
|
||||
`mcp_servers` table. On startup, `load_mcp_config(storage=)` uses
|
||||
first-match-wins priority: DB rows (if any enabled) take precedence over
|
||||
config files. The console can trigger a cluster-wide reload (`POST
|
||||
/_internal/mcp-reload`) that causes each node to call `reconcile_sync()`,
|
||||
which diffs the running MCP connections against the current DB state and
|
||||
adds, removes, or reconnects servers as needed.
|
||||
|
||||
**Lifecycle:**
|
||||
1. `create_mcp_client()` reads server configs from TOML or JSON
|
||||
1. `create_mcp_client()` reads server configs from TOML/JSON and database
|
||||
2. `MCPClientManager.start()` launches the background event loop thread
|
||||
3. `_connect_all()` connects to each server (stdio subprocess or HTTP), runs
|
||||
`initialize()` + `list_tools()`, converts schemas to OpenAI format, detects
|
||||
@@ -1016,8 +1030,8 @@ Three hierarchical scopes control endpoint access:
|
||||
- **Console** is the auth management hub — it hosts the admin endpoints for
|
||||
creating users, issuing API tokens, and managing channel mappings. User
|
||||
records and token hashes live in the shared storage backend. The console
|
||||
dashboard includes an **admin panel** (Users and Tokens tabs) for managing
|
||||
credentials through the browser.
|
||||
dashboard includes an **admin panel** (14 tabs) for managing
|
||||
credentials, governance, MCP servers, and runtime settings through the browser.
|
||||
- **Server** is a JWT validator only — it validates tokens on each request but
|
||||
never creates users or tokens. Both processes share the same `jwt_secret`
|
||||
(via `TURNSTONE_JWT_SECRET` env var or `[auth].jwt_secret` config).
|
||||
@@ -1188,8 +1202,13 @@ The bridge dispatches it to `POST /v1/api/cancel` on the server owning the works
|
||||
which sets the cooperative cancel flag and unblocks any pending approval/plan waits.
|
||||
|
||||
**Completion detection:** The bridge tracks which `correlation_id` maps to which
|
||||
`ws_id` for active sends. When the global SSE reports `ws_state → idle` for a tracked
|
||||
workstream, the bridge emits a synthetic `TurnCompleteEvent` with the correlation ID.
|
||||
`ws_id` for active sends. The server accumulates content tokens in the WebUI and
|
||||
piggybacks the full response text onto the `ws_state → idle` global SSE event.
|
||||
When the bridge receives this event, it emits a synthetic `TurnCompleteEvent`
|
||||
carrying the correlation ID and the server-provided `content`. This lets downstream
|
||||
consumers (e.g. the Discord bot) recover the full response when individual
|
||||
`ContentEvent`s were missed, and serves as the primary delivery path for
|
||||
bidirectional notification DM forwarding.
|
||||
|
||||
**Multi-node routing:** Each bridge retrieves its `node_id` from the server's
|
||||
`/health` endpoint on startup (with exponential backoff retry). The server
|
||||
@@ -1353,11 +1372,23 @@ directly over HTTP for lower latency: `_exec_notify()` queries the
|
||||
`services` database table for healthy channel gateways (heartbeat within
|
||||
120 seconds), authenticates with a service JWT (`aud: turnstone-channel`),
|
||||
and POSTs to `POST /v1/api/notify` on the first healthy gateway. The
|
||||
gateway validates the JWT, resolves the target (username lookup via
|
||||
payload includes the originating `ws_id` for reply routing. The gateway
|
||||
validates the JWT, resolves the target (username lookup via
|
||||
`channel_users` or direct `channel_type`+`channel_id`), and delegates to
|
||||
the appropriate `ChannelAdapter.send()`. Delivery retries up to 3 times
|
||||
with backoff, re-querying the service registry on each attempt. See
|
||||
[Notification Flow diagram](diagrams/png/17-notify-flow.png).
|
||||
`ChannelAdapter.send_notification()` which sends the message and tracks
|
||||
the outgoing message ID → `(ws_id, target_user_id)` mapping. Delivery
|
||||
retries up to 3 times with backoff, re-querying the service registry on
|
||||
each attempt. See [Notification Flow diagram](diagrams/png/17-notify-flow.png).
|
||||
|
||||
**Bidirectional replies:** When a user replies to a notification DM, the
|
||||
Discord bot looks up the originating `ws_id` from the tracked message ID,
|
||||
verifies the replying user matches the notification recipient, and routes
|
||||
the reply to the workstream via `router.send_message()`. The workstream's
|
||||
response is forwarded back to the DM via a temporary entry in
|
||||
`_notify_reply_channels`. On `TurnCompleteEvent`, the response message is
|
||||
itself tracked for further replies, enabling multi-turn DM conversations
|
||||
without requiring the user to open the web UI. Tracking entries are capped
|
||||
at 100 (FIFO eviction) and cleaned up on workstream close.
|
||||
|
||||
---
|
||||
|
||||
@@ -1391,7 +1422,10 @@ enforcement tracks consumption in `session.send()` with 80% warning and
|
||||
100% approval gate via the `__budget_override__` synthetic tool name.
|
||||
|
||||
The console admin panel adds 6 governance tabs (Roles, Policies, Templates,
|
||||
WS Templates, Usage, Audit) for a total of 11 tabs, all permission-gated.
|
||||
WS Templates, Usage, Audit), a Memories tab, a Settings tab (form-based
|
||||
editor for all ConfigStore settings), and an MCP Servers tab (database-backed
|
||||
server definitions with live connection status and cluster-wide reload) for a
|
||||
total of 14 tabs, all permission-gated.
|
||||
Both Python and TypeScript SDKs expose governance methods on the console
|
||||
client.
|
||||
|
||||
|
||||
+40
-4
@@ -30,8 +30,8 @@ Key components:
|
||||
|
||||
- **ChannelAdapter protocol** (`turnstone/channels/_protocol.py`) — generic
|
||||
interface for any messaging platform. Defines `start()`, `stop()`,
|
||||
`send()`, `edit_message()`, `send_approval_request()`,
|
||||
`send_plan_review()`, and `create_thread()`.
|
||||
`send()`, `send_notification()`, `edit_message()`,
|
||||
`send_approval_request()`, `send_plan_review()`, and `create_thread()`.
|
||||
- **ChannelRouter** (`turnstone/channels/_routing.py`) — maps
|
||||
channel/thread IDs to turnstone workstream IDs. Handles workstream
|
||||
creation via MQ, stale route detection, and user identity resolution.
|
||||
@@ -271,13 +271,43 @@ gateway directly over HTTP:
|
||||
2. `_exec_notify()` queries the `services` table for healthy channel
|
||||
gateways (heartbeat within the last 120 seconds)
|
||||
3. The server mints a service JWT (`aud: turnstone-channel`) via
|
||||
`ServiceTokenManager` and POSTs to the first healthy gateway
|
||||
`ServiceTokenManager` and POSTs to the first healthy gateway. The
|
||||
payload includes the originating `ws_id` for reply routing.
|
||||
4. The gateway validates the JWT, resolves the target, and calls
|
||||
`adapter.send()` on the appropriate platform adapter
|
||||
`adapter.send_notification()` which sends the message and tracks
|
||||
the outgoing message ID for reply routing
|
||||
5. On failure, the server tries the next gateway. If all fail, it
|
||||
retries up to 2 more times (delays: 1s, 3s), re-querying the
|
||||
service registry on each attempt
|
||||
|
||||
### Bidirectional Replies
|
||||
|
||||
Notifications support multi-turn DM conversations. When a user replies
|
||||
to a notification DM:
|
||||
|
||||
1. The bot looks up the originating `ws_id` from the tracked message ID
|
||||
(`_notify_ws_map`)
|
||||
2. Verifies the replying user matches the original notification
|
||||
recipient (defence in depth — Discord DMs are already private)
|
||||
3. Routes the reply to the workstream via `router.send_message()`
|
||||
4. Registers the DM channel for response forwarding
|
||||
(`_notify_reply_channels`)
|
||||
5. When the workstream responds (`TurnCompleteEvent`), the response is
|
||||
forwarded to the DM
|
||||
6. The response message is itself tracked, so the user can reply again
|
||||
for another turn
|
||||
|
||||
This enables scenarios like an oncall engineer responding to a CI/CD
|
||||
failure notification from their phone before opening a laptop.
|
||||
|
||||
**Limits:**
|
||||
|
||||
- Tracking map capped at 100 entries (FIFO eviction of oldest)
|
||||
- Entries cleaned up on workstream close/unsubscribe
|
||||
- Replying to an expired notification sends
|
||||
*"This notification is no longer active."*
|
||||
- DM reply content capped at 4096 characters
|
||||
|
||||
### Service Registry
|
||||
|
||||
The channel gateway registers itself in the `services` database table
|
||||
@@ -328,12 +358,18 @@ class ChannelAdapter(Protocol):
|
||||
async def start(self) -> None: ...
|
||||
async def stop(self) -> None: ...
|
||||
async def send(self, channel_id: str, content: str) -> str: ...
|
||||
async def send_notification(self, channel_id: str, content: str, ws_id: str) -> str: ...
|
||||
async def edit_message(self, channel_id: str, message_id: str, content: str) -> None: ...
|
||||
async def send_approval_request(self, channel_id: str, ws_id: str, correlation_id: str, items: list[dict]) -> None: ...
|
||||
async def send_plan_review(self, channel_id: str, ws_id: str, correlation_id: str, content: str) -> None: ...
|
||||
async def create_thread(self, parent_channel_id: str, name: str, message_id: str = "") -> str: ...
|
||||
```
|
||||
|
||||
`send_notification()` is like `send()` but associates the outgoing
|
||||
message with a `ws_id` so that user replies can be routed back to the
|
||||
originating workstream. Adapters must track the mapping from outgoing
|
||||
message ID to `(ws_id, target_user_id)` and handle DM replies.
|
||||
|
||||
To add a new platform:
|
||||
|
||||
1. Create `turnstone/channels/<platform>/` package
|
||||
|
||||
+3
-2
@@ -421,8 +421,9 @@ The browser maintains a local `clusterState` object that mirrors the cluster sna
|
||||
|
||||
Accessed via the "admin" button in the header (visible when authenticated
|
||||
with `approve` scope). Provides user, API token, channel link, and workstream
|
||||
template management with 11 tabs (see also [Governance](governance.md) for
|
||||
the Roles, Policies, Templates, WS Templates, Usage, and Audit tabs):
|
||||
template management with 13 tabs (see also [Governance](governance.md) for
|
||||
the Roles, Policies, Templates, WS Templates, Usage, and Audit tabs, and
|
||||
[Settings](settings.md) for the database-backed configuration editor):
|
||||
|
||||
**Users tab:**
|
||||
|
||||
|
||||
@@ -75,6 +75,7 @@ package "turnstone/ui/" <<Rectangle>> {
|
||||
component [colors.py\nANSI colors] as colors <<ui>>
|
||||
component [markdown.py\nMD rendering] as markdown <<ui>>
|
||||
component [spinner.py\nTerminal spinner] as spinner <<ui>>
|
||||
component [renderer.js\nBrowser MD + LaTeX] as renderer <<ui>>
|
||||
}
|
||||
|
||||
' API schemas
|
||||
|
||||
@@ -127,7 +127,7 @@ group loop [while tool_calls present]
|
||||
math → sandboxed subprocess
|
||||
web_fetch → httpx + LLM summarize
|
||||
web_search → provider-native or Tavily fallback
|
||||
remember/recall/forget → SQLite
|
||||
memory/recall → SQLite
|
||||
end note
|
||||
|
||||
note right of TP
|
||||
|
||||
@@ -24,7 +24,7 @@ partition "Phase 1: Prepare" #E8F5E9 {
|
||||
:Dispatch to _prepare_{func_name}();
|
||||
|
||||
note right
|
||||
**Dispatch table (18 tools):**
|
||||
**Dispatch table (17 tools):**
|
||||
┌───────────────┬──────────────────┐
|
||||
│ Tool │ Needs Approval? │
|
||||
├───────────────┼──────────────────┤
|
||||
@@ -40,9 +40,8 @@ partition "Phase 1: Prepare" #E8F5E9 {
|
||||
│ tool_search │ ✗ Auto-approve │
|
||||
│ task │ ✓ Yes │
|
||||
│ plan │ ✓ Yes │
|
||||
│ remember │ ✗ Auto-approve │
|
||||
│ memory │ ✗ Auto-approve │
|
||||
│ recall │ ✗ Auto-approve │
|
||||
│ forget │ ✗ Auto-approve │
|
||||
│ notify │ ✗ Auto-approve │
|
||||
│ read_resource │ ✓ Yes │
|
||||
│ use_prompt │ ✓ Yes │
|
||||
@@ -70,8 +69,8 @@ partition "Phase 2: Approve" #FFF3E0 {
|
||||
**TerminalUI**: Print headers/previews,
|
||||
prompt [y/n/a, optional message]
|
||||
If user chose "always":
|
||||
Set ui.auto_approve = True
|
||||
(auto-approve all future tools in this session)
|
||||
Add pending tool names to auto_approve_tools
|
||||
(auto-approve these tool types going forward)
|
||||
**WebUI**: Enqueue approve_request,
|
||||
block on _approval_event.wait()
|
||||
**NullUI**: Return (True, None)
|
||||
@@ -116,9 +115,8 @@ partition "Phase 3: Execute" #E3F2FD {
|
||||
├─ _exec_task: _run_agent(TASK_AGENT_TOOLS)
|
||||
├─ _exec_plan: _run_agent(AGENT_TOOLS, read-only)
|
||||
├─ _exec_notify: HTTP POST to channel gateway
|
||||
├─ _exec_remember: SQLite INSERT OR REPLACE
|
||||
├─ _exec_recall: SQLite FTS5/LIKE search
|
||||
├─ _exec_forget: SQLite DELETE
|
||||
├─ _exec_memory: structured memory save/search/delete/list
|
||||
├─ _exec_recall: conversation history FTS5 search
|
||||
├─ _exec_read_resource: MCPClientManager.read_resource_sync()
|
||||
├─ _exec_use_prompt: MCPClientManager.get_prompt_sync()
|
||||
└─ _exec_mcp_tool: MCPClientManager.call_tool_sync()
|
||||
|
||||
@@ -174,6 +174,7 @@ package "Outbound Events (Bridge → Client)" #E3F2FD {
|
||||
}
|
||||
class TurnCompleteEvent {
|
||||
type = "turn_complete"
|
||||
+ content: str
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -39,8 +39,8 @@ BridgeA -> Redis : PUBLISH turnstone:events:abc12345\nAckEvent(status:"ok")
|
||||
... SSE events flow: content, tool_output_chunk, tool_result, status, state_change ...
|
||||
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:abc12345\nContentEvent, ToolResultEvent, ...
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:global\nStateChangeEvent(state:"idle")
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:abc12345\nTurnCompleteEvent
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:global\nStateChangeEvent(state:"idle", content:"...")
|
||||
BridgeA -> Redis : PUBLISH turnstone:events:abc12345\nTurnCompleteEvent(content:"...")
|
||||
|
||||
== Scenario B: Directed Message to Specific Node ==
|
||||
|
||||
@@ -79,7 +79,7 @@ note right of BridgeA
|
||||
1. _ws_auto_approve[ws_id]? → auto
|
||||
2. All tools in safe set? → auto
|
||||
(read_file, search, man,
|
||||
remember, recall, forget)
|
||||
memory, recall)
|
||||
3. Otherwise → manual approval
|
||||
end note
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ class "DiscordBot" as Bot <<service>> {
|
||||
+on_message(msg)
|
||||
+on_interaction(interaction)
|
||||
+send(channel_id, content)
|
||||
+send_notification(channel_id, content, ws_id)
|
||||
+run(token)
|
||||
--
|
||||
discord.py Client
|
||||
@@ -61,6 +62,9 @@ class "DiscordBot" as Bot <<service>> {
|
||||
Creates threads for workstreams
|
||||
Renders approval buttons
|
||||
escape_mentions() on send
|
||||
--
|
||||
_notify_ws_map: msg_id → (ws_id, user_id)
|
||||
_notify_reply_channels: ws_id → (dm, user_id)
|
||||
}
|
||||
|
||||
class "ChannelRouter" as Router <<service>> {
|
||||
@@ -240,11 +244,20 @@ note bottom of SVC
|
||||
3. Queries services table for healthy gateways
|
||||
4. Mints JWT (aud: turnstone-channel) via
|
||||
ServiceTokenManager
|
||||
5. POSTs to first healthy gateway
|
||||
5. POSTs to first healthy gateway (incl. ws_id)
|
||||
6. Gateway validates JWT, resolves target
|
||||
7. adapter.send() → Discord API
|
||||
7. adapter.send_notification() → Discord API
|
||||
(tracks msg_id → ws_id for reply routing)
|
||||
8. On failure: retry up to 3× (1s, 3s backoff)
|
||||
9. SSRF: only http(s) URLs allowed
|
||||
|
||||
**Bidirectional DM Replies**
|
||||
1. User replies to notification DM
|
||||
2. Bot looks up ws_id from _notify_ws_map
|
||||
3. Verifies author == notification recipient
|
||||
4. Routes reply via router.send_message()
|
||||
5. Response forwarded to DM on TurnCompleteEvent
|
||||
6. Response tracked for multi-turn conversation
|
||||
end note
|
||||
|
||||
@enduml
|
||||
|
||||
@@ -103,6 +103,41 @@ alt all retries exhausted
|
||||
Session --> Session : "Error: notification delivery failed"
|
||||
end
|
||||
|
||||
== Bidirectional Reply (User responds to notification DM) ==
|
||||
|
||||
Discord -> Adapter : user replies to\nnotification message
|
||||
Adapter -> Adapter : lookup message_id\nin _notify_ws_map
|
||||
note right
|
||||
Maps message_id →
|
||||
(ws_id, target_user_id)
|
||||
Atomic pop prevents TOCTOU
|
||||
end note
|
||||
|
||||
alt message not tracked
|
||||
Adapter -> Discord : "This notification\nis no longer active."
|
||||
else tracked
|
||||
Adapter -> Adapter : verify author ==\ntarget_user_id
|
||||
Adapter -> Adapter : resolve_user()\n(unlinked → drop)
|
||||
Adapter -> Adapter : router.send_message(ws_id, content)
|
||||
note right
|
||||
Routes reply via MQ to
|
||||
the originating workstream.
|
||||
Registers DM channel in
|
||||
_notify_reply_channels[ws_id]
|
||||
end note
|
||||
|
||||
... workstream processes reply ...
|
||||
|
||||
Adapter <- Adapter : TurnCompleteEvent\n(with content)
|
||||
Adapter -> Discord : forward response to DM
|
||||
Adapter -> Adapter : track response message\nfor multi-turn replies
|
||||
note right
|
||||
Response message_id added
|
||||
to _notify_ws_map — user can
|
||||
reply again indefinitely
|
||||
end note
|
||||
end
|
||||
|
||||
== Service Registry (Background) ==
|
||||
|
||||
note over Gateway, Storage
|
||||
|
||||
@@ -85,6 +85,12 @@ tload --> trender : template content
|
||||
trender --> tsys : rendered content
|
||||
tset --> tload : name or None
|
||||
|
||||
note right of pt_db
|
||||
Read-only listing:
|
||||
GET /v1/api/templates
|
||||
(read scope, summary only)
|
||||
end note
|
||||
|
||||
govjs --> wt_db : /v1/api/admin/ws-templates
|
||||
wtr --> wt_db : get_ws_template_by_name()
|
||||
wtr --> wta : template settings
|
||||
|
||||
@@ -16,8 +16,27 @@ participant "ChatSession\n(session.py)" as Session <<session>>
|
||||
participant "StorageBackend\n(governance)" as Storage <<storage>>
|
||||
participant "Server / Console\n(health + UI)" as UI <<server>>
|
||||
|
||||
participant "Console Admin UI\n(admin panel)" as Admin <<ui>>
|
||||
participant "Database\n(mcp_servers table)" as DB <<storage>>
|
||||
|
||||
== Admin-Driven Configuration ==
|
||||
|
||||
Admin -> DB : CRUD MCP server definitions\n(POST/PUT/DELETE /v1/api/admin/mcp-servers)
|
||||
|
||||
Admin -> UI : POST /v1/api/admin/mcp-servers/reload
|
||||
UI -> MCPMgr : POST /_internal/mcp-reload\n(forwarded to each node)
|
||||
MCPMgr -> MCPMgr : reconcile_sync()
|
||||
note right
|
||||
Diffs running servers against DB:
|
||||
- New entries → connect
|
||||
- Removed entries → disconnect
|
||||
- Changed entries → reconnect
|
||||
end note
|
||||
|
||||
== Startup: Connection & Discovery ==
|
||||
|
||||
MCPMgr -> DB : load_mcp_config(storage=)\n(merge config file + DB)
|
||||
|
||||
MCPMgr -> MCPSrv : initialize (stdio or HTTP)
|
||||
MCPSrv --> MCPMgr : capabilities\n(tools, resources, prompts)
|
||||
|
||||
|
||||
@@ -48,6 +48,14 @@ end note
|
||||
Admin -> Server : GET /v1/api/admin/ws-templates
|
||||
Server -> Storage : list_ws_templates()
|
||||
|
||||
Server <-- Server : GET /v1/api/ws-templates\n(read scope, summary only)
|
||||
note right
|
||||
**Read-only listing:**
|
||||
name, description, model.
|
||||
Used by creation UI dropdowns.
|
||||
Available on both server + console.
|
||||
end note
|
||||
|
||||
Admin -> Server : DELETE /v1/api/admin/ws-templates/{id}
|
||||
Server -> Storage : delete_ws_template(id)
|
||||
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Structured Memory Architecture
|
||||
|
||||
skinparam participant {
|
||||
BackgroundColor<<session>> #C8E6C9
|
||||
BackgroundColor<<facade>> #FFE0B2
|
||||
BackgroundColor<<storage>> #B3E5FC
|
||||
BackgroundColor<<api>> #E8EAF6
|
||||
BackgroundColor<<sdk>> #F5F5F5
|
||||
}
|
||||
|
||||
participant "ChatSession\n(session.py)" as Session <<session>>
|
||||
participant "MemoryFacade\n(memory.py)" as Facade <<facade>>
|
||||
participant "MemoryRelevance\n(memory_relevance.py)" as Relevance <<facade>>
|
||||
participant "StorageBackend\n(SQLite)" as Storage <<storage>>
|
||||
participant "Server API\n(server.py)" as API <<api>>
|
||||
participant "Console Admin\n(console/server.py)" as Admin <<api>>
|
||||
participant "SDK Client\n(sdk/)" as SDK <<sdk>>
|
||||
|
||||
== Phase 1: Tool Path (session.send) ==
|
||||
|
||||
Session -> Session : _prepare_tool_calls()\nparse memory(action=...)
|
||||
note right
|
||||
Tool schema: 4 actions
|
||||
save, search, delete, list
|
||||
Auto-approved (no approval needed)
|
||||
end note
|
||||
|
||||
Session -> Session : _exec_memory(item)
|
||||
|
||||
alt action = save
|
||||
Session -> Facade : save_structured_memory(\nname, content, description,\nmem_type, scope, scope_id)
|
||||
Facade -> Facade : normalize_key(name)
|
||||
Facade -> Storage : create_structured_memory()
|
||||
alt unique constraint violation
|
||||
Storage --> Facade : IntegrityError
|
||||
Facade -> Storage : get_structured_memory_by_name()
|
||||
Storage --> Facade : existing row
|
||||
Facade -> Storage : update_structured_memory()
|
||||
end
|
||||
Storage --> Facade : memory_id
|
||||
Facade --> Session : (memory_id, old_content)
|
||||
Session -> Session : _init_system_messages()\nrefresh BM25 context
|
||||
end
|
||||
|
||||
alt action = search
|
||||
Session -> Facade : search_structured_memories(\nquery, mem_type, scope,\nscope_id, limit)
|
||||
Facade -> Storage : search_structured_memories()
|
||||
Storage --> Session : matched rows
|
||||
end
|
||||
|
||||
alt action = delete
|
||||
Session -> Facade : delete_structured_memory(\nname, scope, scope_id)
|
||||
Facade -> Storage : delete_structured_memory()
|
||||
Storage --> Session : bool (existed)
|
||||
Session -> Session : _init_system_messages()\nrefresh BM25 context
|
||||
end
|
||||
|
||||
== Phase 2: BM25 Relevance Injection ==
|
||||
|
||||
Session -> Session : _init_system_messages()\nevery conversation turn
|
||||
|
||||
Session -> Session : _get_visible_memories(\nlimit=fetch_limit)
|
||||
note right
|
||||
**Scope resolution:**
|
||||
1. global scope (always)
|
||||
2. workstream scope (ws_id)
|
||||
3. user scope (user_id, if auth)
|
||||
Combined and deduplicated.
|
||||
end note
|
||||
|
||||
Session -> Facade : list_structured_memories()\nper scope
|
||||
Facade -> Storage : list_structured_memories()
|
||||
Storage --> Session : up to fetch_limit rows
|
||||
|
||||
Session -> Relevance : extract_recent_context(\nmessages, max_messages=3)
|
||||
Relevance --> Session : user text context
|
||||
|
||||
Session -> Relevance : score_memories(\nmemories, context,\nk=relevance_k)
|
||||
note right
|
||||
**BM25 scoring:**
|
||||
Index over name + description
|
||||
+ content[:200] for each memory.
|
||||
Returns top-k by relevance.
|
||||
Empty query returns most recent k.
|
||||
end note
|
||||
Relevance --> Session : top-k memories
|
||||
|
||||
Session -> Relevance : build_memory_context(\nrelevant_memories)
|
||||
note right
|
||||
Formats as XML block:
|
||||
<memories>
|
||||
<memory name="..." type="..."
|
||||
scope="..." description="...">
|
||||
content (max 500 chars)
|
||||
</memory>
|
||||
</memories>
|
||||
end note
|
||||
Relevance --> Session : XML string
|
||||
|
||||
Session -> Session : inject into\nsystem message
|
||||
|
||||
== Phase 3: Server API Path ==
|
||||
|
||||
SDK -> API : GET /v1/api/memories\n?type=project&limit=20
|
||||
API -> Facade : list_structured_memories()
|
||||
Facade -> Storage : list_structured_memories()
|
||||
Storage --> API : rows
|
||||
API --> SDK : {"memories": [...], "total": N}
|
||||
|
||||
SDK -> API : POST /v1/api/memories\n{name, content, ...}
|
||||
API -> API : validate type, scope,\nname length, content length
|
||||
API -> Facade : save_structured_memory()
|
||||
Facade -> Storage : create / update
|
||||
Storage --> API : memory row
|
||||
API --> SDK : 201 (created) / 200 (updated)
|
||||
|
||||
SDK -> API : POST /v1/api/memories/search\n{query, type, ...}
|
||||
API -> Facade : search_structured_memories()
|
||||
Facade -> Storage : search_structured_memories()
|
||||
Storage --> API : matched rows
|
||||
API --> SDK : {"memories": [...], "total": N}
|
||||
|
||||
SDK -> API : DELETE /v1/api/memories/{name}\n?scope=global
|
||||
API -> Facade : delete_structured_memory()
|
||||
Facade -> Storage : delete row
|
||||
API --> SDK : {"status": "ok"}
|
||||
|
||||
== Phase 4: Console Admin Path ==
|
||||
|
||||
SDK -> Admin : GET /v1/api/admin/memories\n?type=&scope=&limit=
|
||||
Admin -> Admin : require_permission(\n"admin.memories")
|
||||
Admin -> Storage : list_structured_memories()
|
||||
Storage --> Admin : rows
|
||||
Admin --> SDK : {"memories": [...], "total": N}
|
||||
|
||||
SDK -> Admin : GET /v1/api/admin/memories/{id}
|
||||
Admin -> Storage : get_structured_memory(id)
|
||||
Storage --> Admin : memory row
|
||||
Admin --> SDK : memory JSON
|
||||
|
||||
SDK -> Admin : DELETE /v1/api/admin/memories/{id}
|
||||
Admin -> Storage : delete_structured_memory_by_id()
|
||||
Admin -> Admin : record_audit(\n"memory.delete")
|
||||
Admin --> SDK : {"status": "ok"}
|
||||
|
||||
== Configuration ==
|
||||
|
||||
note over Session, Relevance
|
||||
**MemoryConfig** (from [memory] in config.toml):
|
||||
relevance_k = 5 -- top-k memories per turn
|
||||
fetch_limit = 50 -- max memories fetched for scoring
|
||||
max_content = 32768 -- max content length per memory
|
||||
nudge_cooldown = 300 -- seconds between metacognitive nudges
|
||||
nudges = true -- enable/disable memory nudges
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -0,0 +1,151 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — Settings Architecture
|
||||
|
||||
skinparam participant {
|
||||
BackgroundColor<<session>> #C8E6C9
|
||||
BackgroundColor<<config>> #FFE0B2
|
||||
BackgroundColor<<storage>> #B3E5FC
|
||||
BackgroundColor<<api>> #E8EAF6
|
||||
BackgroundColor<<sdk>> #F5F5F5
|
||||
}
|
||||
|
||||
participant "Server\n(main)" as Server <<session>>
|
||||
participant "ConfigStore\n(config_store.py)" as Store <<config>>
|
||||
participant "SettingsRegistry\n(settings_registry.py)" as Registry <<config>>
|
||||
participant "StorageBackend\n(SQLite)" as Storage <<storage>>
|
||||
participant "Console Admin\n(console/server.py)" as Admin <<api>>
|
||||
participant "SDK Client\n(sdk/)" as SDK <<sdk>>
|
||||
participant "ChatSession\n(session.py)" as Session <<session>>
|
||||
|
||||
== Phase 1: Server Startup ==
|
||||
|
||||
Server -> Server : parse_args()\nCLI flags override defaults
|
||||
Server -> Server : init_storage()\nSQLite / PostgreSQL
|
||||
|
||||
Server -> Store ** : ConfigStore(storage, node_id)
|
||||
Store -> Storage : get_system_settings_bulk(node_id)
|
||||
note right
|
||||
1. Load global settings (node_id="")
|
||||
2. Overlay per-node settings
|
||||
Returns {key: json_value} dict
|
||||
end note
|
||||
Storage --> Store : raw settings
|
||||
Store -> Registry : deserialize_value(key, json)\nper entry
|
||||
Registry --> Store : typed values
|
||||
Store -> Store : swap _cache atomically\nincrement _version
|
||||
|
||||
Server -> Server : warn_migrated_settings()
|
||||
note right
|
||||
Scans config.toml for keys
|
||||
now managed by ConfigStore.
|
||||
Logs warning for each overlap.
|
||||
end note
|
||||
|
||||
Server -> Server : session_factory captures\nConfigStore reference
|
||||
|
||||
== Phase 2: Settings Read (session creation) ==
|
||||
|
||||
Server -> Session : session_factory(ws_id)
|
||||
Session -> Store : get("model.temperature")
|
||||
Store -> Store : cache[key] lookup\n(lock-free)
|
||||
alt key in cache
|
||||
Store --> Session : stored value
|
||||
else key not in cache
|
||||
Store -> Registry : SETTINGS[key].default
|
||||
Registry --> Store : default value
|
||||
Store --> Session : default value
|
||||
end
|
||||
note right of Session
|
||||
Settings are captured once
|
||||
at workstream creation.
|
||||
Not re-read on every turn.
|
||||
end note
|
||||
|
||||
== Phase 3: Admin API — List / Schema ==
|
||||
|
||||
SDK -> Admin : GET /v1/api/admin/settings
|
||||
Admin -> Admin : require_permission(\n"admin.settings")
|
||||
Admin -> Store : all_effective()
|
||||
Store -> Store : merge cache with\nregistry defaults
|
||||
Store --> Admin : {key: effective_value}
|
||||
Admin -> Registry : SETTINGS (metadata)
|
||||
note right
|
||||
Annotates each setting with:
|
||||
type, default, description,
|
||||
is_stored, is_secret, constraints,
|
||||
changed_by, updated
|
||||
end note
|
||||
Admin --> SDK : {"settings": [...], "total": N}
|
||||
|
||||
SDK -> Admin : GET /v1/api/admin/settings/schema
|
||||
Admin -> Admin : require_permission(\n"admin.settings")
|
||||
Admin -> Registry : SETTINGS catalog
|
||||
Admin --> SDK : {"settings": [...], "total": N}
|
||||
|
||||
== Phase 4: Admin API — Update ==
|
||||
|
||||
SDK -> Admin : PUT /v1/api/admin/settings/\nmodel.temperature\n{"value": 0.7}
|
||||
Admin -> Admin : require_permission(\n"admin.settings")
|
||||
Admin -> Registry : validate_key("model.temperature")
|
||||
Registry --> Admin : SettingDef
|
||||
alt is_secret == true
|
||||
Admin --> SDK : 403 Forbidden
|
||||
else
|
||||
Admin -> Registry : validate_value(key, 0.7)
|
||||
note right
|
||||
Type coercion: float(0.7)
|
||||
Range check: 0.0 <= 0.7 <= 2.0
|
||||
Choices check: (none for this key)
|
||||
end note
|
||||
Registry --> Admin : typed value
|
||||
Admin -> Store : set(key, 0.7, changed_by="admin")
|
||||
Store -> Registry : serialize_value(0.7)\n=> "0.7"
|
||||
Store -> Storage : upsert_system_setting(\nkey, "0.7", node_id, ...)
|
||||
Storage --> Store : ok
|
||||
Store -> Store : swap _cache atomically
|
||||
Admin -> Admin : record_audit(\n"setting.update")
|
||||
Admin --> SDK : {"key": "...", "value": 0.7,\n"previous": 0.5}
|
||||
end
|
||||
|
||||
== Phase 5: Admin API — Delete (reset to default) ==
|
||||
|
||||
SDK -> Admin : DELETE /v1/api/admin/settings/\nmodel.temperature
|
||||
Admin -> Admin : require_permission(\n"admin.settings")
|
||||
Admin -> Store : delete("model.temperature")
|
||||
Store -> Registry : validate_key(key)
|
||||
Store -> Storage : delete_system_setting(key, node_id)
|
||||
Storage --> Store : bool (existed)
|
||||
Store -> Store : remove from cache,\nswap atomically
|
||||
Admin -> Admin : record_audit(\n"setting.delete")
|
||||
Admin --> SDK : {"status": "ok",\n"key": "...", "default": 0.5}
|
||||
|
||||
== Phase 6: Hot Reload ==
|
||||
|
||||
SDK -> Admin : POST /v1/api/_internal/\nconfig-reload
|
||||
Admin -> Store : reload()
|
||||
Store -> Storage : get_system_settings_bulk(node_id)
|
||||
Storage --> Store : all settings
|
||||
Store -> Store : rebuild cache,\nswap atomically,\nincrement _version
|
||||
note right
|
||||
Existing sessions: unchanged
|
||||
(frozen at creation time).
|
||||
New sessions: pick up
|
||||
updated values immediately.
|
||||
end note
|
||||
Admin --> SDK : {"status": "ok"}
|
||||
|
||||
== Precedence Summary ==
|
||||
|
||||
note over Server, Registry
|
||||
**Server entry point:**
|
||||
CLI flag > ConfigStore (database) > registry default
|
||||
|
||||
**CLI entry point:**
|
||||
CLI flag > config.toml > argparse default
|
||||
|
||||
**Bootstrap settings** (database, Redis, auth, server bind):
|
||||
Always from config.toml / env vars — never in ConfigStore.
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -0,0 +1,147 @@
|
||||
@startuml
|
||||
!theme plain
|
||||
title Turnstone — OIDC Authorization Code Flow with PKCE
|
||||
|
||||
skinparam participant {
|
||||
BackgroundColor<<browser>> #E8EAF6
|
||||
BackgroundColor<<server>> #FFE0B2
|
||||
BackgroundColor<<storage>> #B3E5FC
|
||||
BackgroundColor<<idp>> #C8E6C9
|
||||
}
|
||||
|
||||
participant "Browser" as Browser <<browser>>
|
||||
participant "Turnstone\n(Server / Console)" as Server <<server>>
|
||||
database "SQLite /\nPostgreSQL" as DB <<storage>>
|
||||
participant "Identity Provider\n(IdP)" as IdP <<idp>>
|
||||
|
||||
== Page Load ==
|
||||
|
||||
Browser -> Server : GET /v1/api/auth/status
|
||||
Server --> Browser : {oidc_enabled: true,\noidc_provider_name: "...",\npassword_enabled: true}
|
||||
|
||||
note right of Browser
|
||||
Login screen renders
|
||||
"Continue with {provider_name}"
|
||||
button alongside password form.
|
||||
If password_enabled=false,
|
||||
only the SSO button is shown.
|
||||
end note
|
||||
|
||||
== Authorization Request ==
|
||||
|
||||
Browser -> Server : GET /v1/api/auth/oidc/authorize
|
||||
|
||||
Server -> Server : Generate state (random)\nnonce (random)\nPKCE code_verifier + code_challenge
|
||||
|
||||
Server -> DB : create_oidc_pending_state(\nstate, nonce, code_verifier, audience)
|
||||
note right of DB
|
||||
Stored with created_at timestamp.
|
||||
Expires after 5 minutes.
|
||||
end note
|
||||
|
||||
Server --> Browser : 302 Redirect to IdP\nauthorization_endpoint
|
||||
|
||||
Browser -> IdP : GET /authorize?\nresponse_type=code&\nclient_id=...&\nredirect_uri=...&\nscope=openid email profile&\nstate=...&nonce=...&\ncode_challenge=...&\ncode_challenge_method=S256
|
||||
|
||||
== User Authentication (at IdP) ==
|
||||
|
||||
IdP -> Browser : Login page (if no\nexisting IdP session)
|
||||
Browser -> IdP : User authenticates\n(username/password, MFA, etc.)
|
||||
|
||||
IdP --> Browser : 302 Redirect to callback\n?code=AUTH_CODE&state=STATE
|
||||
|
||||
== Callback Processing ==
|
||||
|
||||
Browser -> Server : GET /v1/api/auth/oidc/callback\n?code=AUTH_CODE&state=STATE
|
||||
|
||||
Server -> Server : Rate limit check\n(5 per 5min per IP)
|
||||
|
||||
Server -> DB : cleanup_expired_oidc_states(300)
|
||||
note right of DB
|
||||
Lazy cleanup of states
|
||||
older than 5 minutes.
|
||||
end note
|
||||
|
||||
Server -> DB : pop_oidc_pending_state(state)
|
||||
DB --> Server : {nonce, code_verifier, audience}
|
||||
note right of Server
|
||||
Atomic fetch-and-delete.
|
||||
Returns None if state is
|
||||
expired or unknown.
|
||||
end note
|
||||
|
||||
== Token Exchange ==
|
||||
|
||||
Server -> IdP : POST /token\ngrant_type=authorization_code&\ncode=AUTH_CODE&\nclient_id=...&\nclient_secret=...&\ncode_verifier=...&\nredirect_uri=...
|
||||
note right of Server
|
||||
Client secret + PKCE verifier
|
||||
sent server-side only.
|
||||
Never exposed to browser.
|
||||
end note
|
||||
|
||||
IdP --> Server : {id_token: "eyJ...",\naccess_token: "..."}
|
||||
|
||||
== ID Token Validation ==
|
||||
|
||||
Server -> IdP : Fetch JWKS public keys\n(cached at startup, refreshed\non-demand when unknown kid\nencountered — key rotation)
|
||||
|
||||
Server -> Server : Validate ID token:\n1. Verify signature (RS256/ES256)\n2. Check iss == configured issuer\n3. Check aud == client_id\n4. Check exp (not expired)\n5. Verify nonce matches
|
||||
|
||||
== User Provisioning ==
|
||||
|
||||
Server -> DB : get_oidc_identity(issuer, sub)
|
||||
|
||||
alt Existing identity found
|
||||
DB --> Server : {user_id, ...}
|
||||
Server -> DB : update_oidc_identity_login()\nupdate last_login timestamp
|
||||
Server -> DB : get_user(user_id)
|
||||
DB --> Server : user record
|
||||
else New user (first login)
|
||||
Server -> Server : Derive username from\npreferred_username / email
|
||||
Server -> DB : create_user(user_id, username,\ndisplay_name, "!oidc")
|
||||
note right of DB
|
||||
Password hash set to sentinel
|
||||
value "!oidc" — not a valid
|
||||
bcrypt hash, so password login
|
||||
is always rejected.
|
||||
end note
|
||||
Server -> DB : create_oidc_identity(\nissuer, sub, user_id, email)
|
||||
end
|
||||
|
||||
opt Role mapping configured
|
||||
Server -> Server : Read role_claim from ID token\nMap values via role_map
|
||||
Server -> DB : Sync roles: add new,\nrevoke stale OIDC-assigned,\npreserve manually assigned
|
||||
end
|
||||
|
||||
== Issue Turnstone JWT ==
|
||||
|
||||
Server -> Server : Load user permissions\nDerive scopes from permissions
|
||||
Server -> Server : Create JWT (HS256)\nsub: user_id\nscopes: read,write,...\nsrc: "oidc"\naud: turnstone-server\nexp: +24h
|
||||
|
||||
Server --> Browser : 302 Redirect to /?oidc_success=1\nSet-Cookie: session=JWT\n(HttpOnly, SameSite=Lax, Secure)
|
||||
|
||||
== Browser Success Detection ==
|
||||
|
||||
Browser -> Browser : Detect ?oidc_success=1\nStrip param from URL\n(history.replaceState)
|
||||
Browser -> Browser : Hide login overlay\nCall onLoginSuccess()
|
||||
|
||||
note right of Browser
|
||||
Browser is now authenticated.
|
||||
JWT cookie sent on all
|
||||
subsequent requests.
|
||||
end note
|
||||
|
||||
== Error Paths ==
|
||||
|
||||
note over Browser, IdP
|
||||
**Error handling:**
|
||||
- IdP returns error param → redirect to /?oidc_error=...
|
||||
- State missing/expired → redirect to /?oidc_error=Login+session+expired
|
||||
- Token exchange fails → redirect to /?oidc_error=...
|
||||
- ID token validation fails → redirect to /?oidc_error=...
|
||||
- No admin user exists → redirect to /?oidc_error=Initial+setup+required
|
||||
- Rate limit exceeded → redirect to /?oidc_error=Too+many+login+attempts
|
||||
All errors are shown as toast messages on the login screen.
|
||||
end note
|
||||
|
||||
@enduml
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c9daca81971ba7a8ed6736d23d5373c69435158fa6240b9880d14fc4759ab580
|
||||
size 329673
|
||||
oid sha256:efcc7cbe8161a54b5ec24bdfd47e8a142f70029e6e66c707e811b99369f85ebf
|
||||
size 310079
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:6dd3c923d1e1c49b5f91d8d342fb4b0d49a46d432460379ad146a9e3b075a05a
|
||||
size 277234
|
||||
oid sha256:43844b07d36beb04db871f6795a3f3be17852a6a484fdc0ea207403bd7f512a6
|
||||
size 274286
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:2229801220548e4794baa67e27a0a39dc7968c826a28fa8144763e678c8ed733
|
||||
size 192556
|
||||
oid sha256:6e94a10f039a7f69517e84d0946e0c649035c15b38ebc2314e7b9cd501eb244d
|
||||
size 192559
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1380065cbb5f95b5ea7dc6b2a00986c455b82888af60784980dffbd936460dcf
|
||||
size 431129
|
||||
oid sha256:6fc99bb8d84d6e9f3dac9d5c12ac7f569a041b29431c57612c24b50f332982ed
|
||||
size 462992
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f0f6097840fccdbfe16cd5e4c9f5d063b2c36942460a944df68b8ec947e63ea3
|
||||
size 221452
|
||||
oid sha256:cc4c511c34a2e5d286fd128c3509405a5b240ca02a4bafb395d2e94d002a5b8b
|
||||
size 293203
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f4dac4948d928b4705936d73b4d159aa1e89315ec0397616ca914bbf19e7a1ce
|
||||
size 206479
|
||||
oid sha256:3aaca1ae4c6c255dc9569f59e3ccc24f8b3bab0ac2a9b08c85e2af72d6a400c7
|
||||
size 218575
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:8e6dc5142c7908314ce01229b3c4f13bf9450adcbb62a178838bd4cf81d9f4da
|
||||
size 250417
|
||||
oid sha256:e4593873599342b2830fedd5d783e9a28eab0bb0d6589798ef6ef2649eeee80f
|
||||
size 324518
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c06d7086d7965eb9fe333396f027133d42507cf120bfe8dc851c009a8768ec48
|
||||
size 284926
|
||||
oid sha256:fadf5b07f8230ecf97805a86b308eaa9eb30516dd26900e5c9f70e6fb7562bab
|
||||
size 296339
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c89628ed917dfd576c1af75c68fe5fed9beadaaee9dcea7aa7a1643867c4f1b9
|
||||
size 344323
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:83c0e6aad3eb19f6bc475a30a77215e801da3da5930f0462417fe7eb6eda6be2
|
||||
size 347144
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1c21910e3916be789b0377c8a0dcc8f47d66a967861a543d5bdd0c26da185259
|
||||
size 309584
|
||||
+1
-1
@@ -99,7 +99,7 @@ Workstream templates are behavioral profiles applied at workstream creation —
|
||||
|
||||
**Admin API:** 7 endpoints under `/v1/api/admin/ws-templates` (list, create, get, update, delete, version history) plus a read-only summary at `/v1/api/ws-templates`. Permission: `admin.ws_templates`.
|
||||
|
||||
**Console UI:** "WS Templates" tab (11th admin tab) with CRUD table, create/edit modals (name, description, system prompt source toggle, model, auto-approve, per-tool auto-approve, temperature, reasoning effort, max tokens, agent max turns, token budget, enabled), and version history modal. "Profile" dropdown on workstream creation modal. "WS Template" dropdown on scheduler create/edit modals.
|
||||
**Console UI:** "WS Templates" tab with CRUD table, create/edit modals (name, description, system prompt source toggle, model, auto-approve, per-tool auto-approve, temperature, reasoning effort, max tokens, agent max turns, token budget, enabled), and version history modal. "Profile" dropdown on workstream creation modal. "WS Template" dropdown on scheduler create/edit modals.
|
||||
|
||||
**Token budget enforcement:** Tracked in `session.send()`. At 80% consumption, emits an info message. At 100%, the next turn requires explicit approval via the `__budget_override__` synthetic tool name (reuses existing approval UI — inline in browser, Discord buttons, bridge auto-approve). The synthetic name can be targeted by tool policies (e.g. `__budget_override__` → `allow` for admins).
|
||||
|
||||
|
||||
+569
@@ -0,0 +1,569 @@
|
||||
# Structured Memory
|
||||
|
||||
> See also: [Memory Architecture diagram](diagrams/png/23-memory-architecture.png)
|
||||
|
||||
The structured memory system gives the AI persistent, typed, scoped memories
|
||||
that survive across sessions and workstreams. Memories are automatically
|
||||
surfaced in the system message via BM25 relevance scoring, so the model has
|
||||
contextual recall without explicit search.
|
||||
|
||||
## Overview
|
||||
|
||||
Each memory has three dimensions:
|
||||
|
||||
- **Type** -- categorizes the memory's purpose
|
||||
- **Scope** -- controls visibility boundaries
|
||||
- **Name** -- unique identifier within a scope (snake_case, normalized)
|
||||
|
||||
### Memory types
|
||||
|
||||
| Type | Purpose |
|
||||
|-------------|------------------------------------------------------------|
|
||||
| `user` | User preferences, conventions, working style |
|
||||
| `project` | Project-specific knowledge, architecture, patterns |
|
||||
| `feedback` | Corrections, lessons learned, things to avoid |
|
||||
| `reference` | Reference material, documentation, specifications |
|
||||
|
||||
### Memory scopes
|
||||
|
||||
| Scope | Visibility |
|
||||
|--------------|-----------------------------------------------------------|
|
||||
| `global` | Visible to all workstreams and users |
|
||||
| `workstream` | Visible only within the originating workstream |
|
||||
| `user` | Follows the authenticated user across workstreams |
|
||||
|
||||
A memory's identity is the tuple `(name, scope, scope_id)`. Saving a memory
|
||||
with the same identity upserts -- updating content while preserving the ID.
|
||||
|
||||
### BM25 relevance injection
|
||||
|
||||
On every conversation turn, the system:
|
||||
|
||||
1. Fetches up to `fetch_limit` memories visible in the current scope
|
||||
2. Extracts context from the last 3 user messages
|
||||
3. Scores memories against that context using a BM25 index
|
||||
4. Injects the top `relevance_k` memories into the system message as
|
||||
`<memories>` XML tags
|
||||
5. Appends a hint telling the model how many memories are in scope
|
||||
|
||||
This means the model always has its most relevant memories available without
|
||||
explicit recall -- but can still use `memory(action='search')` for deeper
|
||||
lookup.
|
||||
|
||||
### Nudges
|
||||
|
||||
The metacognition layer can nudge the model to save memories at appropriate
|
||||
moments (e.g., after a correction or when resuming a workstream). Nudges are
|
||||
rate-limited by `nudge_cooldown` and can be disabled entirely.
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
### config.toml
|
||||
|
||||
```toml
|
||||
[memory]
|
||||
relevance_k = 5 # top-k memories injected per turn
|
||||
fetch_limit = 50 # max memories fetched from storage for scoring
|
||||
max_content = 32768 # max content length per memory (characters)
|
||||
nudge_cooldown = 300 # minimum seconds between memory nudges
|
||||
nudges = true # enable/disable metacognitive nudges
|
||||
```
|
||||
|
||||
All fields are optional. Defaults are shown above.
|
||||
|
||||
---
|
||||
|
||||
## Tool Usage
|
||||
|
||||
The `memory` tool supports four actions:
|
||||
|
||||
### save
|
||||
|
||||
Store or update a memory.
|
||||
|
||||
```json
|
||||
{
|
||||
"action": "save",
|
||||
"name": "project_architecture",
|
||||
"content": "The project uses a hexagonal architecture with...",
|
||||
"description": "Core architecture patterns",
|
||||
"type": "project",
|
||||
"scope": "global"
|
||||
}
|
||||
```
|
||||
|
||||
| Parameter | Required | Default | Description |
|
||||
|---------------|----------|-------------|------------------------------------------|
|
||||
| `name` | yes | -- | Snake_case identifier (max 256 chars) |
|
||||
| `content` | yes | -- | Memory content (max `max_content` chars) |
|
||||
| `description` | no | `""` | Short description for relevance matching |
|
||||
| `type` | no | `"project"` | One of: user, project, feedback, reference |
|
||||
| `scope` | no | `"global"` | One of: global, workstream, user |
|
||||
|
||||
### search
|
||||
|
||||
Find memories by query (BM25 full-text search).
|
||||
|
||||
```json
|
||||
{
|
||||
"action": "search",
|
||||
"query": "authentication patterns",
|
||||
"type": "project",
|
||||
"limit": 10
|
||||
}
|
||||
```
|
||||
|
||||
| Parameter | Required | Default | Description |
|
||||
|-----------|----------|---------|--------------------------------------|
|
||||
| `query` | yes | -- | Search query |
|
||||
| `type` | no | `""` | Filter by type |
|
||||
| `scope` | no | `""` | Filter by scope |
|
||||
| `limit` | no | `20` | Max results (capped at 50) |
|
||||
|
||||
### delete
|
||||
|
||||
Remove a memory by name.
|
||||
|
||||
```json
|
||||
{
|
||||
"action": "delete",
|
||||
"name": "outdated_pattern",
|
||||
"scope": "global"
|
||||
}
|
||||
```
|
||||
|
||||
| Parameter | Required | Default | Description |
|
||||
|------------|----------|------------|--------------------------|
|
||||
| `name` | yes | -- | Memory name to delete |
|
||||
| `scope` | no | `"global"` | Scope of the memory |
|
||||
|
||||
### list
|
||||
|
||||
List all memories with optional filters.
|
||||
|
||||
```json
|
||||
{
|
||||
"action": "list",
|
||||
"type": "feedback",
|
||||
"limit": 50
|
||||
}
|
||||
```
|
||||
|
||||
| Parameter | Required | Default | Description |
|
||||
|-----------|----------|---------|----------------------------|
|
||||
| `type` | no | `""` | Filter by type |
|
||||
| `scope` | no | `""` | Filter by scope |
|
||||
| `limit` | no | `20` | Max results (capped at 50) |
|
||||
|
||||
---
|
||||
|
||||
## Server API
|
||||
|
||||
Four endpoints on the server for programmatic memory access.
|
||||
|
||||
### `GET /v1/api/memories`
|
||||
|
||||
List memories with optional filters.
|
||||
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Required | Default | Description |
|
||||
|------------|--------|----------|---------|------------------------------|
|
||||
| `type` | string | no | `""` | Filter by memory type |
|
||||
| `scope` | string | no | `""` | Filter by scope |
|
||||
| `scope_id` | string | no | `""` | Filter by scope ID |
|
||||
| `limit` | int | no | `100` | Max results (capped at 200) |
|
||||
|
||||
When `scope=user` and `scope_id` is omitted, the authenticated user's ID is
|
||||
used automatically.
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"memories": [
|
||||
{
|
||||
"memory_id": "a1b2c3d4-e5f6-...",
|
||||
"name": "project_architecture",
|
||||
"description": "Core architecture patterns",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"scope_id": "",
|
||||
"content": "The project uses a hexagonal architecture...",
|
||||
"created": "2026-03-10T10:00:00",
|
||||
"updated": "2026-03-12T14:30:00"
|
||||
}
|
||||
],
|
||||
"total": 1
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/memories`
|
||||
|
||||
Save or upsert a structured memory.
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "deployment_process",
|
||||
"content": "Deploy via GitHub Actions. Staging auto-deploys on push to main.",
|
||||
"description": "CI/CD deployment workflow",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"scope_id": ""
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Default | Description |
|
||||
|--------------|--------|----------|-------------|--------------------------------------|
|
||||
| `name` | string | yes | -- | Memory name (max 256 chars) |
|
||||
| `content` | string | yes | -- | Memory content (max 65536 chars) |
|
||||
| `description`| string | no | `""` | Short description for search ranking |
|
||||
| `type` | string | no | `"project"` | One of: user, project, feedback, reference |
|
||||
| `scope` | string | no | `"global"` | One of: global, workstream, user |
|
||||
| `scope_id` | string | no | `""` | Scope qualifier (auto-resolved for user scope) |
|
||||
|
||||
**Response (created):** `201`
|
||||
|
||||
```json
|
||||
{
|
||||
"memory_id": "a1b2c3d4-e5f6-...",
|
||||
"name": "deployment_process",
|
||||
"description": "CI/CD deployment workflow",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"scope_id": "",
|
||||
"content": "Deploy via GitHub Actions...",
|
||||
"created": "2026-03-14T10:00:00",
|
||||
"updated": "2026-03-14T10:00:00"
|
||||
}
|
||||
```
|
||||
|
||||
**Response (updated):** `200` -- same schema, returned when a memory with the
|
||||
same `(name, scope, scope_id)` already existed.
|
||||
|
||||
**Errors:**
|
||||
|
||||
| Status | Condition |
|
||||
|--------|------------------------------------|
|
||||
| 400 | Missing name, empty content, invalid type/scope, content too long |
|
||||
|
||||
---
|
||||
|
||||
### `POST /v1/api/memories/search`
|
||||
|
||||
Search memories by query. Uses POST for the request body but is non-mutating
|
||||
(requires only `read` scope).
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{
|
||||
"query": "authentication",
|
||||
"type": "project",
|
||||
"scope": "",
|
||||
"scope_id": "",
|
||||
"limit": 20
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Default | Description |
|
||||
|------------|--------|----------|---------|--------------------------------|
|
||||
| `query` | string | yes | -- | Search query |
|
||||
| `type` | string | no | `""` | Filter by type |
|
||||
| `scope` | string | no | `""` | Filter by scope |
|
||||
| `scope_id` | string | no | `""` | Filter by scope ID |
|
||||
| `limit` | int | no | `20` | Max results (capped at 50) |
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"memories": [
|
||||
{
|
||||
"memory_id": "a1b2c3d4-e5f6-...",
|
||||
"name": "auth_patterns",
|
||||
"description": "Authentication architecture",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"scope_id": "",
|
||||
"content": "JWT tokens with HS256...",
|
||||
"created": "2026-03-10T10:00:00",
|
||||
"updated": "2026-03-12T14:30:00"
|
||||
}
|
||||
],
|
||||
"total": 1
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `DELETE /v1/api/memories/{name}`
|
||||
|
||||
Delete a memory by name and scope.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|--------|----------------------|
|
||||
| `name` | string | Memory name |
|
||||
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Required | Default | Description |
|
||||
|------------|--------|----------|------------|---------------------|
|
||||
| `scope` | string | no | `"global"` | Scope of the memory |
|
||||
| `scope_id` | string | no | `""` | Scope qualifier |
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{"status": "ok", "name": "deployment_process"}
|
||||
```
|
||||
|
||||
**Response (not found):** `404`
|
||||
|
||||
```json
|
||||
{"error": "Memory 'deployment_process' not found"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Console Admin API
|
||||
|
||||
Four admin endpoints for cross-workstream memory management. All require the
|
||||
`admin.memories` permission.
|
||||
|
||||
### `GET /v1/api/admin/memories`
|
||||
|
||||
List memories across all scopes (no automatic scope resolution).
|
||||
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Required | Default | Description |
|
||||
|------------|--------|----------|---------|------------------------------|
|
||||
| `type` | string | no | `""` | Filter by type |
|
||||
| `scope` | string | no | `""` | Filter by scope |
|
||||
| `scope_id` | string | no | `""` | Filter by scope ID |
|
||||
| `limit` | int | no | `100` | Max results (capped at 200) |
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"memories": [
|
||||
{
|
||||
"memory_id": "a1b2c3d4-e5f6-...",
|
||||
"name": "project_architecture",
|
||||
"description": "Core architecture patterns",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"scope_id": "",
|
||||
"content": "The project uses...",
|
||||
"created": "2026-03-10T10:00:00",
|
||||
"updated": "2026-03-12T14:30:00"
|
||||
}
|
||||
],
|
||||
"total": 1
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/admin/memories/search`
|
||||
|
||||
Search memories by query (uses query parameters, not POST body).
|
||||
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Required | Default | Description |
|
||||
|------------|--------|----------|---------|-------------------------------|
|
||||
| `q` | string | yes | -- | Search query |
|
||||
| `type` | string | no | `""` | Filter by type |
|
||||
| `scope` | string | no | `""` | Filter by scope |
|
||||
| `scope_id` | string | no | `""` | Filter by scope ID |
|
||||
| `limit` | int | no | `20` | Max results (capped at 50) |
|
||||
|
||||
**Response:** `200` -- same schema as `GET /v1/api/admin/memories`.
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/admin/memories/{memory_id}`
|
||||
|
||||
Get a single memory by ID.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-------------|--------|------------------------|
|
||||
| `memory_id` | string | Memory UUID |
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"memory_id": "a1b2c3d4-e5f6-...",
|
||||
"name": "project_architecture",
|
||||
"description": "Core architecture patterns",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"scope_id": "",
|
||||
"content": "The project uses...",
|
||||
"created": "2026-03-10T10:00:00",
|
||||
"updated": "2026-03-12T14:30:00"
|
||||
}
|
||||
```
|
||||
|
||||
**Response (not found):** `404`
|
||||
|
||||
```json
|
||||
{"error": "Memory not found"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `DELETE /v1/api/admin/memories/{memory_id}`
|
||||
|
||||
Delete a memory by ID. Records an audit event (`memory.delete`).
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-------------|--------|------------------------|
|
||||
| `memory_id` | string | Memory UUID |
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{"status": "ok"}
|
||||
```
|
||||
|
||||
**Response (not found):** `404`
|
||||
|
||||
```json
|
||||
{"error": "Memory not found"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## SDK
|
||||
|
||||
### Python
|
||||
|
||||
The server SDK uses `mem_type` (not `type`) to avoid shadowing the Python
|
||||
builtin.
|
||||
|
||||
```python
|
||||
from turnstone.sdk import TurnstoneServer
|
||||
|
||||
with TurnstoneServer("http://localhost:8080", token="tok_xxx") as client:
|
||||
# Save a memory
|
||||
mem = client.save_memory(
|
||||
"api_conventions",
|
||||
"All endpoints use /v1/ prefix. JSON responses.",
|
||||
description="API design patterns",
|
||||
mem_type="project",
|
||||
scope="global",
|
||||
)
|
||||
print(mem.memory_id)
|
||||
|
||||
# Search memories
|
||||
results = client.search_memories("authentication", mem_type="project", limit=10)
|
||||
for m in results.memories:
|
||||
print(f"{m['name']}: {m['description']}")
|
||||
|
||||
# List memories
|
||||
all_mems = client.list_memories(mem_type="feedback", limit=50)
|
||||
|
||||
# Delete a memory
|
||||
client.delete_memory("api_conventions", scope="global")
|
||||
```
|
||||
|
||||
Console admin SDK:
|
||||
|
||||
```python
|
||||
from turnstone.sdk import TurnstoneConsole
|
||||
|
||||
with TurnstoneConsole("http://localhost:9090", token="tok_xxx") as admin:
|
||||
# List all memories (admin view, no scope auto-resolution)
|
||||
result = admin.list_memories(scope="global", limit=100)
|
||||
|
||||
# Search
|
||||
result = admin.search_memories("architecture", mem_type="project")
|
||||
|
||||
# Get by ID
|
||||
mem = admin.get_memory("a1b2c3d4-e5f6-...")
|
||||
|
||||
# Delete by ID
|
||||
admin.delete_memory("a1b2c3d4-e5f6-...")
|
||||
```
|
||||
|
||||
### TypeScript
|
||||
|
||||
```typescript
|
||||
import { TurnstoneServer } from "@turnstone/sdk";
|
||||
|
||||
const client = new TurnstoneServer({
|
||||
baseUrl: "http://localhost:8080",
|
||||
token: "tok_xxx",
|
||||
});
|
||||
|
||||
// Save a memory
|
||||
const mem = await client.saveMemory({
|
||||
name: "api_conventions",
|
||||
content: "All endpoints use /v1/ prefix. JSON responses.",
|
||||
description: "API design patterns",
|
||||
type: "project",
|
||||
scope: "global",
|
||||
});
|
||||
|
||||
// Search memories
|
||||
const results = await client.searchMemories({
|
||||
query: "authentication",
|
||||
type: "project",
|
||||
limit: 10,
|
||||
});
|
||||
|
||||
// List memories
|
||||
const all = await client.listMemories({ type: "feedback", limit: 50 });
|
||||
|
||||
// Delete a memory
|
||||
await client.deleteMemory("api_conventions", { scope: "global" });
|
||||
```
|
||||
|
||||
Console admin SDK:
|
||||
|
||||
```typescript
|
||||
import { TurnstoneConsole } from "@turnstone/sdk";
|
||||
|
||||
const admin = new TurnstoneConsole({
|
||||
baseUrl: "http://localhost:9090",
|
||||
token: "tok_xxx",
|
||||
});
|
||||
|
||||
// List, search, get, delete by ID
|
||||
const mems = await admin.listMemories({ scope: "global" });
|
||||
const found = await admin.searchMemories({ q: "auth", limit: 20 });
|
||||
const one = await admin.getMemory("a1b2c3d4-e5f6-...");
|
||||
await admin.deleteMemory("a1b2c3d4-e5f6-...");
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Storage
|
||||
|
||||
Memories are stored in the `structured_memories` table (migration 013).
|
||||
The unique constraint on `(name, scope, scope_id)` ensures upsert semantics.
|
||||
The name is normalized on save: lowercased, hyphens and spaces replaced with
|
||||
underscores.
|
||||
|
||||
## Architecture
|
||||
|
||||
See [Memory Architecture diagram](diagrams/png/23-memory-architecture.png) for
|
||||
the full data flow covering the session tool path, API path, admin path, and
|
||||
BM25 relevance injection.
|
||||
+429
@@ -0,0 +1,429 @@
|
||||
# OpenID Connect (OIDC) Single Sign-On
|
||||
|
||||
Turnstone supports OpenID Connect for federated authentication, allowing
|
||||
users to log in with their existing corporate identity provider instead of
|
||||
managing a separate password. OIDC is opt-in: when configured, the login
|
||||
screen shows a "Continue with SSO" button alongside the existing
|
||||
username/password form. When not configured, the login experience is
|
||||
unchanged.
|
||||
|
||||
Any OIDC-compliant provider works: Google, Okta, Azure AD, Keycloak,
|
||||
Auth0, OneLogin, and others that publish a
|
||||
`.well-known/openid-configuration` discovery document.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. A registered **confidential** OIDC client at your identity provider
|
||||
2. The client's redirect URI must include:
|
||||
`https://your-turnstone-host/v1/api/auth/oidc/callback`
|
||||
3. A local admin user must exist in Turnstone (complete the initial setup
|
||||
wizard before enabling OIDC)
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
OIDC is configured via environment variables (preferred) or the `[oidc]`
|
||||
section of `config.toml`. Environment variables take precedence when both
|
||||
are set.
|
||||
|
||||
| Variable | Required | Default | Description |
|
||||
|----------|----------|---------|-------------|
|
||||
| `TURNSTONE_OIDC_ISSUER` | Yes | — | Issuer URL (e.g. `https://accounts.google.com`). Must serve `/.well-known/openid-configuration`. |
|
||||
| `TURNSTONE_OIDC_CLIENT_ID` | Yes | — | OAuth 2.0 client ID from your provider |
|
||||
| `TURNSTONE_OIDC_CLIENT_SECRET` | Yes | — | OAuth 2.0 client secret (confidential client) |
|
||||
| `TURNSTONE_OIDC_SCOPES` | No | `openid email profile` | Space-separated OAuth scopes to request |
|
||||
| `TURNSTONE_OIDC_PROVIDER_NAME` | No | `SSO` | Display name for the login button (e.g. "Google", "Okta") |
|
||||
| `TURNSTONE_OIDC_ROLE_CLAIM` | No | — | ID token claim containing role/group values (see [Role Mapping](#role-mapping)) |
|
||||
| `TURNSTONE_OIDC_ROLE_MAP` | No | — | Mapping from claim values to Turnstone role IDs (see [Role Mapping](#role-mapping)) |
|
||||
| `TURNSTONE_OIDC_PASSWORD_ENABLED` | No | `true` | Set to `false` to hide the password form and block all username/password logins (including admin). API tokens and config-file tokens still work. |
|
||||
| `TURNSTONE_OIDC_REDIRECT_BASE` | No | — | Externally-reachable origin for the OIDC redirect URI (e.g. `https://app.example.com`). Recommended when running behind a reverse proxy. When unset, derived from the request Host header. |
|
||||
|
||||
OIDC is enabled when all three required fields (issuer, client ID, client
|
||||
secret) are non-empty. If any is missing, OIDC is silently disabled and
|
||||
the login screen shows only the password form.
|
||||
|
||||
### Reverse Proxy / Load Balancer
|
||||
|
||||
When Turnstone runs behind a reverse proxy, the internal `Host` header may
|
||||
not match the externally-reachable URL. Set `TURNSTONE_OIDC_REDIRECT_BASE`
|
||||
to the public origin so the redirect URI sent to the identity provider is
|
||||
correct:
|
||||
|
||||
```bash
|
||||
TURNSTONE_OIDC_REDIRECT_BASE=https://app.example.com
|
||||
```
|
||||
|
||||
The resulting callback URL will be
|
||||
`https://app.example.com/v1/api/auth/oidc/callback` — register this as the
|
||||
authorized redirect URI in your identity provider.
|
||||
|
||||
### config.toml alternative
|
||||
|
||||
```toml
|
||||
[oidc]
|
||||
issuer = "https://accounts.google.com"
|
||||
client_id = "your-client-id"
|
||||
client_secret = "your-client-secret"
|
||||
scopes = "openid email profile"
|
||||
provider_name = "Google"
|
||||
role_claim = "groups"
|
||||
password_enabled = true
|
||||
redirect_base = "https://app.example.com"
|
||||
|
||||
[oidc.role_map]
|
||||
admin = "builtin-admin"
|
||||
engineering = "builtin-operator"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Provider-Specific Setup
|
||||
|
||||
### Google
|
||||
|
||||
1. Go to [Google Cloud Console](https://console.cloud.google.com/) >
|
||||
**APIs & Services** > **Credentials**
|
||||
2. Click **Create Credentials** > **OAuth 2.0 Client ID**
|
||||
3. Application type: **Web application**
|
||||
4. Add authorized redirect URI:
|
||||
`https://your-turnstone-host/v1/api/auth/oidc/callback`
|
||||
5. Copy the **Client ID** and **Client secret**
|
||||
|
||||
```bash
|
||||
TURNSTONE_OIDC_ISSUER=https://accounts.google.com
|
||||
TURNSTONE_OIDC_CLIENT_ID=123456789.apps.googleusercontent.com
|
||||
TURNSTONE_OIDC_CLIENT_SECRET=GOCSPX-...
|
||||
TURNSTONE_OIDC_PROVIDER_NAME=Google
|
||||
```
|
||||
|
||||
### Okta
|
||||
|
||||
1. In the Okta Admin Console, go to **Applications** > **Create App
|
||||
Integration**
|
||||
2. Sign-in method: **OIDC - OpenID Connect**
|
||||
3. Application type: **Web Application**
|
||||
4. Add sign-in redirect URI:
|
||||
`https://your-turnstone-host/v1/api/auth/oidc/callback`
|
||||
5. Note the **Issuer** (your Okta domain, e.g.
|
||||
`https://dev-123456.okta.com`)
|
||||
|
||||
```bash
|
||||
TURNSTONE_OIDC_ISSUER=https://dev-123456.okta.com
|
||||
TURNSTONE_OIDC_CLIENT_ID=0oaXXXXXXXXXXXXX
|
||||
TURNSTONE_OIDC_CLIENT_SECRET=...
|
||||
TURNSTONE_OIDC_PROVIDER_NAME=Okta
|
||||
TURNSTONE_OIDC_ROLE_CLAIM=groups
|
||||
TURNSTONE_OIDC_ROLE_MAP="admin:builtin-admin,everyone:builtin-operator"
|
||||
```
|
||||
|
||||
### Azure AD (Entra ID)
|
||||
|
||||
1. In the Azure Portal, go to **App registrations** > **New registration**
|
||||
2. Redirect URI: **Web** >
|
||||
`https://your-turnstone-host/v1/api/auth/oidc/callback`
|
||||
3. Under **Certificates & secrets**, create a new **Client secret** and
|
||||
copy the value immediately
|
||||
4. The issuer URL is
|
||||
`https://login.microsoftonline.com/{tenant-id}/v2.0`
|
||||
|
||||
```bash
|
||||
TURNSTONE_OIDC_ISSUER=https://login.microsoftonline.com/YOUR_TENANT_ID/v2.0
|
||||
TURNSTONE_OIDC_CLIENT_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
|
||||
TURNSTONE_OIDC_CLIENT_SECRET=...
|
||||
TURNSTONE_OIDC_PROVIDER_NAME="Azure AD"
|
||||
TURNSTONE_OIDC_ROLE_CLAIM=roles
|
||||
TURNSTONE_OIDC_ROLE_MAP="Admin:builtin-admin,User:builtin-operator"
|
||||
```
|
||||
|
||||
### Keycloak
|
||||
|
||||
1. In the Keycloak Admin Console, select your **Realm**
|
||||
2. Go to **Clients** > **Create client**
|
||||
3. Client type: **OpenID Connect**
|
||||
4. Set **Client authentication** to **On** (confidential)
|
||||
5. Add valid redirect URI:
|
||||
`https://your-turnstone-host/v1/api/auth/oidc/callback`
|
||||
6. The issuer URL is
|
||||
`https://keycloak.example.com/realms/your-realm`
|
||||
|
||||
```bash
|
||||
TURNSTONE_OIDC_ISSUER=https://keycloak.example.com/realms/your-realm
|
||||
TURNSTONE_OIDC_CLIENT_ID=turnstone
|
||||
TURNSTONE_OIDC_CLIENT_SECRET=...
|
||||
TURNSTONE_OIDC_PROVIDER_NAME=Keycloak
|
||||
TURNSTONE_OIDC_ROLE_CLAIM=realm_access.roles
|
||||
TURNSTONE_OIDC_ROLE_MAP="admin:builtin-admin,operator:builtin-operator"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Role Mapping
|
||||
|
||||
OIDC role mapping assigns Turnstone roles to users based on claims in the
|
||||
ID token. This is optional — without it, OIDC users are provisioned with
|
||||
the `builtin-viewer` role (read-only access) by default.
|
||||
|
||||
### Configuration
|
||||
|
||||
Set `TURNSTONE_OIDC_ROLE_CLAIM` to the name of the claim in the ID token
|
||||
that contains the user's group or role memberships. Then set
|
||||
`TURNSTONE_OIDC_ROLE_MAP` to map claim values to Turnstone role IDs.
|
||||
|
||||
The role map is a comma-separated list of `claim_value:turnstone_role`
|
||||
pairs:
|
||||
|
||||
```bash
|
||||
TURNSTONE_OIDC_ROLE_CLAIM=groups
|
||||
TURNSTONE_OIDC_ROLE_MAP="admin:builtin-admin,engineering:builtin-operator,viewer:builtin-viewer"
|
||||
```
|
||||
|
||||
### Behavior
|
||||
|
||||
- **Synced on every login**: roles are added when new claim values appear,
|
||||
and OIDC-assigned roles are revoked when the corresponding claim value
|
||||
is no longer present. Roles assigned manually (or by other sources) are
|
||||
never touched — only roles with `assigned_by="oidc"` are subject to
|
||||
revocation.
|
||||
- **List or string**: the claim value can be a JSON array
|
||||
(`["admin", "engineering"]`) or a single string (`"admin"`). Both are
|
||||
handled correctly.
|
||||
- **Unknown values**: claim values not present in the role map are silently
|
||||
ignored.
|
||||
- **Missing roles**: if the role map references a Turnstone role ID that
|
||||
does not exist in the database, the assignment is skipped (no error).
|
||||
- **Evaluated on every login**: roles are checked and applied each time
|
||||
the user authenticates via OIDC, so new group memberships are picked
|
||||
up on the next login.
|
||||
|
||||
### Built-in Roles
|
||||
|
||||
| Role ID | Permissions |
|
||||
|---------|-------------|
|
||||
| `builtin-admin` | All permissions |
|
||||
| `builtin-operator` | read, write, workstreams.create, workstreams.close |
|
||||
| `builtin-viewer` | read |
|
||||
|
||||
---
|
||||
|
||||
## User Provisioning
|
||||
|
||||
When a user logs in via OIDC for the first time, Turnstone automatically
|
||||
creates a local user account:
|
||||
|
||||
1. The OIDC identity (`issuer` + `sub` claim) is stored in the
|
||||
`oidc_identities` table and linked to the new user
|
||||
2. The **username** is derived from the `preferred_username` claim,
|
||||
falling back to the email local part, with deduplication if needed
|
||||
3. The **display name** comes from the `name` claim, falling back to
|
||||
`preferred_username` or email
|
||||
4. The user's password hash is set to a sentinel value (`!oidc`) — OIDC
|
||||
users cannot log in with a password
|
||||
|
||||
On subsequent logins, the existing user is matched by `(issuer, sub)` and
|
||||
the `last_login` timestamp is updated. Role mapping is re-evaluated on
|
||||
every login.
|
||||
|
||||
---
|
||||
|
||||
## OIDC-Only Mode
|
||||
|
||||
To enforce OIDC for all logins and hide the password form, set:
|
||||
|
||||
```bash
|
||||
TURNSTONE_OIDC_PASSWORD_ENABLED=false
|
||||
```
|
||||
|
||||
In this mode the login screen shows only the "Continue with SSO" button.
|
||||
The password form, token toggle, and sign-in button are all hidden.
|
||||
All username/password logins are blocked at the API level, including
|
||||
admin accounts.
|
||||
|
||||
The first admin account must be created via the setup wizard (with a
|
||||
password) before OIDC is enabled. The setup wizard always works
|
||||
regardless of this setting because it is only available when zero users
|
||||
exist in the database.
|
||||
|
||||
API token login (`POST /v1/api/auth/login` with a `ts_` token) and
|
||||
config-file tokens (`Authorization: Bearer tok_xxx`) continue to work
|
||||
regardless of this setting. OIDC-only mode affects password-based
|
||||
authentication only.
|
||||
|
||||
---
|
||||
|
||||
## Login Flow
|
||||
|
||||
Both the server and console support OIDC login. The flow is identical:
|
||||
|
||||
1. The browser fetches `GET /v1/api/auth/status` at page load
|
||||
2. If the response includes `oidc_enabled: true`, the login screen shows
|
||||
a "Continue with {provider_name}" button
|
||||
3. Clicking the button navigates to `GET /v1/api/auth/oidc/authorize`
|
||||
4. Turnstone generates a state token, nonce, and PKCE verifier, stores
|
||||
them in the database, and redirects the browser to the identity
|
||||
provider's authorization endpoint
|
||||
5. The user authenticates at the identity provider
|
||||
6. The IdP redirects back to
|
||||
`GET /v1/api/auth/oidc/callback?code=...&state=...`
|
||||
7. Turnstone validates the state, exchanges the authorization code for
|
||||
tokens using the PKCE verifier, validates the ID token against the
|
||||
provider's JWKS public keys, provisions or matches the user, and
|
||||
issues a Turnstone JWT
|
||||
8. The browser is redirected to `/?oidc_success=1` with the JWT set in
|
||||
an `HttpOnly` session cookie
|
||||
9. The browser JavaScript detects the `oidc_success` query parameter,
|
||||
strips it from the URL, hides the login overlay, and calls
|
||||
`onLoginSuccess()` to initialize the application
|
||||
|
||||
---
|
||||
|
||||
## API Endpoints
|
||||
|
||||
| Method | Path | Auth | Description |
|
||||
|--------|------|------|-------------|
|
||||
| GET | `/v1/api/auth/oidc/authorize` | Public | Redirects to identity provider |
|
||||
| GET | `/v1/api/auth/oidc/callback` | Public | Handles IdP callback, issues JWT |
|
||||
|
||||
Both endpoints are public (no authentication required) because they are
|
||||
part of the login flow itself.
|
||||
|
||||
### Auth status response
|
||||
|
||||
When OIDC is enabled, `GET /v1/api/auth/status` includes additional
|
||||
fields:
|
||||
|
||||
```json
|
||||
{
|
||||
"auth_enabled": true,
|
||||
"has_users": true,
|
||||
"setup_required": false,
|
||||
"oidc_enabled": true,
|
||||
"oidc_provider_name": "Google",
|
||||
"password_enabled": true
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Database Schema
|
||||
|
||||
Migration 018 creates two tables:
|
||||
|
||||
```sql
|
||||
CREATE TABLE oidc_identities (
|
||||
issuer TEXT NOT NULL,
|
||||
subject TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
email TEXT NOT NULL DEFAULT '',
|
||||
created TEXT NOT NULL,
|
||||
last_login TEXT NOT NULL,
|
||||
PRIMARY KEY (issuer, subject)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_oidc_identities_user_id ON oidc_identities(user_id);
|
||||
|
||||
CREATE TABLE oidc_pending_states (
|
||||
state TEXT PRIMARY KEY,
|
||||
nonce TEXT NOT NULL,
|
||||
code_verifier TEXT NOT NULL,
|
||||
audience TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
```
|
||||
|
||||
The `oidc_identities` table links an OIDC subject (identified by
|
||||
`issuer` + `subject`) to a Turnstone `user_id`. A single user can have
|
||||
multiple OIDC identities (e.g. from different providers).
|
||||
|
||||
The `oidc_pending_states` table stores authorization flow state for
|
||||
callback validation. Entries are automatically cleaned up after 5 minutes.
|
||||
|
||||
---
|
||||
|
||||
## Security Notes
|
||||
|
||||
- **Authorization Code Flow with PKCE**: the recommended OAuth 2.0 flow
|
||||
for web applications. PKCE prevents authorization code interception
|
||||
attacks even without a client secret (though the client secret is still
|
||||
used for additional security).
|
||||
- **ID token validation**: all tokens are validated using the provider's
|
||||
JWKS public keys (RS256 or ES256). The signature, issuer, audience,
|
||||
and expiry are all checked.
|
||||
- **State parameter**: a cryptographically random state token prevents
|
||||
CSRF attacks on the callback endpoint. The state is stored server-side
|
||||
and verified on callback.
|
||||
- **Nonce**: a random nonce is included in the authorization request and
|
||||
verified in the ID token to prevent replay attacks.
|
||||
- **Client secret**: never leaves the server — it is only used in the
|
||||
server-to-IdP token exchange, not exposed to the browser.
|
||||
- **OIDC users cannot use password login**: the sentinel password hash
|
||||
(`!oidc`) ensures `verify_password()` always rejects password attempts
|
||||
for OIDC-provisioned users.
|
||||
- **Rate limiting**: the callback endpoint shares the login rate limiter
|
||||
(5 attempts per 5-minute window per IP).
|
||||
- **State TTL**: pending authorization states expire after 5 minutes.
|
||||
Expired states are lazily cleaned up on each callback.
|
||||
- **Setup guard**: OIDC login requires at least one local admin user to
|
||||
exist. This ensures the initial admin account is always created via the
|
||||
setup wizard with a password, not hijacked by an external identity.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "OIDC not configured"
|
||||
|
||||
All three required environment variables must be set:
|
||||
`TURNSTONE_OIDC_ISSUER`, `TURNSTONE_OIDC_CLIENT_ID`, and
|
||||
`TURNSTONE_OIDC_CLIENT_SECRET`. Check that none are empty or
|
||||
whitespace-only.
|
||||
|
||||
### "Login session expired"
|
||||
|
||||
The authorization flow must complete within 5 minutes. If the user takes
|
||||
too long at the identity provider, the pending state expires. Try again.
|
||||
|
||||
### "Initial setup required"
|
||||
|
||||
OIDC login is blocked until at least one local admin user exists.
|
||||
Complete the setup wizard first (navigate to the Turnstone URL and follow
|
||||
the prompts to create an admin user with a password).
|
||||
|
||||
### Discovery fails at startup
|
||||
|
||||
Check that the issuer URL is reachable from the Turnstone server and
|
||||
serves a valid `/.well-known/openid-configuration` document. The server
|
||||
logs the discovery attempt at startup:
|
||||
|
||||
```
|
||||
OIDC discovery failed for https://your-issuer.example.com: ...
|
||||
```
|
||||
|
||||
OIDC is automatically disabled when discovery fails. Restart the server
|
||||
after fixing the connectivity issue.
|
||||
|
||||
### Redirect URI mismatch
|
||||
|
||||
The redirect URI configured at the identity provider must exactly match
|
||||
`https://your-host/v1/api/auth/oidc/callback`. Common issues:
|
||||
|
||||
- **Scheme mismatch**: the redirect uses `https://` — make sure TLS is
|
||||
configured or a reverse proxy sets the `X-Forwarded-Proto` header
|
||||
- **Port mismatch**: if running on a non-standard port, include it in
|
||||
the redirect URI
|
||||
- **Path mismatch**: the path must include the `/v1` API version prefix
|
||||
|
||||
### User not assigned expected roles
|
||||
|
||||
Check that:
|
||||
|
||||
1. `TURNSTONE_OIDC_ROLE_CLAIM` matches the exact claim name in the ID
|
||||
token (case-sensitive)
|
||||
2. `TURNSTONE_OIDC_ROLE_MAP` maps the correct claim values to valid
|
||||
Turnstone role IDs
|
||||
3. The roles referenced in the map exist in the database (check the
|
||||
admin panel > Roles tab)
|
||||
4. The identity provider is configured to include the claim in the ID
|
||||
token (some providers require explicit scope or claim configuration)
|
||||
+101
-2
@@ -54,7 +54,7 @@ Claims:
|
||||
|-------|-------------|
|
||||
| `sub` | User ID |
|
||||
| `scopes` | Comma-separated scope list (`read,write,approve`) |
|
||||
| `src` | Token source (`password`, `api_token`, `config`) |
|
||||
| `src` | Token source (`password`, `api_token`, `config`, `oidc`) |
|
||||
| `iss` | Issuer — always `turnstone` |
|
||||
| `aud` | Audience — `turnstone-server` or `turnstone-console` |
|
||||
| `iat` | Issued-at timestamp |
|
||||
@@ -90,7 +90,8 @@ Scopes are hierarchical — higher scopes imply all lower ones.
|
||||
|
||||
Public paths bypass authentication entirely: `/`, `/health`, `/metrics`,
|
||||
`/static/*`, `/shared/*`, `/docs`, `/openapi.json`, `/api/auth/login`,
|
||||
`/api/auth/logout`, `/api/auth/status`, `/api/auth/setup`.
|
||||
`/api/auth/logout`, `/api/auth/status`, `/api/auth/setup`,
|
||||
`/api/auth/oidc/authorize`, `/api/auth/oidc/callback`.
|
||||
|
||||
### RBAC (Granular Permissions)
|
||||
|
||||
@@ -199,6 +200,94 @@ Response:
|
||||
The response also sets an `HttpOnly` session cookie containing the JWT,
|
||||
so the browser is immediately authenticated after setup completes.
|
||||
|
||||
### OIDC SSO (Single Sign-On)
|
||||
|
||||
Turnstone supports OIDC Authorization Code Flow with PKCE for
|
||||
single sign-on with external identity providers (Okta, Azure AD,
|
||||
Google, etc.). SSO is opt-in — enabled when the three required
|
||||
environment variables are set. Users are auto-provisioned on first
|
||||
login.
|
||||
|
||||
#### Configuration
|
||||
|
||||
| Variable | Required | Description |
|
||||
|----------|----------|-------------|
|
||||
| `TURNSTONE_OIDC_ISSUER` | Yes | OIDC issuer URL (e.g., `https://accounts.google.com`) |
|
||||
| `TURNSTONE_OIDC_CLIENT_ID` | Yes | Client ID from the identity provider |
|
||||
| `TURNSTONE_OIDC_CLIENT_SECRET` | Yes | Client secret (confidential client) |
|
||||
| `TURNSTONE_OIDC_SCOPES` | No | OIDC scopes (default: `openid email profile`) |
|
||||
| `TURNSTONE_OIDC_PROVIDER_NAME` | No | Display name for the SSO button (default: `SSO`) |
|
||||
| `TURNSTONE_OIDC_ROLE_CLAIM` | No | Claim name in the ID token for role mapping (e.g., `groups`) |
|
||||
| `TURNSTONE_OIDC_ROLE_MAP` | No | Comma-separated `claim_value:role_id` pairs (e.g., `admin:builtin-admin,eng:builtin-operator`) |
|
||||
| `TURNSTONE_OIDC_PASSWORD_ENABLED` | No | Set to `false` to hide password login and force SSO-only |
|
||||
|
||||
OIDC is enabled when all three required variables (`ISSUER`,
|
||||
`CLIENT_ID`, `CLIENT_SECRET`) are set.
|
||||
|
||||
#### Login flow
|
||||
|
||||
1. User clicks "Continue with [Provider]" on the login page
|
||||
2. `GET /v1/api/auth/oidc/authorize` generates state, nonce, and PKCE
|
||||
challenge, stores them in the database, and redirects to the IdP
|
||||
3. User authenticates at the identity provider
|
||||
4. IdP redirects to `/v1/api/auth/oidc/callback` with `code` + `state`
|
||||
5. Server validates state, exchanges the authorization code (with PKCE
|
||||
verifier), and validates the ID token (JWKS signature, issuer,
|
||||
audience, nonce)
|
||||
6. Provisions or matches the user by `(issuer, sub)` — never by
|
||||
username or email
|
||||
7. Issues a JWT (`src: oidc`), sets a session cookie, and redirects to
|
||||
the application
|
||||
|
||||
#### Security measures
|
||||
|
||||
- **PKCE (S256)** — prevents authorization code interception
|
||||
- **State parameter** — one-time use, 5-minute TTL, database-backed
|
||||
(multi-node safe)
|
||||
- **Nonce** — prevents ID token replay
|
||||
- **JWKS validation** — asymmetric algorithm allowlist (RS/ES/PS
|
||||
256-512), HMAC excluded
|
||||
- **Algorithm allowlist enforced** — the signing key is resolved from
|
||||
the JWKS by ``kid``; PyJWK infers the key's algorithm from the JWKS
|
||||
``alg``/``kty`` fields; the token header's ``alg`` must be in the
|
||||
allowlist AND match the key type, preventing algorithm confusion
|
||||
- **Identity matching by (issuer, sub) only** — prevents account
|
||||
takeover via email or username reuse
|
||||
- **`password_enabled=false` enforced server-side** — not just a UI
|
||||
toggle
|
||||
- **Rate limiting** on both authorize and callback endpoints
|
||||
- **OIDC-provisioned users cannot password-login** — the password hash
|
||||
is set to the `!oidc` sentinel, which never matches bcrypt verify
|
||||
|
||||
#### Role mapping
|
||||
|
||||
When `TURNSTONE_OIDC_ROLE_CLAIM` is set (e.g., `groups`), the server
|
||||
reads that claim from the ID token and maps values to Turnstone roles
|
||||
via `TURNSTONE_OIDC_ROLE_MAP`. Roles are synced on every login:
|
||||
matching claim values are added, and stale OIDC-assigned roles are
|
||||
revoked. Roles assigned manually (not by OIDC) are never touched.
|
||||
|
||||
If no role mapping is configured, OIDC users are provisioned with the
|
||||
`builtin-viewer` role by default.
|
||||
|
||||
#### OIDC-only mode
|
||||
|
||||
Setting `TURNSTONE_OIDC_PASSWORD_ENABLED=false` hides the password
|
||||
form on the login page and blocks password-based login at the API
|
||||
level. The setup wizard always works regardless of this setting — the
|
||||
first admin user is created with a password before OIDC is relevant.
|
||||
API tokens and config-file tokens are unaffected by this setting.
|
||||
|
||||
#### Known limitations
|
||||
|
||||
- **No session revocation** — deprovisioned IdP users retain their JWT
|
||||
until the 24-hour expiry
|
||||
- **Single IdP** — configuration supports one issuer (the database
|
||||
schema supports multiple for future expansion)
|
||||
- **Redirect URI** — defaults to request Host header; deployments behind
|
||||
reverse proxies should set `TURNSTONE_OIDC_REDIRECT_BASE` to the
|
||||
externally-reachable origin to pin the redirect URI
|
||||
|
||||
---
|
||||
|
||||
## Token Detection Order
|
||||
@@ -483,3 +572,13 @@ and browsers enforce same-origin policy.
|
||||
refresh, eliminating long-lived static tokens for inter-service auth.
|
||||
- **Secret strength validation** — warning logged when JWT secret is
|
||||
shorter than 32 characters.
|
||||
- **OIDC PKCE enforcement** — S256 code challenge on every
|
||||
authorization request prevents code interception in transit.
|
||||
- **OIDC state/nonce in database** — one-time-use, TTL-bounded tokens
|
||||
stored in the database, safe for multi-node deployments.
|
||||
- **OIDC JWKS-only validation** — ID tokens are verified using the
|
||||
provider's published JWKS keys with asymmetric algorithms only;
|
||||
HMAC-based algorithms are rejected to prevent algorithm confusion.
|
||||
- **OIDC identity binding by (issuer, sub)** — user matching uses the
|
||||
immutable subject identifier, not email or username, preventing
|
||||
account takeover via IdP attribute changes.
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
# System Settings
|
||||
|
||||
> See also: [Settings Architecture diagram](diagrams/png/24-settings-architecture.png)
|
||||
|
||||
The system settings feature provides database-backed configuration for server
|
||||
nodes. Settings are stored in the `system_settings` table and managed through
|
||||
the admin API or console Settings tab. This replaces `config.toml` for
|
||||
non-bootstrap settings on server entry points, while the CLI continues to read
|
||||
`config.toml` directly.
|
||||
|
||||
## Overview
|
||||
|
||||
Settings follow a typed registry pattern: every storable setting has a
|
||||
`SettingDef` entry in `settings_registry.py` with type, default, description,
|
||||
validation constraints, and a `restart_required` flag. Unknown keys are rejected
|
||||
at the API boundary.
|
||||
|
||||
At runtime, `ConfigStore` loads all settings from storage into an in-memory
|
||||
cache. Reads are lock-free dict lookups on an immutable snapshot. Writes acquire
|
||||
a lock, persist to storage, and swap the cache atomically.
|
||||
|
||||
---
|
||||
|
||||
## Precedence
|
||||
|
||||
Settings resolution differs between entry points:
|
||||
|
||||
| Entry point | Chain |
|
||||
|-------------|-------|
|
||||
| **Server** (`turnstone-server`, `turnstone-bridge`) | CLI flag > ConfigStore > registry default |
|
||||
| **CLI** (`turnstone`) | CLI flag > config.toml > argparse default |
|
||||
|
||||
The server's `apply_config()` ignores config.toml sections that overlap with
|
||||
ConfigStore. A startup warning is logged for each overlapping key, directing
|
||||
users to the admin Settings API.
|
||||
|
||||
---
|
||||
|
||||
## Bootstrap vs ConfigStore
|
||||
|
||||
**Bootstrap settings** are required before storage is available (database
|
||||
connection, Redis, auth secrets, server bind address). These stay in
|
||||
`config.toml` and environment variables.
|
||||
|
||||
| Category | Section | Where |
|
||||
|----------|---------|-------|
|
||||
| API credentials | `[api]` | config.toml / env |
|
||||
| Database | `[database]` | config.toml / env |
|
||||
| Redis | `[redis]` | config.toml / env |
|
||||
| Auth | `[auth]` | config.toml / env |
|
||||
| Bridge identity | `[bridge]` | config.toml / env |
|
||||
| Console bind | `[console]` | config.toml / env |
|
||||
|
||||
**ConfigStore settings** (~40 settings) are loaded from the database after
|
||||
storage initialization:
|
||||
|
||||
| Section | Settings |
|
||||
|---------|----------|
|
||||
| `model` | name, temperature, max_tokens, reasoning_effort, context_window |
|
||||
| `session` | instructions, retention_days, compact_max_tokens, auto_compact_pct |
|
||||
| `tools` | timeout, truncation, agent_max_turns, skip_permissions, search, search_threshold, search_max_results |
|
||||
| `server` | workstream_idle_timeout, max_workstreams |
|
||||
| `mcp` | config_path, refresh_interval |
|
||||
| `ratelimit` | enabled, requests_per_second, burst |
|
||||
| `health` | backend_probe_interval, backend_probe_timeout, circuit_breaker_threshold, circuit_breaker_cooldown |
|
||||
| `judge` | enabled, model, provider, base_url, api_key, confidence_threshold, max_context_ratio, timeout, read_only_tools |
|
||||
| `memory` | relevance_k, fetch_limit, max_content, nudge_cooldown, nudges |
|
||||
|
||||
Settings are addressed by dotted key (e.g. `memory.relevance_k`). Each has a
|
||||
declared type (`int`, `float`, `str`, `bool`), optional `min_value`/`max_value`
|
||||
range, optional `choices` list, and an `is_secret` flag.
|
||||
|
||||
---
|
||||
|
||||
## Storage
|
||||
|
||||
The `system_settings` table (migration 015) stores settings as JSON-encoded
|
||||
values with a composite primary key of `(key, node_id)`:
|
||||
|
||||
| Column | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| `key` | text | Dotted setting key (e.g. `model.temperature`) |
|
||||
| `value` | text | JSON-encoded value |
|
||||
| `node_id` | text | Node ID for per-node overrides (empty string = global) |
|
||||
| `is_secret` | int | 1 if the setting contains secrets |
|
||||
| `changed_by` | text | Username of last editor |
|
||||
| `created` | text | ISO timestamp |
|
||||
| `updated` | text | ISO timestamp |
|
||||
|
||||
Per-node overrides layer on top of global settings. When `ConfigStore` loads,
|
||||
it fetches global settings first, then overlays per-node values.
|
||||
|
||||
---
|
||||
|
||||
## Admin API
|
||||
|
||||
Four endpoints on the **console** server, all requiring the `admin.settings`
|
||||
permission.
|
||||
|
||||
### `GET /v1/api/admin/settings`
|
||||
|
||||
List all settings with their effective values, defaults, and metadata.
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"settings": [
|
||||
{
|
||||
"key": "model.temperature",
|
||||
"value": 0.7,
|
||||
"source": "storage",
|
||||
"type": "float",
|
||||
"description": "Sampling temperature",
|
||||
"section": "model",
|
||||
"is_secret": false,
|
||||
"node_id": "",
|
||||
"changed_by": "admin",
|
||||
"updated": "2026-03-14T10:00:00",
|
||||
"restart_required": false
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `GET /v1/api/admin/settings/schema`
|
||||
|
||||
Return the full registry catalog (all defined settings with metadata). Useful
|
||||
for building dynamic admin UIs.
|
||||
|
||||
**Response:** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"schema": [
|
||||
{
|
||||
"key": "model.temperature",
|
||||
"type": "float",
|
||||
"default": 0.5,
|
||||
"description": "Sampling temperature",
|
||||
"section": "model",
|
||||
"is_secret": false,
|
||||
"min_value": 0.0,
|
||||
"max_value": 2.0,
|
||||
"choices": null,
|
||||
"restart_required": false
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `PUT /v1/api/admin/settings/{key}`
|
||||
|
||||
Update a setting. The value is validated against the registry (type coercion,
|
||||
range, choices). Secret settings (`is_secret=true`) cannot be written via the
|
||||
API -- they must be configured via config.toml or environment variables.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|--------|-------------|
|
||||
| `key` | string | Dotted setting key (e.g. `model.temperature`) |
|
||||
|
||||
**Request body:**
|
||||
|
||||
```json
|
||||
{
|
||||
"value": 0.7,
|
||||
"node_id": ""
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Default | Description |
|
||||
|-----------|--------|----------|---------|-------------|
|
||||
| `value` | any | yes | -- | New value (type-coerced against registry) |
|
||||
| `node_id` | string | no | `""` | Node ID for per-node override |
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{
|
||||
"key": "model.temperature",
|
||||
"value": 0.7,
|
||||
"source": "storage",
|
||||
"type": "float",
|
||||
"description": "Sampling temperature",
|
||||
"section": "model",
|
||||
"is_secret": false,
|
||||
"node_id": "",
|
||||
"changed_by": "admin",
|
||||
"updated": "",
|
||||
"restart_required": false
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:**
|
||||
|
||||
| Status | Condition |
|
||||
|--------|-----------|
|
||||
| 400 | Unknown key, invalid value, type mismatch, out of range |
|
||||
| 403 | Secret setting (must use config.toml or env) |
|
||||
|
||||
---
|
||||
|
||||
### `DELETE /v1/api/admin/settings/{key}`
|
||||
|
||||
Reset a setting to its registry default by removing it from storage.
|
||||
|
||||
**Path parameters:**
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|--------|-------------|
|
||||
| `key` | string | Dotted setting key |
|
||||
|
||||
**Query parameters:**
|
||||
|
||||
| Parameter | Type | Required | Default | Description |
|
||||
|-----------|--------|----------|---------|-------------|
|
||||
| `node_id` | string | no | `""` | Node ID (empty = global) |
|
||||
|
||||
**Response (success):** `200`
|
||||
|
||||
```json
|
||||
{"status": "ok", "key": "model.temperature", "default": 0.5}
|
||||
```
|
||||
|
||||
**Response (not found):** `404`
|
||||
|
||||
```json
|
||||
{"error": "Setting 'model.temperature' has no stored value"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Secret Settings
|
||||
|
||||
Settings with `is_secret=True` (currently only `judge.api_key`) are blocked
|
||||
from the write API with a `403` response. This prevents accidental exposure
|
||||
through the admin UI or audit logs. Secret settings must be configured via
|
||||
`config.toml` or environment variables.
|
||||
|
||||
The list endpoint masks secret values: stored secrets appear as `"***"`
|
||||
rather than their actual value.
|
||||
|
||||
---
|
||||
|
||||
## Hot Reload
|
||||
|
||||
`ConfigStore` caches all settings in memory for fast, lock-free reads. To
|
||||
refresh the cache after external changes (e.g. direct database edits or
|
||||
cluster-wide propagation):
|
||||
|
||||
```
|
||||
POST /v1/api/_internal/config-reload
|
||||
```
|
||||
|
||||
This triggers `ConfigStore.reload()`, which re-reads all settings from storage
|
||||
and atomically swaps the cache. The `version` counter increments on every
|
||||
reload.
|
||||
|
||||
**Behavior after reload:**
|
||||
|
||||
- New workstreams pick up updated values immediately (via `session_factory`)
|
||||
- Existing sessions keep their frozen configuration (settings are captured at
|
||||
workstream creation time, not read on every turn)
|
||||
- Settings marked `restart_required=True` need a server restart to take effect
|
||||
|
||||
---
|
||||
|
||||
## Migration from config.toml
|
||||
|
||||
On startup, `warn_migrated_settings()` scans `config.toml` for keys that are
|
||||
now managed by ConfigStore. Each overlap produces a warning:
|
||||
|
||||
```
|
||||
WARNING config.toml [model] temperature is now managed via Settings API —
|
||||
this value will be ignored. Use the admin Settings tab or
|
||||
PUT /v1/api/admin/settings/model.temperature to configure.
|
||||
```
|
||||
|
||||
To migrate:
|
||||
|
||||
1. Note the values from `config.toml` for sections that overlap with ConfigStore
|
||||
2. Use `PUT /v1/api/admin/settings/{key}` or the console Settings tab to set
|
||||
each value
|
||||
3. Remove the migrated sections from `config.toml`
|
||||
4. Restart the server to verify no warnings
|
||||
|
||||
---
|
||||
|
||||
## SDK
|
||||
|
||||
### Python
|
||||
|
||||
```python
|
||||
from turnstone.sdk import TurnstoneConsole
|
||||
|
||||
with TurnstoneConsole("http://localhost:9090", token="tok_xxx") as admin:
|
||||
# List all settings with effective values
|
||||
result = admin.list_settings()
|
||||
for s in result["settings"]:
|
||||
print(f"{s['key']} = {s['value']} (source: {s['source']})")
|
||||
|
||||
# Get the schema catalog
|
||||
schema = admin.get_settings_schema()
|
||||
|
||||
# Update a setting
|
||||
admin.update_setting("model.temperature", value=0.7)
|
||||
|
||||
# Update with per-node override
|
||||
admin.update_setting("model.temperature", value=0.3, node_id="node-2")
|
||||
|
||||
# Reset to default
|
||||
admin.delete_setting("model.temperature")
|
||||
```
|
||||
|
||||
### TypeScript
|
||||
|
||||
```typescript
|
||||
import { TurnstoneConsole } from "@turnstone/sdk";
|
||||
|
||||
const admin = new TurnstoneConsole({
|
||||
baseUrl: "http://localhost:9090",
|
||||
token: "tok_xxx",
|
||||
});
|
||||
|
||||
// List all settings
|
||||
const result = await admin.listSettings();
|
||||
for (const s of result.settings) {
|
||||
console.log(`${s.key} = ${s.value} (source: ${s.source})`);
|
||||
}
|
||||
|
||||
// Get schema catalog
|
||||
const schema = await admin.getSettingsSchema();
|
||||
|
||||
// Update a setting
|
||||
await admin.updateSetting("model.temperature", { value: 0.7 });
|
||||
|
||||
// Reset to default
|
||||
await admin.deleteSetting("model.temperature");
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
See [Settings Architecture diagram](diagrams/png/24-settings-architecture.png)
|
||||
for the full data flow covering server startup, admin API writes, hot reload,
|
||||
and settings precedence.
|
||||
+37
-42
@@ -1,6 +1,6 @@
|
||||
# Tools Reference
|
||||
|
||||
turnstone exposes 18 built-in tools plus any number of external MCP tools to the
|
||||
turnstone exposes 17 built-in tools plus any number of external MCP tools to the
|
||||
LLM via the OpenAI function-calling interface. Built-in tools are defined as JSON
|
||||
files under `turnstone/tools/` and loaded at startup by `turnstone/core/tools.py`.
|
||||
MCP tools are discovered from configured MCP servers at startup by
|
||||
@@ -46,12 +46,12 @@ schema plus turnstone-specific metadata keys:
|
||||
|
||||
| Name | Description |
|
||||
|---------------------|-------------|
|
||||
| `TOOLS` | All 18 tool definitions (sent to the model). |
|
||||
| `TOOLS` | All 17 tool definitions (sent to the model). |
|
||||
| `AGENT_TOOLS` | Tools with `agent: true` -- available to plan sub-agents. Read-only tools. |
|
||||
| `TASK_AGENT_TOOLS` | Tools with `task_agent: true` -- available to task sub-agents. Includes write operations. |
|
||||
| `AGENT_AUTO_TOOLS` | Set of tool names with `auto_approve: true` -- no user confirmation needed. |
|
||||
| `TASK_AUTO_TOOLS` | Same as `AGENT_AUTO_TOOLS` (identical filter). |
|
||||
| `BUILTIN_TOOL_NAMES`| Frozenset of all 18 built-in tool names. Used by tool search to distinguish always-on tools from deferrable MCP tools. |
|
||||
| `BUILTIN_TOOL_NAMES`| Frozenset of all 17 built-in tool names. Used by tool search to distinguish always-on tools from deferrable MCP tools. |
|
||||
| `PRIMARY_KEY_MAP` | Dict mapping tool name to its `primary_key` parameter name. |
|
||||
|
||||
---
|
||||
@@ -69,7 +69,7 @@ Tool execution follows a three-phase pipeline inside `ChatSession._execute_tools
|
||||
- Parses the JSON arguments (with fallback for malformed JSON).
|
||||
- If JSON parsing fails entirely, uses `PRIMARY_KEY_MAP` to map a bare string
|
||||
to the correct parameter.
|
||||
- Dispatches to the matching `_prepare_{func_name}()` handler. There are 18
|
||||
- Dispatches to the matching `_prepare_{func_name}()` handler. There are 17
|
||||
built-in tools plus `tool_search` (synthetic, client-side BM25 fallback) and
|
||||
the generic `_prepare_mcp_tool()` handler for MCP tools.
|
||||
- Validates arguments and builds a preview dict containing:
|
||||
@@ -87,8 +87,11 @@ All prepared items are sent to the UI via `ui.approve_tools(items)`.
|
||||
but do not block execution.
|
||||
- Items where `needs_approval` is `True` require the user to accept or deny.
|
||||
- The user can provide feedback alongside their approval (e.g. "y, use full path").
|
||||
- If `auto_approve` is `True` on the session (headless mode), all tools are
|
||||
approved automatically.
|
||||
- Choosing "always" (key `a`) adds the pending tool names to `auto_approve_tools`,
|
||||
so that specific tool type is auto-approved going forward (other tool types still
|
||||
prompt). This is per-tool, not blanket.
|
||||
- If `auto_approve` is `True` on the session (via `--skip-permissions` or workstream
|
||||
template), all tools are approved automatically.
|
||||
|
||||
### Phase 3: Execute
|
||||
|
||||
@@ -114,9 +117,8 @@ Each item's `execute` callable is invoked:
|
||||
- `read_file` -- reads files, no side effects
|
||||
- `search` -- grep-style search, no side effects
|
||||
- `man` -- reads man pages, no side effects
|
||||
- `remember` -- writes to persistent memory database (lightweight, always auto-approved)
|
||||
- `recall` -- reads from persistent memory database
|
||||
- `forget` -- deletes from persistent memory database (lightweight, always auto-approved)
|
||||
- `memory` -- structured persistent memory (save/search/delete/list)
|
||||
- `recall` -- searches conversation history
|
||||
- `notify` -- sends notifications to linked channels (time-sensitive, auto-approved for urgency)
|
||||
|
||||
**Requires user confirmation** (write operations, network access, side effects):
|
||||
@@ -164,9 +166,8 @@ Every tool defines a `primary_key`. The mapping is:
|
||||
| `web_search` | `query` |
|
||||
| `task` | `prompt` |
|
||||
| `plan` | `prompt` |
|
||||
| `remember` | `key` |
|
||||
| `memory` | `name` |
|
||||
| `recall` | `query` |
|
||||
| `forget` | `key` |
|
||||
| `notify` | `message` |
|
||||
| `read_resource` | `uri` |
|
||||
| `use_prompt` | `name` |
|
||||
@@ -353,16 +354,22 @@ Plan before implementing -- an autonomous agent explores the codebase and writes
|
||||
|
||||
## Memory
|
||||
|
||||
### remember
|
||||
### memory
|
||||
|
||||
Save a persistent memory that persists across sessions.
|
||||
Structured persistent memory across sessions with typed, scoped entries.
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|-------------|
|
||||
| `key` | string | yes | Short identifier (e.g. `user_name`). |
|
||||
| `value` | string | yes | Content to remember. |
|
||||
| Parameter | Type | Required | Description |
|
||||
|---------------|---------|----------|-------------|
|
||||
| `action` | string | yes | `save`, `search`, `delete`, or `list`. |
|
||||
| `name` | string | save/delete | Short snake_case identifier for the memory. |
|
||||
| `content` | string | save | Memory content to store. |
|
||||
| `description` | string | no | Short description for relevance matching (recommended for `save`). |
|
||||
| `type` | string | no | Memory type: `user`, `project`, `feedback`, or `reference`. Default: `project`. |
|
||||
| `scope` | string | no | Memory scope: `global`, `workstream`, or `user`. Default: `global`. |
|
||||
| `query` | string | search | Search query for finding memories. |
|
||||
| `limit` | integer | no | Max results for `search` or `list`. Default: 20. |
|
||||
|
||||
- **What it does**: Stores a key-value pair in the SQLite memory database. Memories persist across sessions and are included in the system prompt on startup.
|
||||
- **What it does**: Manages structured persistent memories in the database. Memories persist across sessions, have a type classification (user preferences, project knowledge, feedback, reference material) and a scope (global across all workstreams, private to a workstream, or following a user). Relevant memories are included in the system prompt on startup.
|
||||
- **Auto-approve**: Yes.
|
||||
- **Agent availability**: Not available to sub-agents (top-level only).
|
||||
|
||||
@@ -370,28 +377,14 @@ Save a persistent memory that persists across sessions.
|
||||
|
||||
### recall
|
||||
|
||||
Search memories and past conversations.
|
||||
Search conversation history for past messages and tool results.
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|---------|----------|-------------|
|
||||
| `query` | string | no | Search term or phrase. Omit to list all memories. |
|
||||
| `limit` | integer | no | Max conversation results to return (default 20). |
|
||||
| `query` | string | yes | Search term or phrase to find in conversation history. |
|
||||
| `limit` | integer | no | Max results to return (default 20). |
|
||||
|
||||
- **What it does**: With no query, lists all saved memories. With a query, searches both the memory store and conversation history using FTS5 full-text search.
|
||||
- **Auto-approve**: Yes.
|
||||
- **Agent availability**: Not available to sub-agents (top-level only).
|
||||
|
||||
---
|
||||
|
||||
### forget
|
||||
|
||||
Remove a persistent memory by key.
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|--------|----------|-------------|
|
||||
| `key` | string | yes | The memory key to remove (e.g. `user_name`). |
|
||||
|
||||
- **What it does**: Deletes the memory entry with the given key from the SQLite database.
|
||||
- **What it does**: Searches conversation history across sessions using FTS5 full-text search. Returns matching messages, tool calls, and tool results with timestamps and workstream context.
|
||||
- **Auto-approve**: Yes.
|
||||
- **Agent availability**: Not available to sub-agents (top-level only).
|
||||
|
||||
@@ -514,9 +507,8 @@ data.get("mergedAt") is not None
|
||||
| `web_search` | Info | No | Yes | Yes | `query` |
|
||||
| `task` | Agent | No | No | No | `prompt` |
|
||||
| `plan` | Agent | No | No | No | `prompt` |
|
||||
| `remember` | Memory | Yes | No | No | `key` |
|
||||
| `memory` | Memory | Yes | No | No | `name` |
|
||||
| `recall` | Memory | Yes | No | No | `query` |
|
||||
| `forget` | Memory | Yes | No | No | `key` |
|
||||
| `notify` | Notify | Yes | Yes | Yes | `message` |
|
||||
| `watch` | Monitor | No (create) | No | No | `command` |
|
||||
| `read_resource`| MCP | No | Yes | Yes | `uri` |
|
||||
@@ -573,7 +565,7 @@ CLI flags override the config file:
|
||||
search stays off and all tools are sent to the model directly.
|
||||
|
||||
2. **Partitioning**: When active, tools are split into two sets:
|
||||
- **Always-on** -- the 18 built-in tools (members of `BUILTIN_TOOL_NAMES`).
|
||||
- **Always-on** -- the 17 built-in tools (members of `BUILTIN_TOOL_NAMES`).
|
||||
These are always visible to the model.
|
||||
- **Deferred** -- all MCP tools. These are not sent in the tool list unless
|
||||
the model searches for them.
|
||||
@@ -616,7 +608,7 @@ MCP-compatible service.
|
||||
3. **Schema conversion**: Each MCP tool's `inputSchema` is converted to OpenAI
|
||||
function-calling format. The tool name is prefixed: `mcp__{server}__{tool}`.
|
||||
|
||||
4. **Merging**: MCP tools are appended after the 18 built-in tools via
|
||||
4. **Merging**: MCP tools are appended after the 17 built-in tools via
|
||||
`merge_mcp_tools()`. Built-in tools appear first, giving them natural LLM priority.
|
||||
When dynamic tool search is active, MCP tools are deferred rather than directly
|
||||
visible -- the model discovers them via search as needed (see
|
||||
@@ -631,8 +623,11 @@ MCP-compatible service.
|
||||
MCP tools **require user approval by default** (`needs_approval: True`). turnstone
|
||||
does not auto-approve MCP tools based on their schema, since it cannot guarantee
|
||||
that external tools are read-only. However, global overrides such as
|
||||
`--skip-permissions` or the UI's "always allow" setting will auto-approve all
|
||||
tools, including MCP tools.
|
||||
`--skip-permissions` will auto-approve all tools, including MCP tools. The
|
||||
interactive "Always" button adds specific tool types to the per-tool auto-approve
|
||||
set. The web UI and server use `approval_label` for MCP tools, giving
|
||||
per-prompt/per-resource granularity. The CLI and bridge use `func_name`, which
|
||||
gives per-tool-type granularity (e.g., all `use_prompt` calls).
|
||||
|
||||
### Sub-agent availability
|
||||
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
# MCP Cluster Ops
|
||||
|
||||
An MCP server that exposes tools for executing commands across a [Turnstone](https://github.com/turnstonelabs/turnstone) cluster. Serves as a reference implementation for both MCP server patterns and Turnstone MQ client SDK usage.
|
||||
|
||||
## How it works
|
||||
|
||||
This server uses Turnstone's MQ client (`TurnstoneClient`) to dispatch shell commands to specific nodes via Redis. Remote agents execute the command and the raw bash output is captured directly from the `ToolResultEvent` stream — bypassing the costly "agent reads output → re-generates output as completion tokens" round-trip.
|
||||
|
||||
Multi-node dispatches run in parallel via `asyncio.gather`, so total wall time is bounded by the slowest node rather than the sum.
|
||||
|
||||
## Tools
|
||||
|
||||
| Tool | Description |
|
||||
|------|-------------|
|
||||
| `list_nodes` | Discover active nodes in the cluster |
|
||||
| `run_on_node` | Execute a command on a specific node |
|
||||
| `run_on_nodes` | Execute a command on selected nodes in parallel |
|
||||
| `run_on_all_nodes` | Execute a command on ALL active nodes in parallel |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- A running Turnstone cluster (at least one `turnstone-server` + `turnstone-bridge`)
|
||||
- Redis accessible from wherever this MCP server runs
|
||||
- Python 3.11+
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
# From the turnstone repo root:
|
||||
pip install -e ./examples/mcp-cluster-ops
|
||||
|
||||
# Or install turnstone with MQ support first, then the example:
|
||||
pip install -e ".[mq]"
|
||||
pip install -e ./examples/mcp-cluster-ops
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `REDIS_HOST` | `localhost` | Redis host |
|
||||
| `REDIS_PORT` | `6379` | Redis port |
|
||||
| `REDIS_PASSWORD` | _(none)_ | Redis password (use env vars, not config files) |
|
||||
| `MCP_CLUSTER_OPS_TIMEOUT` | `120` | Default command timeout (seconds, clamped 5-3600) |
|
||||
| `MCP_CLUSTER_OPS_MAX_OUTPUT` | `8192` | Max output bytes per node (0 = unlimited) |
|
||||
| `MCP_CLUSTER_OPS_MAX_NODES` | `32` | Max concurrent node dispatches |
|
||||
| `MCP_CLUSTER_OPS_MAX_COMMAND` | `65536` | Max command string length |
|
||||
|
||||
### Register with Turnstone
|
||||
|
||||
**TOML** (`~/.config/turnstone/config.toml`):
|
||||
|
||||
```toml
|
||||
[mcp.servers.cluster-ops]
|
||||
command = "mcp-cluster-ops"
|
||||
|
||||
[mcp.servers.cluster-ops.env]
|
||||
REDIS_HOST = "redis.example.com"
|
||||
```
|
||||
|
||||
**JSON** (via `--mcp-config`):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"cluster-ops": {
|
||||
"command": "mcp-cluster-ops",
|
||||
"env": {
|
||||
"REDIS_HOST": "redis.example.com"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Usage Examples
|
||||
|
||||
Once registered, the tools appear in any Turnstone session. The model can:
|
||||
|
||||
```
|
||||
> Check disk usage across the cluster
|
||||
|
||||
[calls list_nodes → discovers node-1, node-2, node-3]
|
||||
[calls run_on_all_nodes with "df -h /"]
|
||||
|
||||
node-1: /dev/sda1 500G 320G 180G 64% /
|
||||
node-2: /dev/sda1 500G 410G 90G 82% /
|
||||
node-3: /dev/sda1 1.0T 200G 800G 20% /
|
||||
```
|
||||
|
||||
## Why MQ client instead of HTTP SDK?
|
||||
|
||||
The HTTP SDK (`TurnstoneServer`) talks to a single server instance. The MQ client (`TurnstoneClient`) routes through Redis with `target_node` support, which is the entire point of cross-node cluster operations.
|
||||
|
||||
## Security Considerations
|
||||
|
||||
**This MCP server grants the calling agent shell access to cluster nodes.**
|
||||
|
||||
- Commands are executed with `auto_approve=True` and the privileges of the
|
||||
Turnstone server process on the target node.
|
||||
- Command output (which may contain secrets, credentials, or sensitive data)
|
||||
is returned through the MCP tool result and becomes part of the LLM context.
|
||||
- The security boundary is at the MCP host layer -- use Turnstone's tool
|
||||
policy system to restrict which agents can invoke these tools.
|
||||
- Set `REDIS_PASSWORD` via your environment or a secrets manager -- avoid
|
||||
hardcoding passwords in config files.
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
cd examples/mcp-cluster-ops
|
||||
|
||||
# Run tests
|
||||
pip install -e ".[test]"
|
||||
pytest
|
||||
|
||||
# Lint
|
||||
pip install -e ".[dev]"
|
||||
ruff check mcp_cluster_ops/
|
||||
mypy --strict mcp_cluster_ops/
|
||||
```
|
||||
@@ -0,0 +1,3 @@
|
||||
"""MCP server for Turnstone cluster operations."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,4 @@
|
||||
from mcp_cluster_ops.server import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,404 @@
|
||||
"""MCP server for Turnstone cluster operations.
|
||||
|
||||
Exposes tools to execute commands on specific nodes in a Turnstone cluster.
|
||||
Uses the MQ client (``TurnstoneClient``) for direct node targeting via Redis.
|
||||
|
||||
Usage::
|
||||
|
||||
mcp-cluster-ops # via entry point
|
||||
python -m mcp_cluster_ops # via module
|
||||
|
||||
Configure in ``~/.config/turnstone/config.toml``::
|
||||
|
||||
[mcp.servers.cluster-ops]
|
||||
command = "mcp-cluster-ops"
|
||||
|
||||
[mcp.servers.cluster-ops.env]
|
||||
REDIS_HOST = "redis.example.com"
|
||||
|
||||
Environment variables
|
||||
---------------------
|
||||
REDIS_HOST Redis host (default: localhost)
|
||||
REDIS_PORT Redis port (default: 6379)
|
||||
REDIS_PASSWORD Redis password (default: none)
|
||||
MCP_CLUSTER_OPS_TIMEOUT Default command timeout in seconds (default: 120)
|
||||
MCP_CLUSTER_OPS_MAX_OUTPUT Max output bytes per node (default: 8192, 0=unlimited)
|
||||
|
||||
Performance notes
|
||||
-----------------
|
||||
Remote agents are told to reply with only "ok" or "failed" — the raw bash
|
||||
output is captured directly from the ToolResultEvent that already flows
|
||||
through Redis, bypassing the costly "agent reads output then re-generates
|
||||
output as completion tokens" round-trip.
|
||||
|
||||
All multi-node dispatches run in parallel via ``asyncio.gather`` so total
|
||||
wall time is bounded by the slowest node, not the sum of all nodes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from mcp.server.fastmcp import Context, FastMCP
|
||||
from turnstone.mq.client import TurnResult, TurnstoneClient
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DEFAULT_TIMEOUT = int(os.environ.get("MCP_CLUSTER_OPS_TIMEOUT", "120"))
|
||||
_DEFAULT_MAX_OUTPUT = int(os.environ.get("MCP_CLUSTER_OPS_MAX_OUTPUT", "8192"))
|
||||
_MAX_CONCURRENT_NODES = int(os.environ.get("MCP_CLUSTER_OPS_MAX_NODES", "32"))
|
||||
_MAX_COMMAND_LEN = int(os.environ.get("MCP_CLUSTER_OPS_MAX_COMMAND", "65536"))
|
||||
_MIN_TIMEOUT = 5
|
||||
_MAX_TIMEOUT = 3600
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers (pure functions, easily testable)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _redis_kwargs() -> dict[str, Any]:
|
||||
"""Build Redis connection kwargs from environment variables.
|
||||
|
||||
Follows the same env var convention as ``turnstone.mq.broker.add_redis_args``:
|
||||
``REDIS_HOST``, ``REDIS_PORT``, ``REDIS_PASSWORD``.
|
||||
"""
|
||||
kwargs: dict[str, Any] = {"host": os.environ.get("REDIS_HOST", "localhost")}
|
||||
port = os.environ.get("REDIS_PORT")
|
||||
if port is not None:
|
||||
kwargs["port"] = int(port)
|
||||
password = os.environ.get("REDIS_PASSWORD")
|
||||
if password:
|
||||
kwargs["password"] = password
|
||||
return kwargs
|
||||
|
||||
|
||||
def _exec_prompt(command: str) -> str:
|
||||
"""Build the prompt sent to the remote agent.
|
||||
|
||||
Instructs it to run the command and reply minimally so that the raw
|
||||
bash output (captured via ToolResultEvent) is the primary result,
|
||||
avoiding token waste from re-transcription.
|
||||
"""
|
||||
return (
|
||||
"Execute this shell command using the bash tool:\n"
|
||||
f" {command}\n\n"
|
||||
"After the tool completes, reply with only 'ok' or 'failed'.\n"
|
||||
"Do NOT repeat, quote, or summarise the command output in your reply."
|
||||
)
|
||||
|
||||
|
||||
def _extract_output(result: TurnResult) -> str:
|
||||
"""Extract useful output from a TurnResult.
|
||||
|
||||
Prefers raw bash ToolResultEvent output (zero LLM re-transcription cost)
|
||||
over agent content. Falls back through tool results and content.
|
||||
"""
|
||||
bash_outputs = [out for name, out in result.tool_results if name == "bash"]
|
||||
if bash_outputs:
|
||||
return "\n".join(bash_outputs)
|
||||
content: str = result.content
|
||||
if content:
|
||||
return content
|
||||
if result.tool_results:
|
||||
return str(result.tool_results[0][1])
|
||||
return ""
|
||||
|
||||
|
||||
def _truncate(text: str, max_bytes: int) -> str:
|
||||
"""Truncate *text* to at most *max_bytes* UTF-8 bytes.
|
||||
|
||||
Appends a marker when truncation occurs. Handles multi-byte characters
|
||||
safely by decoding with ``errors='ignore'``.
|
||||
|
||||
Pass ``max_bytes=0`` to disable truncation.
|
||||
"""
|
||||
if max_bytes <= 0:
|
||||
return text
|
||||
encoded = text.encode("utf-8")
|
||||
if len(encoded) <= max_bytes:
|
||||
return text
|
||||
truncated = encoded[:max_bytes].decode("utf-8", errors="ignore")
|
||||
omitted = len(encoded) - len(truncated.encode("utf-8"))
|
||||
return truncated + f"\n... [truncated: {omitted} bytes omitted]"
|
||||
|
||||
|
||||
def _clamp_timeout(timeout: int) -> float:
|
||||
"""Clamp timeout to a safe range."""
|
||||
return float(max(_MIN_TIMEOUT, min(timeout, _MAX_TIMEOUT)))
|
||||
|
||||
|
||||
def _validate_command(command: str) -> str | None:
|
||||
"""Validate a command string. Returns an error message or None."""
|
||||
if not command.strip():
|
||||
return "command must be a non-empty string"
|
||||
if len(command) > _MAX_COMMAND_LEN:
|
||||
return f"command too long ({len(command)} chars, max {_MAX_COMMAND_LEN})"
|
||||
return None
|
||||
|
||||
|
||||
def _format_node_result(
|
||||
node_id: str,
|
||||
result: TurnResult,
|
||||
max_output: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Format a single node's TurnResult for JSON output."""
|
||||
raw = _extract_output(result)
|
||||
output = _truncate(raw, max_output)
|
||||
entry: dict[str, Any] = {
|
||||
"node": node_id,
|
||||
"ok": result.ok,
|
||||
}
|
||||
if result.timed_out:
|
||||
entry["timed_out"] = True
|
||||
if result.ok:
|
||||
entry["output"] = output
|
||||
else:
|
||||
entry["output"] = output or None
|
||||
if result.errors:
|
||||
entry["error"] = "; ".join(result.errors)
|
||||
return entry
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core dispatch functions (testable with mocked TurnstoneClient)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _exec_on_node_sync(
|
||||
redis_kw: dict[str, Any],
|
||||
node_id: str,
|
||||
command: str,
|
||||
timeout: float,
|
||||
) -> tuple[str, TurnResult]:
|
||||
"""Dispatch *command* to *node_id* and block until complete.
|
||||
|
||||
Runs inside ``asyncio.to_thread`` so it does not block the event loop.
|
||||
Each call creates its own ``TurnstoneClient`` to avoid Redis pub/sub
|
||||
subscription conflicts between concurrent dispatches.
|
||||
"""
|
||||
prompt = _exec_prompt(command)
|
||||
with TurnstoneClient(**redis_kw) as client:
|
||||
result = client.send_and_wait(
|
||||
message=prompt,
|
||||
target_node=node_id,
|
||||
auto_approve=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
return node_id, result
|
||||
|
||||
|
||||
async def _dispatch_parallel(
|
||||
redis_kw: dict[str, Any],
|
||||
node_ids: list[str],
|
||||
command: str,
|
||||
timeout: float,
|
||||
max_output: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Dispatch *command* to all *node_ids* concurrently.
|
||||
|
||||
Total wall time is bounded by the slowest node.
|
||||
"""
|
||||
tasks = [
|
||||
asyncio.to_thread(_exec_on_node_sync, redis_kw, nid, command, timeout) for nid in node_ids
|
||||
]
|
||||
outcomes = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
for nid, outcome in zip(node_ids, outcomes, strict=True):
|
||||
if isinstance(outcome, BaseException):
|
||||
if not isinstance(outcome, Exception):
|
||||
raise outcome # propagate KeyboardInterrupt, SystemExit, etc.
|
||||
results.append({"node": nid, "ok": False, "error": str(outcome)})
|
||||
else:
|
||||
_, turn_result = outcome
|
||||
results.append(_format_node_result(nid, turn_result, max_output))
|
||||
return results
|
||||
|
||||
|
||||
def _list_nodes_sync(redis_kw: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""List active cluster nodes (blocking)."""
|
||||
with TurnstoneClient(**redis_kw) as client:
|
||||
nodes: list[dict[str, Any]] = client.list_nodes()
|
||||
return nodes
|
||||
|
||||
|
||||
async def _list_nodes_impl(redis_kw: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""List active cluster nodes."""
|
||||
return await asyncio.to_thread(_list_nodes_sync, redis_kw)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MCP server
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _lifespan(server: FastMCP[dict[str, Any]]) -> AsyncIterator[dict[str, Any]]:
|
||||
"""Lifespan context — stores Redis kwargs for tool handlers."""
|
||||
kw = _redis_kwargs()
|
||||
yield {"redis_kwargs": kw}
|
||||
|
||||
|
||||
mcp = FastMCP(
|
||||
"turnstone-cluster-ops",
|
||||
instructions=(
|
||||
"Tools for executing commands across a Turnstone AI cluster. "
|
||||
"Use list_nodes first to discover available nodes, then run_on_node "
|
||||
"to execute commands on specific nodes or run_on_all_nodes for "
|
||||
"cluster-wide operations."
|
||||
),
|
||||
lifespan=_lifespan,
|
||||
)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
async def list_nodes(ctx: Context[Any, Any, Any]) -> str:
|
||||
"""List all active nodes in the Turnstone cluster.
|
||||
|
||||
Call this before dispatching work to discover available node IDs.
|
||||
Returns a JSON array of node metadata objects.
|
||||
"""
|
||||
redis_kw: dict[str, Any] = ctx.request_context.lifespan_context["redis_kwargs"]
|
||||
nodes = await _list_nodes_impl(redis_kw)
|
||||
return json.dumps(nodes, indent=2)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
async def run_on_node(
|
||||
node_id: str,
|
||||
command: str,
|
||||
ctx: Context[Any, Any, Any],
|
||||
timeout: int = _DEFAULT_TIMEOUT,
|
||||
) -> str:
|
||||
"""Execute a shell command on a specific node and return the raw output.
|
||||
|
||||
Use list_nodes first to discover available node IDs.
|
||||
|
||||
Args:
|
||||
node_id: Target node ID (e.g. 'worker-1.example.com').
|
||||
command: Shell command to execute on the target node.
|
||||
timeout: Timeout in seconds (default: 120).
|
||||
"""
|
||||
node_id = node_id.strip()
|
||||
if not node_id:
|
||||
return json.dumps({"error": "node_id must be a non-empty string"})
|
||||
cmd_err = _validate_command(command)
|
||||
if cmd_err:
|
||||
return json.dumps({"error": cmd_err})
|
||||
|
||||
redis_kw: dict[str, Any] = ctx.request_context.lifespan_context["redis_kwargs"]
|
||||
max_output = _DEFAULT_MAX_OUTPUT
|
||||
|
||||
log.info("run_on_node node=%s cmd=%r", node_id, command)
|
||||
_, result = await asyncio.to_thread(
|
||||
_exec_on_node_sync, redis_kw, node_id, command, _clamp_timeout(timeout)
|
||||
)
|
||||
formatted = _format_node_result(node_id, result, max_output)
|
||||
return json.dumps(formatted, indent=2)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
async def run_on_nodes(
|
||||
node_ids: list[str],
|
||||
command: str,
|
||||
ctx: Context[Any, Any, Any],
|
||||
timeout: int = _DEFAULT_TIMEOUT,
|
||||
) -> str:
|
||||
"""Execute a shell command on specific nodes in parallel.
|
||||
|
||||
Results are collected from each node. Total wall time is bounded by
|
||||
the slowest node rather than the sum.
|
||||
|
||||
Args:
|
||||
node_ids: List of node IDs to target.
|
||||
command: Shell command to execute.
|
||||
timeout: Timeout per node in seconds (default: 120).
|
||||
"""
|
||||
cmd_err = _validate_command(command)
|
||||
if cmd_err:
|
||||
return json.dumps({"error": cmd_err})
|
||||
|
||||
redis_kw: dict[str, Any] = ctx.request_context.lifespan_context["redis_kwargs"]
|
||||
max_output = _DEFAULT_MAX_OUTPUT
|
||||
|
||||
clean_ids = list(dict.fromkeys(nid.strip() for nid in node_ids if nid.strip()))
|
||||
if not clean_ids:
|
||||
return json.dumps({"error": "node_ids must be a non-empty list"})
|
||||
if len(clean_ids) > _MAX_CONCURRENT_NODES:
|
||||
return json.dumps(
|
||||
{"error": f"Too many nodes ({len(clean_ids)}), max is {_MAX_CONCURRENT_NODES}"}
|
||||
)
|
||||
|
||||
log.info("run_on_nodes nodes=%s cmd=%r", clean_ids, command)
|
||||
results = await _dispatch_parallel(
|
||||
redis_kw, clean_ids, command, _clamp_timeout(timeout), max_output
|
||||
)
|
||||
return json.dumps(results, indent=2)
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
async def run_on_all_nodes(
|
||||
command: str,
|
||||
ctx: Context[Any, Any, Any],
|
||||
timeout: int = _DEFAULT_TIMEOUT,
|
||||
) -> str:
|
||||
"""Execute a shell command on ALL active nodes in parallel.
|
||||
|
||||
Discovers nodes automatically, then dispatches in parallel. Useful for
|
||||
cluster-wide operations like checking disk usage, GPU status, or
|
||||
running processes.
|
||||
|
||||
Args:
|
||||
command: Shell command to execute on every node.
|
||||
timeout: Timeout per node in seconds (default: 120).
|
||||
"""
|
||||
cmd_err = _validate_command(command)
|
||||
if cmd_err:
|
||||
return json.dumps({"error": cmd_err})
|
||||
|
||||
redis_kw: dict[str, Any] = ctx.request_context.lifespan_context["redis_kwargs"]
|
||||
max_output = _DEFAULT_MAX_OUTPUT
|
||||
|
||||
nodes = await _list_nodes_impl(redis_kw)
|
||||
if not nodes:
|
||||
return json.dumps({"error": "No active nodes found in cluster"})
|
||||
|
||||
node_ids = list(
|
||||
dict.fromkeys(
|
||||
nid.strip() for n in nodes if (nid := n.get("node_id") or n.get("id")) and nid.strip()
|
||||
)
|
||||
)
|
||||
if not node_ids:
|
||||
return json.dumps({"error": "No nodes with identifiable IDs found"})
|
||||
if len(node_ids) > _MAX_CONCURRENT_NODES:
|
||||
return json.dumps(
|
||||
{"error": f"Too many nodes ({len(node_ids)}), max is {_MAX_CONCURRENT_NODES}"}
|
||||
)
|
||||
log.info("run_on_all_nodes nodes=%s cmd=%r", node_ids, command)
|
||||
results = await _dispatch_parallel(
|
||||
redis_kw, node_ids, command, _clamp_timeout(timeout), max_output
|
||||
)
|
||||
return json.dumps(results, indent=2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Run the MCP cluster-ops server via stdio transport."""
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
mcp.run(transport="stdio")
|
||||
@@ -0,0 +1,57 @@
|
||||
[build-system]
|
||||
requires = ["hatchling>=1.29"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "mcp-cluster-ops"
|
||||
version = "0.1.0"
|
||||
description = "MCP server for Turnstone cluster operations — reference implementation."
|
||||
requires-python = ">=3.11"
|
||||
license = "BUSL-1.1"
|
||||
dependencies = [
|
||||
"turnstone[mq]",
|
||||
"mcp>=1.6",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
mcp-cluster-ops = "mcp_cluster_ops.server:main"
|
||||
|
||||
[project.optional-dependencies]
|
||||
test = ["pytest>=9.0"]
|
||||
dev = ["ruff>=0.9", "mypy>=1.14", "types-redis>=4.6"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py311"
|
||||
line-length = 100
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "W", "I", "N", "UP", "B", "A", "SIM", "TCH"]
|
||||
ignore = ["E501"]
|
||||
|
||||
[tool.ruff.format]
|
||||
quote-style = "double"
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.11"
|
||||
strict = true
|
||||
warn_return_any = true
|
||||
warn_unused_configs = true
|
||||
disallow_untyped_defs = true
|
||||
disallow_incomplete_defs = true
|
||||
check_untyped_defs = true
|
||||
no_implicit_optional = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["mcp", "mcp.*"]
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = ["turnstone", "turnstone.*"]
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "tests.*"
|
||||
disallow_untyped_defs = false
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Tests for pure helper functions in mcp_cluster_ops.server."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from turnstone.mq.client import TurnResult
|
||||
|
||||
from mcp_cluster_ops.server import (
|
||||
_clamp_timeout,
|
||||
_exec_prompt,
|
||||
_extract_output,
|
||||
_format_node_result,
|
||||
_truncate,
|
||||
_validate_command,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _truncate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTruncate:
|
||||
def test_empty_string(self):
|
||||
assert _truncate("", 100) == ""
|
||||
|
||||
def test_under_limit(self):
|
||||
assert _truncate("hello", 100) == "hello"
|
||||
|
||||
def test_at_limit(self):
|
||||
text = "x" * 50
|
||||
assert _truncate(text, 50) == text
|
||||
|
||||
def test_over_limit(self):
|
||||
text = "x" * 200
|
||||
result = _truncate(text, 50)
|
||||
assert result.startswith("x" * 50)
|
||||
assert "truncated" in result
|
||||
assert "150 bytes omitted" in result
|
||||
|
||||
def test_unicode_boundary(self):
|
||||
# U+00E9 (é) is 2 bytes in UTF-8 (0xC3 0xA9), so 5 chars = 10 bytes
|
||||
text = "\u00e9\u00e9\u00e9\u00e9\u00e9"
|
||||
result = _truncate(text, 5)
|
||||
# Should not crash, should truncate cleanly
|
||||
assert "truncated" in result
|
||||
|
||||
def test_zero_disables(self):
|
||||
text = "x" * 10000
|
||||
assert _truncate(text, 0) == text
|
||||
|
||||
def test_custom_max(self):
|
||||
text = "abcdefghij" # 10 bytes
|
||||
result = _truncate(text, 5)
|
||||
assert result.startswith("abcde")
|
||||
assert "truncated" in result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _extract_output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractOutput:
|
||||
def test_bash_result_preferred(self):
|
||||
r = TurnResult(
|
||||
content_parts=["agent said something"],
|
||||
tool_results=[("bash", "raw output")],
|
||||
)
|
||||
assert _extract_output(r) == "raw output"
|
||||
|
||||
def test_multiple_bash_results_joined(self):
|
||||
r = TurnResult(
|
||||
tool_results=[("bash", "line1"), ("bash", "line2")],
|
||||
)
|
||||
assert _extract_output(r) == "line1\nline2"
|
||||
|
||||
def test_content_fallback(self):
|
||||
r = TurnResult(
|
||||
content_parts=["agent response"],
|
||||
tool_results=[("read_file", "file contents")],
|
||||
)
|
||||
assert _extract_output(r) == "agent response"
|
||||
|
||||
def test_any_tool_fallback(self):
|
||||
r = TurnResult(
|
||||
tool_results=[("read_file", "file contents")],
|
||||
)
|
||||
assert _extract_output(r) == "file contents"
|
||||
|
||||
def test_empty_result(self):
|
||||
r = TurnResult()
|
||||
assert _extract_output(r) == ""
|
||||
|
||||
def test_bash_preferred_over_content(self):
|
||||
r = TurnResult(
|
||||
content_parts=["I ran the command"],
|
||||
tool_results=[("read_file", "data"), ("bash", "output")],
|
||||
)
|
||||
assert _extract_output(r) == "output"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _exec_prompt
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExecPrompt:
|
||||
def test_contains_command(self):
|
||||
result = _exec_prompt("ls -la /tmp")
|
||||
assert "ls -la /tmp" in result
|
||||
|
||||
def test_suppression_instruction(self):
|
||||
result = _exec_prompt("echo hello")
|
||||
assert "Do NOT repeat" in result
|
||||
assert "ok" in result.lower() or "failed" in result.lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _format_node_result
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFormatNodeResult:
|
||||
def test_success(self):
|
||||
r = TurnResult(tool_results=[("bash", "output data")])
|
||||
fmt = _format_node_result("node-1", r, 8192)
|
||||
assert fmt["node"] == "node-1"
|
||||
assert fmt["ok"] is True
|
||||
assert fmt["output"] == "output data"
|
||||
assert "timed_out" not in fmt
|
||||
|
||||
def test_timeout(self):
|
||||
r = TurnResult(timed_out=True)
|
||||
fmt = _format_node_result("node-1", r, 8192)
|
||||
assert fmt["ok"] is False
|
||||
assert fmt["timed_out"] is True
|
||||
|
||||
def test_error(self):
|
||||
r = TurnResult(errors=["connection refused"])
|
||||
fmt = _format_node_result("node-1", r, 8192)
|
||||
assert fmt["ok"] is False
|
||||
assert fmt["error"] == "connection refused"
|
||||
|
||||
def test_truncation_applied(self):
|
||||
r = TurnResult(tool_results=[("bash", "x" * 200)])
|
||||
fmt = _format_node_result("node-1", r, 50)
|
||||
assert "truncated" in fmt["output"]
|
||||
|
||||
def test_unlimited_output(self):
|
||||
big = "x" * 100000
|
||||
r = TurnResult(tool_results=[("bash", big)])
|
||||
fmt = _format_node_result("node-1", r, 0)
|
||||
assert fmt["output"] == big
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _validate_command
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestValidateCommand:
|
||||
def test_valid(self):
|
||||
assert _validate_command("ls -la") is None
|
||||
|
||||
def test_empty(self):
|
||||
assert _validate_command("") is not None
|
||||
|
||||
def test_whitespace_only(self):
|
||||
assert _validate_command(" ") is not None
|
||||
|
||||
def test_too_long(self):
|
||||
err = _validate_command("x" * 100000)
|
||||
assert err is not None
|
||||
assert "too long" in err
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _clamp_timeout
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestClampTimeout:
|
||||
def test_normal(self):
|
||||
assert _clamp_timeout(60) == 60.0
|
||||
|
||||
def test_too_low(self):
|
||||
assert _clamp_timeout(1) == 5.0
|
||||
|
||||
def test_too_high(self):
|
||||
assert _clamp_timeout(99999) == 3600.0
|
||||
|
||||
def test_negative(self):
|
||||
assert _clamp_timeout(-1) == 5.0
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Tests for MCP tool handlers with mocked TurnstoneClient."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from turnstone.mq.client import TurnResult
|
||||
|
||||
from mcp_cluster_ops.server import (
|
||||
_dispatch_parallel,
|
||||
_exec_on_node_sync,
|
||||
_list_nodes_impl,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _list_nodes_impl
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestListNodesImpl:
|
||||
def test_returns_nodes(self):
|
||||
nodes = [{"node_id": "a", "model": "gpt-5"}, {"node_id": "b", "model": "gpt-5"}]
|
||||
with patch("mcp_cluster_ops.server.TurnstoneClient") as mock_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client.list_nodes.return_value = nodes
|
||||
mock_cls.return_value.__enter__ = MagicMock(return_value=mock_client)
|
||||
mock_cls.return_value.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
result = asyncio.run(_list_nodes_impl({"host": "localhost"}))
|
||||
assert result == nodes
|
||||
|
||||
def test_empty_cluster(self):
|
||||
with patch("mcp_cluster_ops.server.TurnstoneClient") as mock_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client.list_nodes.return_value = []
|
||||
mock_cls.return_value.__enter__ = MagicMock(return_value=mock_client)
|
||||
mock_cls.return_value.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
result = asyncio.run(_list_nodes_impl({"host": "localhost"}))
|
||||
assert result == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _exec_on_node_sync
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExecOnNodeSync:
|
||||
def test_success(self):
|
||||
turn_result = TurnResult(
|
||||
tool_results=[("bash", "hello world")],
|
||||
)
|
||||
with patch("mcp_cluster_ops.server.TurnstoneClient") as mock_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client.send_and_wait.return_value = turn_result
|
||||
mock_cls.return_value.__enter__ = MagicMock(return_value=mock_client)
|
||||
mock_cls.return_value.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
node_id, result = _exec_on_node_sync(
|
||||
{"host": "localhost"}, "node-1", "echo hello", 60.0
|
||||
)
|
||||
assert node_id == "node-1"
|
||||
assert result.ok
|
||||
mock_client.send_and_wait.assert_called_once()
|
||||
call_kwargs = mock_client.send_and_wait.call_args
|
||||
assert call_kwargs.kwargs["target_node"] == "node-1"
|
||||
assert call_kwargs.kwargs["auto_approve"] is True
|
||||
|
||||
def test_timeout(self):
|
||||
turn_result = TurnResult(timed_out=True)
|
||||
with patch("mcp_cluster_ops.server.TurnstoneClient") as mock_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client.send_and_wait.return_value = turn_result
|
||||
mock_cls.return_value.__enter__ = MagicMock(return_value=mock_client)
|
||||
mock_cls.return_value.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
_, result = _exec_on_node_sync({"host": "localhost"}, "node-1", "sleep 9999", 1.0)
|
||||
assert result.timed_out
|
||||
assert not result.ok
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _dispatch_parallel
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDispatchParallel:
|
||||
def test_parallel_success(self):
|
||||
def fake_exec(redis_kw: Any, node_id: str, command: str, timeout: float) -> Any:
|
||||
return (node_id, TurnResult(tool_results=[("bash", f"output-{node_id}")]))
|
||||
|
||||
with patch("mcp_cluster_ops.server._exec_on_node_sync", side_effect=fake_exec):
|
||||
results = asyncio.run(
|
||||
_dispatch_parallel(
|
||||
{"host": "localhost"},
|
||||
["a", "b", "c"],
|
||||
"echo hi",
|
||||
60.0,
|
||||
8192,
|
||||
)
|
||||
)
|
||||
assert len(results) == 3
|
||||
assert all(r["ok"] for r in results)
|
||||
outputs = {r["node"]: r["output"] for r in results}
|
||||
assert outputs["a"] == "output-a"
|
||||
assert outputs["b"] == "output-b"
|
||||
|
||||
def test_partial_failure(self):
|
||||
def fake_exec(redis_kw: Any, node_id: str, command: str, timeout: float) -> Any:
|
||||
if node_id == "bad":
|
||||
raise ConnectionError("Redis down")
|
||||
return (node_id, TurnResult(tool_results=[("bash", "ok")]))
|
||||
|
||||
with patch("mcp_cluster_ops.server._exec_on_node_sync", side_effect=fake_exec):
|
||||
results = asyncio.run(
|
||||
_dispatch_parallel(
|
||||
{"host": "localhost"},
|
||||
["good", "bad"],
|
||||
"echo hi",
|
||||
60.0,
|
||||
8192,
|
||||
)
|
||||
)
|
||||
assert len(results) == 2
|
||||
good = next(r for r in results if r["node"] == "good")
|
||||
bad = next(r for r in results if r["node"] == "bad")
|
||||
assert good["ok"] is True
|
||||
assert bad["ok"] is False
|
||||
assert "Redis down" in bad["error"]
|
||||
|
||||
def test_all_fail(self):
|
||||
def fake_exec(redis_kw: Any, node_id: str, command: str, timeout: float) -> Any:
|
||||
raise RuntimeError(f"fail-{node_id}")
|
||||
|
||||
with patch("mcp_cluster_ops.server._exec_on_node_sync", side_effect=fake_exec):
|
||||
results = asyncio.run(
|
||||
_dispatch_parallel(
|
||||
{"host": "localhost"},
|
||||
["a", "b"],
|
||||
"echo hi",
|
||||
60.0,
|
||||
8192,
|
||||
)
|
||||
)
|
||||
assert all(not r["ok"] for r in results)
|
||||
assert "fail-a" in results[0]["error"]
|
||||
assert "fail-b" in results[1]["error"]
|
||||
+6
-2
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "turnstone"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
description = "Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."
|
||||
readme = "README.md"
|
||||
license = "BUSL-1.1"
|
||||
@@ -51,7 +51,7 @@ sim = ["redis>=7.2"]
|
||||
anthropic = ["anthropic>=0.39"]
|
||||
postgres = ["psycopg[binary]>=3.2"]
|
||||
discord = ["discord.py>=2.4", "redis>=7.2"]
|
||||
|
||||
all = ["turnstone[mq,console,sim,anthropic,postgres,discord]"]
|
||||
|
||||
[project.scripts]
|
||||
turnstone = "turnstone.cli:main"
|
||||
@@ -76,6 +76,9 @@ include = [
|
||||
"turnstone/console/static/*.js",
|
||||
"turnstone/shared_static/*.css",
|
||||
"turnstone/shared_static/*.js",
|
||||
"turnstone/shared_static/katex-0.16.38/**/*",
|
||||
"turnstone/shared_static/hljs-11.11.1/**/*",
|
||||
"turnstone/shared_static/mermaid-11.13.0/**/*",
|
||||
"turnstone/sdk/py.typed",
|
||||
]
|
||||
|
||||
@@ -90,6 +93,7 @@ line-length = 100
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "W", "I", "N", "UP", "B", "A", "SIM", "TCH"]
|
||||
ignore = ["E501"]
|
||||
per-file-ignores = { "turnstone/core/sandbox.py" = ["N802"] }
|
||||
|
||||
[tool.ruff.format]
|
||||
quote-style = "double"
|
||||
|
||||
Executable
+137
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
# Update a vendored JavaScript library in turnstone/shared_static/.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/update-vendored-js.sh katex 0.16.39
|
||||
# scripts/update-vendored-js.sh hljs 11.12.0
|
||||
# scripts/update-vendored-js.sh mermaid 11.14.0
|
||||
#
|
||||
# This script:
|
||||
# 1. Downloads the new version from CDN
|
||||
# 2. Creates the new versioned directory
|
||||
# 3. Updates all version references in source files
|
||||
# 4. Removes the old versioned directory
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
STATIC_DIR="turnstone/shared_static"
|
||||
CDN="https://cdn.jsdelivr.net/npm"
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 <katex|hljs|mermaid> <version>"
|
||||
echo "Example: $0 katex 0.16.39"
|
||||
exit 1
|
||||
}
|
||||
|
||||
[[ $# -eq 2 ]] || usage
|
||||
|
||||
LIB="$1"
|
||||
VERSION="$2"
|
||||
|
||||
# Detect current version from pyproject.toml
|
||||
detect_old_version() {
|
||||
local pattern="$1"
|
||||
grep -oE "${pattern}-[0-9.]+" pyproject.toml | head -1 | sed "s/${pattern}-//"
|
||||
}
|
||||
|
||||
# Update version references across all source files
|
||||
update_refs() {
|
||||
local old_pattern="$1" # e.g. katex-0.16.38
|
||||
local new_pattern="$2" # e.g. katex-0.16.39
|
||||
|
||||
# Find all files with version references (excludes vendored JS and worktrees)
|
||||
local files
|
||||
files=$(grep -rl --include='*.toml' --include='*.html' --include='*.js' --include='*.md' \
|
||||
-F "$old_pattern" . \
|
||||
--exclude-dir='.claude' --exclude-dir='node_modules' --exclude-dir='shared_static' \
|
||||
2>/dev/null || true)
|
||||
for f in $files; do
|
||||
sed -i "s|${old_pattern}|${new_pattern}|g" "$f"
|
||||
echo " Updated $f"
|
||||
done
|
||||
}
|
||||
|
||||
case "$LIB" in
|
||||
katex)
|
||||
OLD_VERSION=$(detect_old_version "katex")
|
||||
OLD_DIR="${STATIC_DIR}/katex-${OLD_VERSION}"
|
||||
NEW_DIR="${STATIC_DIR}/katex-${VERSION}"
|
||||
|
||||
echo "Updating KaTeX ${OLD_VERSION} -> ${VERSION}"
|
||||
mkdir -p "${NEW_DIR}/fonts"
|
||||
|
||||
echo " Downloading katex.min.js..."
|
||||
curl -sSfL "${CDN}/katex@${VERSION}/dist/katex.min.js" -o "${NEW_DIR}/katex.min.js"
|
||||
echo " Downloading katex.min.css..."
|
||||
curl -sSfL "${CDN}/katex@${VERSION}/dist/katex.min.css" -o "${NEW_DIR}/katex.min.css"
|
||||
|
||||
echo " Downloading fonts..."
|
||||
# Extract font filenames from the CSS
|
||||
font_files=$(curl -sSfL "${CDN}/katex@${VERSION}/dist/katex.min.css" \
|
||||
| grep -oE 'fonts/[^")]+' | sort -u)
|
||||
for font in $font_files; do
|
||||
if ! curl -sSfL "${CDN}/katex@${VERSION}/dist/${font}" -o "${NEW_DIR}/${font}" 2>/dev/null; then
|
||||
echo " WARNING: Failed to download font: ${font}"
|
||||
fi
|
||||
done
|
||||
|
||||
# Copy LICENSE from old dir if present
|
||||
if [[ -f "${OLD_DIR}/LICENSE" ]]; then
|
||||
cp "${OLD_DIR}/LICENSE" "${NEW_DIR}/LICENSE"
|
||||
fi
|
||||
|
||||
update_refs "katex-${OLD_VERSION}" "katex-${VERSION}"
|
||||
rm -rf "${OLD_DIR}"
|
||||
echo "Done. Old directory removed: ${OLD_DIR}"
|
||||
;;
|
||||
|
||||
hljs)
|
||||
OLD_VERSION=$(detect_old_version "hljs")
|
||||
OLD_DIR="${STATIC_DIR}/hljs-${OLD_VERSION}"
|
||||
NEW_DIR="${STATIC_DIR}/hljs-${VERSION}"
|
||||
|
||||
echo "Updating Highlight.js ${OLD_VERSION} -> ${VERSION}"
|
||||
mkdir -p "${NEW_DIR}"
|
||||
|
||||
echo " Downloading highlight.min.js..."
|
||||
curl -sSfL "${CDN}/@highlightjs/cdn-assets@${VERSION}/highlight.min.js" -o "${NEW_DIR}/highlight.min.js"
|
||||
|
||||
if [[ -f "${OLD_DIR}/LICENSE" ]]; then
|
||||
cp "${OLD_DIR}/LICENSE" "${NEW_DIR}/LICENSE"
|
||||
fi
|
||||
|
||||
update_refs "hljs-${OLD_VERSION}" "hljs-${VERSION}"
|
||||
rm -rf "${OLD_DIR}"
|
||||
echo "Done. Old directory removed: ${OLD_DIR}"
|
||||
;;
|
||||
|
||||
mermaid)
|
||||
OLD_VERSION=$(detect_old_version "mermaid")
|
||||
OLD_DIR="${STATIC_DIR}/mermaid-${OLD_VERSION}"
|
||||
NEW_DIR="${STATIC_DIR}/mermaid-${VERSION}"
|
||||
|
||||
echo "Updating Mermaid ${OLD_VERSION} -> ${VERSION}"
|
||||
mkdir -p "${NEW_DIR}"
|
||||
|
||||
echo " Downloading mermaid.min.js..."
|
||||
curl -sSfL "${CDN}/mermaid@${VERSION}/dist/mermaid.min.js" -o "${NEW_DIR}/mermaid.min.js"
|
||||
|
||||
if [[ -f "${OLD_DIR}/LICENSE" ]]; then
|
||||
cp "${OLD_DIR}/LICENSE" "${NEW_DIR}/LICENSE"
|
||||
fi
|
||||
|
||||
update_refs "mermaid-${OLD_VERSION}" "mermaid-${VERSION}"
|
||||
rm -rf "${OLD_DIR}"
|
||||
echo "Done. Old directory removed: ${OLD_DIR}"
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Unknown library: ${LIB}"
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
echo "Verify the update:"
|
||||
echo " git diff --stat"
|
||||
echo " python -m turnstone.server # test locally"
|
||||
+5319
-35
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Generated
+773
-843
File diff suppressed because it is too large
Load Diff
@@ -33,6 +33,6 @@
|
||||
"license": "BUSL-1.1",
|
||||
"devDependencies": {
|
||||
"typescript": "^5.4",
|
||||
"vitest": "^2.0"
|
||||
"vitest": "^4.1"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import { BaseClient, type ClientOptions } from "./base.js";
|
||||
import type { ClusterEvent } from "./events.js";
|
||||
import type {
|
||||
AdminListMemoriesOptions,
|
||||
AdminMemoryInfo,
|
||||
AdminSearchMemoriesOptions,
|
||||
AuditQueryOptions,
|
||||
AuditResponse,
|
||||
AuthLoginResponse,
|
||||
@@ -13,25 +16,35 @@ import type {
|
||||
ConsoleCreateWsRequest,
|
||||
ConsoleCreateWsResponse,
|
||||
ConsoleHealthResponse,
|
||||
CreateMcpServerRequest,
|
||||
CreatePolicyOptions,
|
||||
CreateRoleOptions,
|
||||
CreateScheduleRequest,
|
||||
CreateTemplateOptions,
|
||||
CreateWsTemplateOptions,
|
||||
ImportMcpConfigResponse,
|
||||
ListAdminMemoriesResponse,
|
||||
ListMcpServersResponse,
|
||||
ListScheduleRunsResponse,
|
||||
ListSchedulesResponse,
|
||||
ListSettingSchemaResponse,
|
||||
ListSettingsResponse,
|
||||
McpServerDetail,
|
||||
NodeDetailResponse,
|
||||
NodesOptions,
|
||||
OrgInfo,
|
||||
PromptTemplateInfo,
|
||||
RoleInfo,
|
||||
ScheduleInfo,
|
||||
SettingInfo,
|
||||
StatusResponse,
|
||||
ToolPolicyInfo,
|
||||
UpdateMcpServerRequest,
|
||||
UpdateOrgOptions,
|
||||
UpdatePolicyOptions,
|
||||
UpdateRoleOptions,
|
||||
UpdateScheduleRequest,
|
||||
UpdateSettingOptions,
|
||||
UpdateTemplateOptions,
|
||||
UpdateWsTemplateOptions,
|
||||
UsageQueryOptions,
|
||||
@@ -343,4 +356,108 @@ export class TurnstoneConsole extends BaseClient {
|
||||
if (opts?.offset !== undefined) params.offset = String(opts.offset);
|
||||
return this.request("GET", "/v1/api/admin/audit", { params });
|
||||
}
|
||||
|
||||
// -- Admin: Memories ------------------------------------------------------
|
||||
|
||||
async listMemories(
|
||||
opts?: AdminListMemoriesOptions,
|
||||
): Promise<ListAdminMemoriesResponse> {
|
||||
const params: Record<string, string | number> = {};
|
||||
if (opts?.type) params.type = opts.type;
|
||||
if (opts?.scope) params.scope = opts.scope;
|
||||
if (opts?.scope_id) params.scope_id = opts.scope_id;
|
||||
if (opts?.limit !== undefined) params.limit = opts.limit;
|
||||
return this.request("GET", "/v1/api/admin/memories", { params });
|
||||
}
|
||||
|
||||
async searchMemories(
|
||||
opts: AdminSearchMemoriesOptions,
|
||||
): Promise<ListAdminMemoriesResponse> {
|
||||
const params: Record<string, string | number> = { q: opts.q };
|
||||
if (opts.type) params.type = opts.type;
|
||||
if (opts.scope) params.scope = opts.scope;
|
||||
if (opts.scope_id) params.scope_id = opts.scope_id;
|
||||
if (opts.limit !== undefined) params.limit = opts.limit;
|
||||
return this.request("GET", "/v1/api/admin/memories/search", { params });
|
||||
}
|
||||
|
||||
async getMemory(memoryId: string): Promise<AdminMemoryInfo> {
|
||||
return this.request("GET", `/v1/api/admin/memories/${memoryId}`);
|
||||
}
|
||||
|
||||
async deleteMemory(memoryId: string): Promise<StatusResponse> {
|
||||
return this.request("DELETE", `/v1/api/admin/memories/${memoryId}`);
|
||||
}
|
||||
|
||||
// -- System: Settings -------------------------------------------------------
|
||||
|
||||
async listSettings(): Promise<ListSettingsResponse> {
|
||||
return this.request("GET", "/v1/api/admin/settings");
|
||||
}
|
||||
|
||||
async getSettingsSchema(): Promise<ListSettingSchemaResponse> {
|
||||
return this.request("GET", "/v1/api/admin/settings/schema");
|
||||
}
|
||||
|
||||
async updateSetting(
|
||||
key: string,
|
||||
opts: UpdateSettingOptions,
|
||||
): Promise<SettingInfo> {
|
||||
return this.request("PUT", `/v1/api/admin/settings/${key}`, {
|
||||
json: opts,
|
||||
});
|
||||
}
|
||||
|
||||
async deleteSetting(key: string, nodeId?: string): Promise<StatusResponse> {
|
||||
const params: Record<string, string> = {};
|
||||
if (nodeId) params.node_id = nodeId;
|
||||
return this.request("DELETE", `/v1/api/admin/settings/${key}`, {
|
||||
params,
|
||||
});
|
||||
}
|
||||
|
||||
// -- MCP servers ----------------------------------------------------------
|
||||
|
||||
async listMcpServers(opts?: {
|
||||
reveal?: boolean;
|
||||
}): Promise<ListMcpServersResponse> {
|
||||
const params: Record<string, string> = {};
|
||||
if (opts?.reveal) params.reveal = "true";
|
||||
return this.request("GET", "/v1/api/admin/mcp-servers", { params });
|
||||
}
|
||||
|
||||
async createMcpServer(
|
||||
body: CreateMcpServerRequest,
|
||||
): Promise<McpServerDetail> {
|
||||
return this.request("POST", "/v1/api/admin/mcp-servers", { json: body });
|
||||
}
|
||||
|
||||
async getMcpServer(serverId: string): Promise<McpServerDetail> {
|
||||
return this.request("GET", `/v1/api/admin/mcp-servers/${serverId}`);
|
||||
}
|
||||
|
||||
async updateMcpServer(
|
||||
serverId: string,
|
||||
body: UpdateMcpServerRequest,
|
||||
): Promise<McpServerDetail> {
|
||||
return this.request("PUT", `/v1/api/admin/mcp-servers/${serverId}`, {
|
||||
json: body,
|
||||
});
|
||||
}
|
||||
|
||||
async deleteMcpServer(serverId: string): Promise<StatusResponse> {
|
||||
return this.request("DELETE", `/v1/api/admin/mcp-servers/${serverId}`);
|
||||
}
|
||||
|
||||
async reloadMcpServers(): Promise<StatusResponse> {
|
||||
return this.request("POST", "/v1/api/admin/mcp-servers/reload");
|
||||
}
|
||||
|
||||
async importMcpConfig(
|
||||
config: Record<string, unknown>,
|
||||
): Promise<ImportMcpConfigResponse> {
|
||||
return this.request("POST", "/v1/api/admin/mcp-servers/import", {
|
||||
json: { config },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,6 +115,8 @@ export interface WsStateEvent {
|
||||
context_ratio: number;
|
||||
activity: string;
|
||||
activity_state: string;
|
||||
/** Full assistant response text — populated on idle transitions only. */
|
||||
content?: string;
|
||||
}
|
||||
|
||||
export interface WsActivityEvent {
|
||||
|
||||
@@ -143,6 +143,30 @@ export type {
|
||||
SendAndWaitOptions,
|
||||
NodesOptions,
|
||||
WorkstreamsOptions,
|
||||
// Memory types
|
||||
SaveMemoryRequest,
|
||||
MemoryInfo,
|
||||
ListMemoriesResponse,
|
||||
SearchMemoriesRequest,
|
||||
ListMemoriesOptions,
|
||||
DeleteMemoryOptions,
|
||||
AdminMemoryInfo,
|
||||
ListAdminMemoriesResponse,
|
||||
AdminListMemoriesOptions,
|
||||
AdminSearchMemoriesOptions,
|
||||
// Settings types
|
||||
SettingInfo,
|
||||
ListSettingsResponse,
|
||||
SettingSchemaInfo,
|
||||
ListSettingSchemaResponse,
|
||||
UpdateSettingOptions,
|
||||
// MCP server types
|
||||
McpServerStatus,
|
||||
McpServerDetail,
|
||||
ListMcpServersResponse,
|
||||
CreateMcpServerRequest,
|
||||
UpdateMcpServerRequest,
|
||||
ImportMcpConfigResponse,
|
||||
} from "./types.js";
|
||||
|
||||
// SSE parser (for advanced usage)
|
||||
|
||||
@@ -7,9 +7,17 @@ import type {
|
||||
CreateWorkstreamRequest,
|
||||
CreateWorkstreamResponse,
|
||||
DashboardResponse,
|
||||
DeleteMemoryOptions,
|
||||
HealthResponse,
|
||||
ListMemoriesOptions,
|
||||
ListMemoriesResponse,
|
||||
ListPromptTemplateSummaryResponse,
|
||||
ListSavedWorkstreamsResponse,
|
||||
ListWsTemplateSummaryResponse,
|
||||
ListWorkstreamsResponse,
|
||||
MemoryInfo,
|
||||
SaveMemoryRequest,
|
||||
SearchMemoriesRequest,
|
||||
SendAndWaitOptions,
|
||||
SendResponse,
|
||||
StatusResponse,
|
||||
@@ -190,6 +198,49 @@ export class TurnstoneServer extends BaseClient {
|
||||
return this.request("GET", "/v1/api/workstreams/saved");
|
||||
}
|
||||
|
||||
// -- Templates --------------------------------------------------------------
|
||||
|
||||
async listTemplates(): Promise<ListPromptTemplateSummaryResponse> {
|
||||
return this.request("GET", "/v1/api/templates");
|
||||
}
|
||||
|
||||
async listWsTemplates(): Promise<ListWsTemplateSummaryResponse> {
|
||||
return this.request("GET", "/v1/api/ws-templates");
|
||||
}
|
||||
|
||||
// -- Memories -------------------------------------------------------------
|
||||
|
||||
async listMemories(
|
||||
opts?: ListMemoriesOptions,
|
||||
): Promise<ListMemoriesResponse> {
|
||||
const params: Record<string, string | number> = {};
|
||||
if (opts?.type) params.type = opts.type;
|
||||
if (opts?.scope) params.scope = opts.scope;
|
||||
if (opts?.scope_id) params.scope_id = opts.scope_id;
|
||||
if (opts?.limit !== undefined) params.limit = opts.limit;
|
||||
return this.request("GET", "/v1/api/memories", { params });
|
||||
}
|
||||
|
||||
async saveMemory(opts: SaveMemoryRequest): Promise<MemoryInfo> {
|
||||
return this.request("POST", "/v1/api/memories", { json: opts });
|
||||
}
|
||||
|
||||
async searchMemories(
|
||||
opts: SearchMemoriesRequest,
|
||||
): Promise<ListMemoriesResponse> {
|
||||
return this.request("POST", "/v1/api/memories/search", { json: opts });
|
||||
}
|
||||
|
||||
async deleteMemory(
|
||||
name: string,
|
||||
opts?: DeleteMemoryOptions,
|
||||
): Promise<StatusResponse> {
|
||||
const params: Record<string, string> = {};
|
||||
if (opts?.scope) params.scope = opts.scope;
|
||||
if (opts?.scope_id) params.scope_id = opts.scope_id;
|
||||
return this.request("DELETE", `/v1/api/memories/${name}`, { params });
|
||||
}
|
||||
|
||||
// -- Auth -----------------------------------------------------------------
|
||||
|
||||
async login(opts: {
|
||||
|
||||
@@ -143,6 +143,35 @@ export interface ListSavedWorkstreamsResponse {
|
||||
workstreams: SavedWorkstreamInfo[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Server API — Prompt templates
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface PromptTemplateSummary {
|
||||
name: string;
|
||||
category: string;
|
||||
is_default: boolean;
|
||||
origin: string;
|
||||
}
|
||||
|
||||
export interface ListPromptTemplateSummaryResponse {
|
||||
templates: PromptTemplateSummary[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Server API — Workstream templates
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface WsTemplateSummary {
|
||||
name: string;
|
||||
description: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface ListWsTemplateSummaryResponse {
|
||||
ws_templates: WsTemplateSummary[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Server API — Health
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -644,5 +673,192 @@ export interface WorkstreamsOptions {
|
||||
per_page?: number;
|
||||
}
|
||||
|
||||
// -- Server API: Memories ---------------------------------------------------
|
||||
|
||||
export interface SaveMemoryRequest {
|
||||
name: string;
|
||||
content: string;
|
||||
description?: string;
|
||||
type?: "user" | "project" | "feedback" | "reference";
|
||||
scope?: "global" | "workstream" | "user";
|
||||
scope_id?: string;
|
||||
}
|
||||
|
||||
export interface MemoryInfo {
|
||||
memory_id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
type: string;
|
||||
scope: string;
|
||||
scope_id: string;
|
||||
content: string;
|
||||
created: string;
|
||||
updated: string;
|
||||
}
|
||||
|
||||
export interface ListMemoriesResponse {
|
||||
memories: MemoryInfo[];
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SearchMemoriesRequest {
|
||||
query: string;
|
||||
type?: string;
|
||||
scope?: string;
|
||||
scope_id?: string;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface ListMemoriesOptions {
|
||||
type?: string;
|
||||
scope?: string;
|
||||
scope_id?: string;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface DeleteMemoryOptions {
|
||||
scope?: string;
|
||||
scope_id?: string;
|
||||
}
|
||||
|
||||
// -- Console API: Admin Memories --------------------------------------------
|
||||
|
||||
export interface AdminMemoryInfo {
|
||||
memory_id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
type: string;
|
||||
scope: string;
|
||||
scope_id: string;
|
||||
content: string;
|
||||
created: string;
|
||||
updated: string;
|
||||
last_accessed: string;
|
||||
access_count: number;
|
||||
}
|
||||
|
||||
export interface ListAdminMemoriesResponse {
|
||||
memories: AdminMemoryInfo[];
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface AdminListMemoriesOptions {
|
||||
type?: string;
|
||||
scope?: string;
|
||||
scope_id?: string;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface AdminSearchMemoriesOptions {
|
||||
q: string;
|
||||
type?: string;
|
||||
scope?: string;
|
||||
scope_id?: string;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
// -- Console API: MCP Servers -----------------------------------------------
|
||||
|
||||
export interface McpServerStatus {
|
||||
connected: boolean;
|
||||
tools: number;
|
||||
resources: number;
|
||||
prompts: number;
|
||||
error: string;
|
||||
}
|
||||
|
||||
export interface McpServerDetail {
|
||||
server_id: string;
|
||||
name: string;
|
||||
transport: string;
|
||||
command: string;
|
||||
args: string;
|
||||
url: string;
|
||||
headers: string;
|
||||
env: string;
|
||||
auto_approve: boolean;
|
||||
enabled: boolean;
|
||||
created_by: string;
|
||||
created: string;
|
||||
updated: string;
|
||||
status: Record<string, McpServerStatus>;
|
||||
}
|
||||
|
||||
export interface ListMcpServersResponse {
|
||||
servers: McpServerDetail[];
|
||||
}
|
||||
|
||||
export interface CreateMcpServerRequest {
|
||||
name: string;
|
||||
transport: string;
|
||||
command?: string;
|
||||
args?: string[];
|
||||
url?: string;
|
||||
headers?: Record<string, string>;
|
||||
env?: Record<string, string>;
|
||||
auto_approve?: boolean;
|
||||
enabled?: boolean;
|
||||
}
|
||||
|
||||
export interface UpdateMcpServerRequest {
|
||||
name?: string;
|
||||
transport?: string;
|
||||
command?: string;
|
||||
args?: string[];
|
||||
url?: string;
|
||||
headers?: Record<string, string>;
|
||||
env?: Record<string, string>;
|
||||
auto_approve?: boolean;
|
||||
enabled?: boolean;
|
||||
}
|
||||
|
||||
export interface ImportMcpConfigResponse {
|
||||
imported: string[];
|
||||
skipped: string[];
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
// -- Console API: System Settings -------------------------------------------
|
||||
|
||||
export interface SettingInfo {
|
||||
key: string;
|
||||
value: unknown;
|
||||
source: string;
|
||||
type: string;
|
||||
description: string;
|
||||
section: string;
|
||||
is_secret: boolean;
|
||||
node_id: string;
|
||||
changed_by: string;
|
||||
updated: string;
|
||||
restart_required: boolean;
|
||||
}
|
||||
|
||||
export interface ListSettingsResponse {
|
||||
settings: SettingInfo[];
|
||||
}
|
||||
|
||||
export interface SettingSchemaInfo {
|
||||
key: string;
|
||||
type: string;
|
||||
default: unknown;
|
||||
description: string;
|
||||
section: string;
|
||||
is_secret: boolean;
|
||||
min_value: number | null;
|
||||
max_value: number | null;
|
||||
choices: string[] | null;
|
||||
restart_required: boolean;
|
||||
}
|
||||
|
||||
export interface ListSettingSchemaResponse {
|
||||
schema: SettingSchemaInfo[];
|
||||
}
|
||||
|
||||
export interface UpdateSettingOptions {
|
||||
value: unknown;
|
||||
node_id?: string;
|
||||
}
|
||||
|
||||
// Re-export event types for convenience
|
||||
export type { ServerEvent, ClusterEvent } from "./events.js";
|
||||
|
||||
@@ -143,6 +143,61 @@ class TestRequiredScope:
|
||||
def test_proxy_v1_read_endpoint_needs_read(self):
|
||||
assert required_scope("GET", "/node/node-a/v1/api/workstreams") == "read"
|
||||
|
||||
# Memory endpoints
|
||||
def test_get_memories_needs_read(self):
|
||||
assert required_scope("GET", "/api/memories") == "read"
|
||||
|
||||
def test_post_memories_needs_write(self):
|
||||
assert required_scope("POST", "/api/memories") == "write"
|
||||
|
||||
def test_post_memories_search_needs_read(self):
|
||||
"""Search via POST is non-mutating — requires only read scope."""
|
||||
assert required_scope("POST", "/api/memories/search") == "read"
|
||||
|
||||
def test_delete_memory_needs_write(self):
|
||||
assert required_scope("DELETE", "/api/memories/my_key") == "write"
|
||||
|
||||
def test_v1_post_memories_needs_write(self):
|
||||
assert required_scope("POST", "/v1/api/memories") == "write"
|
||||
|
||||
def test_v1_delete_memory_needs_write(self):
|
||||
assert required_scope("DELETE", "/v1/api/memories/test_key") == "write"
|
||||
|
||||
def test_admin_memories_needs_approve(self):
|
||||
assert required_scope("GET", "/api/admin/memories") == "approve"
|
||||
|
||||
def test_admin_memory_delete_needs_approve(self):
|
||||
assert required_scope("DELETE", "/api/admin/memories/some-id") == "approve"
|
||||
|
||||
# Internal endpoints
|
||||
def test_internal_mcp_reload_needs_approve(self):
|
||||
assert required_scope("POST", "/api/_internal/mcp-reload") == "approve"
|
||||
|
||||
def test_v1_internal_mcp_reload_needs_approve(self):
|
||||
assert required_scope("POST", "/v1/api/_internal/mcp-reload") == "approve"
|
||||
|
||||
def test_internal_config_reload_needs_approve(self):
|
||||
assert required_scope("POST", "/api/_internal/config-reload") == "approve"
|
||||
|
||||
def test_v1_internal_config_reload_needs_approve(self):
|
||||
assert required_scope("POST", "/v1/api/_internal/config-reload") == "approve"
|
||||
|
||||
def test_proxy_internal_config_reload_needs_approve(self):
|
||||
assert required_scope("POST", "/node/n1/v1/api/_internal/config-reload") == "approve"
|
||||
|
||||
def test_proxy_no_v1_internal_config_reload_needs_approve(self):
|
||||
assert required_scope("POST", "/node/n1/api/_internal/config-reload") == "approve"
|
||||
|
||||
def test_proxy_internal_mcp_reload_needs_approve(self):
|
||||
assert required_scope("POST", "/node/n1/v1/api/_internal/mcp-reload") == "approve"
|
||||
|
||||
def test_proxy_no_v1_internal_mcp_reload_needs_approve(self):
|
||||
assert required_scope("POST", "/node/n1/api/_internal/mcp-reload") == "approve"
|
||||
|
||||
def test_get_internal_mcp_reload_needs_read(self):
|
||||
"""Only POST is elevated — GET falls through to read."""
|
||||
assert required_scope("GET", "/api/_internal/mcp-reload") == "read"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestAuthConfig
|
||||
@@ -1349,3 +1404,49 @@ class TestCorsConfigurable:
|
||||
)
|
||||
assert resp.headers.get("Access-Control-Allow-Origin") == "http://example.com"
|
||||
client.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestVerifyPassword — OIDC sentinel handling
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVerifyPassword:
|
||||
def test_valid_bcrypt_hash(self):
|
||||
from turnstone.core.auth import hash_password, verify_password
|
||||
|
||||
hashed = hash_password("mypassword")
|
||||
assert verify_password("mypassword", hashed) is True
|
||||
assert verify_password("wrongpassword", hashed) is False
|
||||
|
||||
def test_oidc_sentinel_rejected(self):
|
||||
from turnstone.core.auth import verify_password
|
||||
|
||||
# OIDC sentinel must return False, not crash with ValueError
|
||||
assert verify_password("anypassword", "!oidc") is False
|
||||
|
||||
def test_non_bcrypt_hash_rejected(self):
|
||||
from turnstone.core.auth import verify_password
|
||||
|
||||
assert verify_password("password", "not_a_hash") is False
|
||||
assert verify_password("password", "") is False
|
||||
|
||||
def test_empty_password_against_oidc_sentinel(self):
|
||||
from turnstone.core.auth import verify_password
|
||||
|
||||
assert verify_password("", "!oidc") is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestOIDCPublicPaths — OIDC endpoints are public
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOIDCPublicPaths:
|
||||
def test_oidc_authorize_is_public(self):
|
||||
assert is_public_path("/api/auth/oidc/authorize") is True
|
||||
assert is_public_path("/v1/api/auth/oidc/authorize") is True
|
||||
|
||||
def test_oidc_callback_is_public(self):
|
||||
assert is_public_path("/api/auth/oidc/callback") is True
|
||||
assert is_public_path("/v1/api/auth/oidc/callback") is True
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Tests for turnstone.core.bm25 — tokenizer and BM25 index."""
|
||||
|
||||
from turnstone.core.bm25 import BM25Index, _tokenize
|
||||
|
||||
|
||||
class TestTokenize:
|
||||
def test_simple_words(self):
|
||||
assert _tokenize("hello world") == ["hello", "world"]
|
||||
|
||||
def test_underscores(self):
|
||||
assert _tokenize("read_file") == ["read", "file"]
|
||||
|
||||
def test_hyphens(self):
|
||||
assert _tokenize("web-search") == ["web", "search"]
|
||||
|
||||
def test_dots(self):
|
||||
assert _tokenize("foo.bar.baz") == ["foo", "bar", "baz"]
|
||||
|
||||
def test_mixed_separators(self):
|
||||
assert _tokenize("mcp__server__read_file") == ["mcp", "server", "read", "file"]
|
||||
|
||||
def test_empty_string(self):
|
||||
assert _tokenize("") == []
|
||||
|
||||
def test_case_folding(self):
|
||||
assert _tokenize("Hello World") == ["hello", "world"]
|
||||
|
||||
|
||||
class TestBM25Index:
|
||||
def test_search_returns_relevant(self):
|
||||
docs = ["read a file from disk", "search for file in directory", "execute a bash command"]
|
||||
index = BM25Index(docs)
|
||||
results = index.search("file", k=2)
|
||||
assert 0 in results
|
||||
assert 1 in results
|
||||
|
||||
def test_search_empty_query(self):
|
||||
docs = ["hello world"]
|
||||
index = BM25Index(docs)
|
||||
assert index.search("") == []
|
||||
|
||||
def test_search_no_match(self):
|
||||
docs = ["hello world", "foo bar"]
|
||||
index = BM25Index(docs)
|
||||
assert index.search("zzzznotfound") == []
|
||||
|
||||
def test_search_respects_k(self):
|
||||
docs = [f"document {i} with common word" for i in range(20)]
|
||||
index = BM25Index(docs)
|
||||
results = index.search("common", k=3)
|
||||
assert len(results) <= 3
|
||||
|
||||
def test_empty_corpus(self):
|
||||
index = BM25Index([])
|
||||
assert index.search("anything") == []
|
||||
|
||||
def test_single_document(self):
|
||||
index = BM25Index(["the only document about turnstone"])
|
||||
results = index.search("turnstone")
|
||||
assert results == [0]
|
||||
|
||||
def test_ordering_by_relevance(self):
|
||||
docs = [
|
||||
"unrelated content about cooking recipes",
|
||||
"python programming with file operations",
|
||||
"read file write file file operations disk io",
|
||||
]
|
||||
index = BM25Index(docs)
|
||||
results = index.search("file operations", k=3)
|
||||
# Doc 2 has more file/operations mentions, should rank higher
|
||||
assert results[0] == 2
|
||||
@@ -3,7 +3,7 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from turnstone.mq.bridge import Bridge
|
||||
from turnstone.mq.protocol import StateChangeEvent, TurnCompleteEvent
|
||||
from turnstone.mq.protocol import ContentEvent, StateChangeEvent, TurnCompleteEvent
|
||||
|
||||
|
||||
def _make_bridge():
|
||||
@@ -67,3 +67,54 @@ class TestIdleTurnComplete:
|
||||
assert len(state_changes) == 1
|
||||
assert state_changes[0].state == "thinking"
|
||||
assert len(turn_completes) == 0
|
||||
|
||||
|
||||
class TestContentPassthrough:
|
||||
"""Bridge should pass through content from the server's idle SSE event."""
|
||||
|
||||
def test_content_passed_through_in_turn_complete(self):
|
||||
"""Content from idle event should be included in TurnCompleteEvent."""
|
||||
bridge = _make_bridge()
|
||||
|
||||
published = []
|
||||
with patch.object(
|
||||
bridge, "_publish_ws", side_effect=lambda ws, ev: published.append((ws, ev))
|
||||
):
|
||||
bridge._handle_global_event(
|
||||
{"type": "ws_state", "ws_id": "ws-1", "state": "idle", "content": "Hello world"}
|
||||
)
|
||||
|
||||
turn_completes = [(ws, ev) for ws, ev in published if isinstance(ev, TurnCompleteEvent)]
|
||||
assert len(turn_completes) == 1
|
||||
_, ev = turn_completes[0]
|
||||
assert ev.content == "Hello world"
|
||||
|
||||
def test_content_empty_when_not_in_event(self):
|
||||
"""TurnCompleteEvent.content should be empty when idle event has no content."""
|
||||
bridge = _make_bridge()
|
||||
|
||||
published = []
|
||||
with patch.object(
|
||||
bridge, "_publish_ws", side_effect=lambda ws, ev: published.append((ws, ev))
|
||||
):
|
||||
bridge._handle_global_event({"type": "ws_state", "ws_id": "ws-1", "state": "idle"})
|
||||
|
||||
turn_completes = [(ws, ev) for ws, ev in published if isinstance(ev, TurnCompleteEvent)]
|
||||
assert len(turn_completes) == 1
|
||||
_, ev = turn_completes[0]
|
||||
assert ev.content == ""
|
||||
|
||||
def test_content_event_still_published(self):
|
||||
"""Content events should still be published to per-ws channel."""
|
||||
bridge = _make_bridge()
|
||||
|
||||
published = []
|
||||
with patch.object(
|
||||
bridge, "_publish_ws", side_effect=lambda ws, ev: published.append((ws, ev))
|
||||
):
|
||||
bridge._handle_ws_event("ws-1", {"type": "content", "text": "hello"})
|
||||
|
||||
content_events = [(ws, ev) for ws, ev in published if isinstance(ev, ContentEvent)]
|
||||
assert len(content_events) == 1
|
||||
_, ev = content_events[0]
|
||||
assert ev.text == "hello"
|
||||
|
||||
@@ -21,7 +21,7 @@ def _run(coro):
|
||||
return asyncio.run(coro)
|
||||
|
||||
|
||||
def _make_message(*, bot=False, guild=True, content="hello", channel=None):
|
||||
def _make_message(*, bot=False, guild=True, content="hello", channel=None, reference=None):
|
||||
"""Build a mock ``discord.Message``."""
|
||||
msg = MagicMock(spec=discord.Message)
|
||||
msg.author = MagicMock()
|
||||
@@ -31,6 +31,7 @@ def _make_message(*, bot=False, guild=True, content="hello", channel=None):
|
||||
msg.guild = MagicMock() if guild else None
|
||||
msg.channel = channel or MagicMock()
|
||||
msg.mentions = []
|
||||
msg.reference = reference
|
||||
return msg
|
||||
|
||||
|
||||
@@ -204,6 +205,8 @@ class TestMessageCog:
|
||||
ts.router.send_message = AsyncMock()
|
||||
ts.config = MagicMock()
|
||||
ts._ws_tasks = {}
|
||||
ts._notify_ws_map = {}
|
||||
ts._notify_reply_channels = {}
|
||||
bot.turnstone = ts
|
||||
|
||||
cog = MessageCog(bot)
|
||||
@@ -328,6 +331,7 @@ class TestWsEventFinalization:
|
||||
bot.config.auto_approve_tools = []
|
||||
bot._streaming = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
|
||||
# Use the real _on_ws_event method
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
@@ -356,6 +360,7 @@ class TestWsEventFinalization:
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot._streaming = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
|
||||
thread = AsyncMock()
|
||||
@@ -387,6 +392,7 @@ class TestApprovalVerdictDisplay:
|
||||
bot.config.auto_approve_tools = []
|
||||
bot._streaming = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._should_auto_approve = MagicMock(return_value=False)
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
@@ -504,6 +510,7 @@ class TestApprovalVerdictDisplay:
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot._streaming = {}
|
||||
bot._pending_approval_msgs = {"ws-1": MagicMock()}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
|
||||
thread = AsyncMock()
|
||||
@@ -513,6 +520,315 @@ class TestApprovalVerdictDisplay:
|
||||
assert "ws-1" not in bot._pending_approval_msgs
|
||||
|
||||
|
||||
class TestContentCatchup:
|
||||
"""TurnCompleteEvent with content field provides catch-up for missed ContentEvents."""
|
||||
|
||||
def _make_bot(self):
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
bot.config.max_message_length = 2000
|
||||
bot.config.streaming_edit_interval = 1.5
|
||||
bot.config.auto_approve = False
|
||||
bot.config.auto_approve_tools = []
|
||||
bot._streaming = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_reply_channels = {}
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
return bot
|
||||
|
||||
def test_catchup_sends_content_when_no_streaming(self):
|
||||
"""TurnCompleteEvent with content but no SM sends catch-up message."""
|
||||
from turnstone.mq.protocol import TurnCompleteEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
raw = TurnCompleteEvent(
|
||||
ws_id="ws-1", correlation_id="", content="Caught up response"
|
||||
).to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, raw))
|
||||
|
||||
thread.send.assert_awaited_once_with("Caught up response")
|
||||
|
||||
def test_catchup_skipped_when_streaming_exists(self):
|
||||
"""TurnCompleteEvent with content and existing SM uses SM finalize, not catch-up."""
|
||||
from turnstone.mq.protocol import ContentEvent, TurnCompleteEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
# Feed content event to create SM
|
||||
content_raw = ContentEvent(ws_id="ws-1", text="Streamed").to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, content_raw))
|
||||
assert "ws-1" in bot._streaming
|
||||
|
||||
# Now TurnCompleteEvent with content — SM should be finalized, not catch-up
|
||||
complete_raw = TurnCompleteEvent(
|
||||
ws_id="ws-1", correlation_id="", content="Streamed"
|
||||
).to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, complete_raw))
|
||||
assert "ws-1" not in bot._streaming
|
||||
|
||||
def test_catchup_empty_content_no_message(self):
|
||||
"""TurnCompleteEvent with empty content and no SM sends nothing."""
|
||||
from turnstone.mq.protocol import TurnCompleteEvent
|
||||
|
||||
bot = self._make_bot()
|
||||
thread = AsyncMock()
|
||||
|
||||
raw = TurnCompleteEvent(ws_id="ws-1", correlation_id="", content="").to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, raw))
|
||||
|
||||
thread.send.assert_not_awaited()
|
||||
|
||||
|
||||
class TestNotificationTracking:
|
||||
"""Tests for notification message tracking and DM reply routing."""
|
||||
|
||||
def test_send_notification_tracks_message(self):
|
||||
"""send_notification should store message_id -> (ws_id, target_user) mapping."""
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot._notify_ws_map = {}
|
||||
bot._MAX_NOTIFY_TRACKING = 100
|
||||
bot.send = AsyncMock(return_value="12345")
|
||||
bot.send_notification = TurnstoneBot.send_notification.__get__(bot, TurnstoneBot)
|
||||
bot._track_notification = TurnstoneBot._track_notification.__get__(bot, TurnstoneBot)
|
||||
|
||||
_run(bot.send_notification("chan-1", "Hello", "ws-abc"))
|
||||
|
||||
assert 12345 in bot._notify_ws_map
|
||||
assert bot._notify_ws_map[12345] == ("ws-abc", "chan-1")
|
||||
|
||||
def test_send_notification_evicts_old_entries(self):
|
||||
"""Oldest notification tracking entries are evicted when cap is reached."""
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot._MAX_NOTIFY_TRACKING = 3
|
||||
bot._notify_ws_map = {
|
||||
1: ("ws-1", "u1"),
|
||||
2: ("ws-2", "u2"),
|
||||
3: ("ws-3", "u3"),
|
||||
}
|
||||
bot.send = AsyncMock(return_value="4")
|
||||
bot.send_notification = TurnstoneBot.send_notification.__get__(bot, TurnstoneBot)
|
||||
bot._track_notification = TurnstoneBot._track_notification.__get__(bot, TurnstoneBot)
|
||||
|
||||
_run(bot.send_notification("chan-1", "Hello", "ws-4"))
|
||||
|
||||
assert 4 in bot._notify_ws_map
|
||||
assert 1 not in bot._notify_ws_map # oldest evicted
|
||||
assert len(bot._notify_ws_map) <= 3
|
||||
|
||||
def test_dm_reply_routes_to_workstream(self):
|
||||
"""DM reply to a tracked notification routes the message to the workstream."""
|
||||
from turnstone.channels.discord.cog import MessageCog
|
||||
|
||||
bot = MagicMock()
|
||||
bot.user = MagicMock()
|
||||
bot.user.id = 99999
|
||||
|
||||
ts = MagicMock()
|
||||
ts._is_allowed_channel = MagicMock(return_value=True)
|
||||
ts.storage = MagicMock()
|
||||
ts.router = MagicMock()
|
||||
ts.router.resolve_user = AsyncMock(return_value="u_abc")
|
||||
ts.router.send_message = AsyncMock()
|
||||
ts.config = MagicMock()
|
||||
# Maps message_id -> (ws_id, target_discord_user_id)
|
||||
ts._notify_ws_map = {77777: ("ws-target", "12345")}
|
||||
ts._notify_reply_channels = {}
|
||||
bot.turnstone = ts
|
||||
|
||||
cog = MessageCog(bot)
|
||||
|
||||
# Build a DM reply to the tracked notification message
|
||||
ref = MagicMock()
|
||||
ref.message_id = 77777
|
||||
msg = _make_message(guild=False, content="additional context", reference=ref)
|
||||
# msg.author.id defaults to 12345 from _make_message
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_awaited_once_with("ws-target", "additional context")
|
||||
assert "ws-target" in ts._notify_reply_channels
|
||||
dm_chan, target_uid = ts._notify_reply_channels["ws-target"]
|
||||
assert target_uid == "12345"
|
||||
assert 77777 not in ts._notify_ws_map # cleaned up
|
||||
|
||||
def test_dm_reply_user_mismatch_rejected_and_preserved(self):
|
||||
"""DM reply from wrong user is rejected; entry re-inserted for legitimate user."""
|
||||
from turnstone.channels.discord.cog import MessageCog
|
||||
|
||||
bot = MagicMock()
|
||||
bot.user = MagicMock()
|
||||
bot.user.id = 99999
|
||||
|
||||
ts = MagicMock()
|
||||
ts.router = MagicMock()
|
||||
ts.router.resolve_user = AsyncMock(return_value="u_abc")
|
||||
ts.router.send_message = AsyncMock()
|
||||
# Target user is "99999" but replying user has author.id = 12345
|
||||
ts._notify_ws_map = {77777: ("ws-target", "99999")}
|
||||
ts._notify_reply_channels = {}
|
||||
bot.turnstone = ts
|
||||
|
||||
cog = MessageCog(bot)
|
||||
ref = MagicMock()
|
||||
ref.message_id = 77777
|
||||
msg = _make_message(guild=False, content="impostor", reference=ref)
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
# Entry should be re-inserted so the legitimate user can still reply.
|
||||
assert 77777 in ts._notify_ws_map
|
||||
assert ts._notify_ws_map[77777] == ("ws-target", "99999")
|
||||
|
||||
def test_dm_reply_stale_notification_feedback(self):
|
||||
"""DM reply to an expired/unknown notification should inform the user."""
|
||||
from turnstone.channels.discord.cog import MessageCog
|
||||
|
||||
bot = MagicMock()
|
||||
bot.user = MagicMock()
|
||||
bot.user.id = 99999
|
||||
|
||||
ts = MagicMock()
|
||||
ts.router = MagicMock()
|
||||
ts.router.send_message = AsyncMock()
|
||||
ts._notify_ws_map = {} # empty — no tracked notifications
|
||||
ts._notify_reply_channels = {}
|
||||
bot.turnstone = ts
|
||||
|
||||
cog = MessageCog(bot)
|
||||
|
||||
ref = MagicMock()
|
||||
ref.message_id = 99999 # not in map
|
||||
dm_channel = AsyncMock()
|
||||
msg = _make_message(guild=False, content="reply", reference=ref, channel=dm_channel)
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
# Should NOT route to any workstream
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
# Should send feedback to the DM channel
|
||||
dm_channel.send.assert_awaited_once_with("*This notification is no longer active.*")
|
||||
|
||||
def test_dm_without_reference_ignored(self):
|
||||
"""DM without a message reference should be ignored."""
|
||||
from turnstone.channels.discord.cog import MessageCog
|
||||
|
||||
bot = MagicMock()
|
||||
bot.user = MagicMock()
|
||||
bot.user.id = 99999
|
||||
|
||||
ts = MagicMock()
|
||||
ts.router = MagicMock()
|
||||
ts.router.send_message = AsyncMock()
|
||||
ts._notify_ws_map = {77777: ("ws-target", "12345")}
|
||||
ts._notify_reply_channels = {}
|
||||
bot.turnstone = ts
|
||||
|
||||
cog = MessageCog(bot)
|
||||
msg = _make_message(guild=False) # reference=None
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
|
||||
def test_dm_reply_unlinked_user_ignored(self):
|
||||
"""DM reply from an unlinked user should be ignored."""
|
||||
from turnstone.channels.discord.cog import MessageCog
|
||||
|
||||
bot = MagicMock()
|
||||
bot.user = MagicMock()
|
||||
bot.user.id = 99999
|
||||
|
||||
ts = MagicMock()
|
||||
ts.router = MagicMock()
|
||||
ts.router.resolve_user = AsyncMock(return_value=None)
|
||||
ts.router.send_message = AsyncMock()
|
||||
ts._notify_ws_map = {77777: ("ws-target", "12345")}
|
||||
ts._notify_reply_channels = {}
|
||||
bot.turnstone = ts
|
||||
|
||||
cog = MessageCog(bot)
|
||||
|
||||
ref = MagicMock()
|
||||
ref.message_id = 77777
|
||||
msg = _make_message(guild=False, content="reply", reference=ref)
|
||||
|
||||
_run(cog._on_message(msg))
|
||||
|
||||
ts.router.send_message.assert_not_awaited()
|
||||
|
||||
def test_turn_complete_forwards_to_dm(self):
|
||||
"""TurnCompleteEvent should forward content to notification reply DM."""
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
from turnstone.mq.protocol import TurnCompleteEvent
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot.config = MagicMock()
|
||||
bot.config.max_message_length = 2000
|
||||
bot._streaming = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_ws_map = {}
|
||||
bot._MAX_NOTIFY_TRACKING = 100
|
||||
|
||||
dm_channel = AsyncMock()
|
||||
sent_msg = MagicMock()
|
||||
sent_msg.id = 88888
|
||||
dm_channel.send = AsyncMock(return_value=sent_msg)
|
||||
bot._notify_reply_channels = {"ws-1": (dm_channel, "u123")}
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
bot._track_notification = TurnstoneBot._track_notification.__get__(bot, TurnstoneBot)
|
||||
|
||||
thread = AsyncMock()
|
||||
|
||||
raw = TurnCompleteEvent(
|
||||
ws_id="ws-1", correlation_id="", content="Here's the response"
|
||||
).to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, raw))
|
||||
|
||||
# Should send to DM channel
|
||||
dm_channel.send.assert_awaited_once_with("Here's the response")
|
||||
# Should clean up forwarding
|
||||
assert "ws-1" not in bot._notify_reply_channels
|
||||
# Response message should be tracked for multi-turn replies
|
||||
assert 88888 in bot._notify_ws_map
|
||||
assert bot._notify_ws_map[88888] == ("ws-1", "u123")
|
||||
|
||||
def test_turn_complete_cleans_up_dm_even_without_content(self):
|
||||
"""TurnCompleteEvent without content should still clean up DM tracking."""
|
||||
from turnstone.channels.discord.bot import TurnstoneBot
|
||||
from turnstone.mq.protocol import TurnCompleteEvent
|
||||
|
||||
bot = MagicMock(spec=TurnstoneBot)
|
||||
bot._streaming = {}
|
||||
bot._pending_approval_msgs = {}
|
||||
bot._notify_ws_map = {}
|
||||
|
||||
dm_channel = AsyncMock()
|
||||
bot._notify_reply_channels = {"ws-1": (dm_channel, "u123")}
|
||||
bot._on_ws_event = TurnstoneBot._on_ws_event.__get__(bot, TurnstoneBot)
|
||||
|
||||
thread = AsyncMock()
|
||||
|
||||
raw = TurnCompleteEvent(ws_id="ws-1", correlation_id="", content="").to_json()
|
||||
_run(bot._on_ws_event("ws-1", thread, raw))
|
||||
|
||||
# DM should not be sent to (no content)
|
||||
dm_channel.send.assert_not_awaited()
|
||||
# But should still be cleaned up
|
||||
assert "ws-1" not in bot._notify_reply_channels
|
||||
# No response tracked (nothing was sent)
|
||||
assert len(bot._notify_ws_map) == 0
|
||||
|
||||
|
||||
class TestChannelCLI:
|
||||
"""Tests for the channel CLI entry point."""
|
||||
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Tests for ConfigStore database-backed configuration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.config_store import ConfigStore
|
||||
from turnstone.core.settings_registry import SETTINGS
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path):
|
||||
return SQLiteBackend(str(tmp_path / "test.db"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(storage):
|
||||
return ConfigStore(storage)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGet:
|
||||
def test_returns_registry_default_when_nothing_stored(self, store):
|
||||
defn = SETTINGS["tools.timeout"]
|
||||
assert store.get("tools.timeout") == defn.default
|
||||
|
||||
def test_returns_stored_value_after_set(self, store):
|
||||
store.set("tools.timeout", 60)
|
||||
assert store.get("tools.timeout") == 60
|
||||
|
||||
def test_explicit_default_for_unknown_key(self, store):
|
||||
# Unknown keys fall back to explicit default
|
||||
assert store.get("nonexistent.key", 42) == 42
|
||||
|
||||
def test_none_for_unknown_key_without_default(self, store):
|
||||
assert store.get("nonexistent.key") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# set() — validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSet:
|
||||
def test_rejects_unknown_key(self, store):
|
||||
with pytest.raises(ValueError, match="Unknown setting"):
|
||||
store.set("bogus.key", "value")
|
||||
|
||||
def test_rejects_out_of_range(self, store):
|
||||
with pytest.raises(ValueError, match="minimum"):
|
||||
store.set("tools.timeout", 0)
|
||||
|
||||
def test_rejects_above_max(self, store):
|
||||
with pytest.raises(ValueError, match="maximum"):
|
||||
store.set("tools.timeout", 9999)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# set() + get() round-trips
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSetGetRoundTrip:
|
||||
def test_int(self, store):
|
||||
store.set("tools.timeout", 30)
|
||||
assert store.get("tools.timeout") == 30
|
||||
assert isinstance(store.get("tools.timeout"), int)
|
||||
|
||||
def test_float(self, store):
|
||||
store.set("model.temperature", 0.42)
|
||||
assert store.get("model.temperature") == 0.42
|
||||
assert isinstance(store.get("model.temperature"), float)
|
||||
|
||||
def test_bool(self, store):
|
||||
store.set("tools.skip_permissions", True)
|
||||
assert store.get("tools.skip_permissions") is True
|
||||
store.set("tools.skip_permissions", False)
|
||||
assert store.get("tools.skip_permissions") is False
|
||||
|
||||
def test_str(self, store):
|
||||
store.set("model.name", "gpt-5")
|
||||
assert store.get("model.name") == "gpt-5"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# delete()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDelete:
|
||||
def test_reverts_to_default(self, store):
|
||||
store.set("tools.timeout", 30)
|
||||
assert store.get("tools.timeout") == 30
|
||||
store.delete("tools.timeout")
|
||||
defn = SETTINGS["tools.timeout"]
|
||||
assert store.get("tools.timeout") == defn.default
|
||||
|
||||
def test_returns_false_for_non_existent(self, store):
|
||||
assert store.delete("tools.timeout") is False
|
||||
|
||||
def test_rejects_unknown_key(self, store):
|
||||
with pytest.raises(ValueError, match="Unknown setting"):
|
||||
store.delete("nonexistent.key")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# reload()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestReload:
|
||||
def test_picks_up_external_storage_changes(self, storage, store):
|
||||
# Write directly to storage, bypassing ConfigStore
|
||||
from turnstone.core.settings_registry import serialize_value
|
||||
|
||||
storage.upsert_system_setting(
|
||||
key="tools.timeout",
|
||||
value=serialize_value(99),
|
||||
node_id="",
|
||||
is_secret=False,
|
||||
changed_by="external",
|
||||
)
|
||||
# Not visible yet (cached)
|
||||
defn = SETTINGS["tools.timeout"]
|
||||
assert store.get("tools.timeout") == defn.default
|
||||
# Reload and verify
|
||||
store.reload()
|
||||
assert store.get("tools.timeout") == 99
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# all_effective()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAllEffective:
|
||||
def test_merges_stored_with_defaults(self, store):
|
||||
store.set("tools.timeout", 30)
|
||||
effective = store.all_effective()
|
||||
# Stored value
|
||||
assert effective["tools.timeout"] == 30
|
||||
# Default for unstored
|
||||
assert effective["memory.relevance_k"] == SETTINGS["memory.relevance_k"].default
|
||||
# All registry keys present
|
||||
assert set(effective.keys()) == set(SETTINGS.keys())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# stored_keys()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStoredKeys:
|
||||
def test_returns_correct_set(self, store):
|
||||
assert store.stored_keys() == frozenset()
|
||||
store.set("tools.timeout", 30)
|
||||
assert store.stored_keys() == frozenset({"tools.timeout"})
|
||||
store.set("model.name", "gpt-5")
|
||||
assert store.stored_keys() == frozenset({"tools.timeout", "model.name"})
|
||||
store.delete("tools.timeout")
|
||||
assert store.stored_keys() == frozenset({"model.name"})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# version
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestVersion:
|
||||
def test_increments_on_set(self, store):
|
||||
v0 = store.version
|
||||
store.set("tools.timeout", 30)
|
||||
assert store.version == v0 + 1
|
||||
|
||||
def test_increments_on_delete(self, store):
|
||||
store.set("tools.timeout", 30)
|
||||
v0 = store.version
|
||||
store.delete("tools.timeout")
|
||||
assert store.version == v0 + 1
|
||||
|
||||
def test_increments_on_reload(self, store):
|
||||
v0 = store.version
|
||||
store.reload()
|
||||
assert store.version == v0 + 1
|
||||
+4
-1
@@ -16,7 +16,10 @@ class TestSchemaCreation:
|
||||
engine = get_storage()._engine # noqa: SLF001
|
||||
with engine.connect() as conn:
|
||||
rows = conn.execute(
|
||||
sa.text("SELECT name FROM sqlite_master WHERE type='table' AND name='memories'")
|
||||
sa.text(
|
||||
"SELECT name FROM sqlite_master "
|
||||
"WHERE type='table' AND name='structured_memories'"
|
||||
)
|
||||
).fetchall()
|
||||
assert len(rows) == 1
|
||||
rows = conn.execute(
|
||||
|
||||
@@ -0,0 +1,695 @@
|
||||
"""Tests for MCP server admin API endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.middleware import Middleware
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.routing import Mount, Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
from turnstone.console.server import (
|
||||
_collect_mcp_status,
|
||||
_notify_nodes_mcp_reload,
|
||||
admin_create_mcp_server,
|
||||
admin_delete_mcp_server,
|
||||
admin_get_mcp_server,
|
||||
admin_import_mcp_config,
|
||||
admin_list_mcp_servers,
|
||||
admin_update_mcp_server,
|
||||
)
|
||||
from turnstone.core.auth import AuthResult
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth middleware variants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _InjectAuthMiddleware(BaseHTTPMiddleware):
|
||||
"""Inject an admin auth result with admin.mcp permission."""
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Any) -> Response:
|
||||
request.state.auth_result = AuthResult(
|
||||
user_id="test-user",
|
||||
scopes=frozenset({"approve"}),
|
||||
token_source="config",
|
||||
permissions=frozenset(
|
||||
{
|
||||
"read",
|
||||
"write",
|
||||
"approve",
|
||||
"admin.mcp",
|
||||
}
|
||||
),
|
||||
)
|
||||
resp: Response = await call_next(request)
|
||||
return resp
|
||||
|
||||
|
||||
class _InjectAuthNoMcpMiddleware(BaseHTTPMiddleware):
|
||||
"""Inject an auth result WITHOUT admin.mcp permission."""
|
||||
|
||||
async def dispatch(self, request: Request, call_next: Any) -> Response:
|
||||
request.state.auth_result = AuthResult(
|
||||
user_id="test-user",
|
||||
scopes=frozenset({"approve"}),
|
||||
token_source="jwt",
|
||||
permissions=frozenset(
|
||||
{
|
||||
"read",
|
||||
"write",
|
||||
"approve",
|
||||
}
|
||||
),
|
||||
)
|
||||
resp: Response = await call_next(request)
|
||||
return resp
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_ROUTES = [
|
||||
Mount(
|
||||
"/v1",
|
||||
routes=[
|
||||
Route("/api/admin/mcp-servers", admin_list_mcp_servers),
|
||||
Route(
|
||||
"/api/admin/mcp-servers",
|
||||
admin_create_mcp_server,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/api/admin/mcp-servers/import",
|
||||
admin_import_mcp_config,
|
||||
methods=["POST"],
|
||||
),
|
||||
Route(
|
||||
"/api/admin/mcp-servers/{server_id}",
|
||||
admin_get_mcp_server,
|
||||
),
|
||||
Route(
|
||||
"/api/admin/mcp-servers/{server_id}",
|
||||
admin_update_mcp_server,
|
||||
methods=["PUT"],
|
||||
),
|
||||
Route(
|
||||
"/api/admin/mcp-servers/{server_id}",
|
||||
admin_delete_mcp_server,
|
||||
methods=["DELETE"],
|
||||
),
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path):
|
||||
return SQLiteBackend(str(tmp_path / "test.db"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(storage):
|
||||
"""TestClient wired to console admin MCP endpoints with full permissions."""
|
||||
app = Starlette(
|
||||
routes=_ROUTES,
|
||||
middleware=[Middleware(_InjectAuthMiddleware)],
|
||||
)
|
||||
app.state.auth_storage = storage
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client_no_perm(storage):
|
||||
"""TestClient without admin.mcp permission."""
|
||||
app = Starlette(
|
||||
routes=_ROUTES,
|
||||
middleware=[Middleware(_InjectAuthNoMcpMiddleware)],
|
||||
)
|
||||
app.state.auth_storage = storage
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _create_server(
|
||||
client: TestClient,
|
||||
*,
|
||||
name: str = "test-server",
|
||||
transport: str = "stdio",
|
||||
command: str = "npx",
|
||||
args: list[str] | None = None,
|
||||
env: dict[str, str] | None = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
url: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""Helper to create a server via the API and return the response dict."""
|
||||
body: dict[str, Any] = {"name": name, "transport": transport}
|
||||
if transport == "stdio":
|
||||
body["command"] = command
|
||||
body["args"] = args or ["-y", "@modelcontextprotocol/server-test"]
|
||||
else:
|
||||
body["url"] = url or "http://localhost:8080/mcp"
|
||||
if env is not None:
|
||||
body["env"] = env
|
||||
if headers is not None:
|
||||
body["headers"] = headers
|
||||
r = client.post("/v1/api/admin/mcp-servers", json=body)
|
||||
assert r.status_code == 200
|
||||
data: dict[str, Any] = r.json()
|
||||
return data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock _collect_mcp_status to avoid real HTTP calls
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PATCH_MCP_STATUS = patch(
|
||||
"turnstone.console.server._collect_mcp_status",
|
||||
new_callable=AsyncMock,
|
||||
return_value={},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# List
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestListMcpServers:
|
||||
def test_list_empty(self, client):
|
||||
with _PATCH_MCP_STATUS:
|
||||
r = client.get("/v1/api/admin/mcp-servers")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["servers"] == []
|
||||
|
||||
def test_list_returns_created_servers(self, client):
|
||||
_create_server(client, name="server-a")
|
||||
_create_server(client, name="server-b")
|
||||
with _PATCH_MCP_STATUS:
|
||||
r = client.get("/v1/api/admin/mcp-servers")
|
||||
assert r.status_code == 200
|
||||
names = [s["name"] for s in r.json()["servers"]]
|
||||
assert "server-a" in names
|
||||
assert "server-b" in names
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Create
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCreateMcpServer:
|
||||
def test_create_stdio_server(self, client):
|
||||
data = _create_server(client, name="my-mcp", transport="stdio", command="node")
|
||||
assert data["name"] == "my-mcp"
|
||||
assert data["transport"] == "stdio"
|
||||
assert data["command"] == "node"
|
||||
assert data["server_id"]
|
||||
assert data["enabled"] is True
|
||||
|
||||
def test_create_http_server(self, client):
|
||||
data = _create_server(
|
||||
client,
|
||||
name="remote-mcp",
|
||||
transport="streamable-http",
|
||||
url="http://mcp.example.com/sse",
|
||||
)
|
||||
assert data["name"] == "remote-mcp"
|
||||
assert data["transport"] == "streamable-http"
|
||||
assert data["url"] == "http://mcp.example.com/sse"
|
||||
|
||||
def test_create_invalid_name_spaces(self, client):
|
||||
r = client.post(
|
||||
"/v1/api/admin/mcp-servers",
|
||||
json={"name": "bad name!", "transport": "stdio", "command": "x"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "name" in r.json()["error"].lower()
|
||||
|
||||
def test_create_invalid_name_double_underscore(self, client):
|
||||
r = client.post(
|
||||
"/v1/api/admin/mcp-servers",
|
||||
json={"name": "bad__name", "transport": "stdio", "command": "x"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "__" in r.json()["error"]
|
||||
|
||||
def test_create_invalid_transport(self, client):
|
||||
r = client.post(
|
||||
"/v1/api/admin/mcp-servers",
|
||||
json={"name": "ok-name", "transport": "grpc"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "transport" in r.json()["error"].lower()
|
||||
|
||||
def test_create_duplicate_name(self, client):
|
||||
_create_server(client, name="dup-test")
|
||||
r = client.post(
|
||||
"/v1/api/admin/mcp-servers",
|
||||
json={"name": "dup-test", "transport": "stdio", "command": "x"},
|
||||
)
|
||||
assert r.status_code == 409
|
||||
assert "already exists" in r.json()["error"]
|
||||
|
||||
def test_create_missing_name(self, client):
|
||||
r = client.post(
|
||||
"/v1/api/admin/mcp-servers",
|
||||
json={"transport": "stdio", "command": "x"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "name" in r.json()["error"].lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Get single
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetMcpServer:
|
||||
def test_get_existing(self, client):
|
||||
created = _create_server(client, name="get-test")
|
||||
sid = created["server_id"]
|
||||
with _PATCH_MCP_STATUS:
|
||||
r = client.get(f"/v1/api/admin/mcp-servers/{sid}")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["name"] == "get-test"
|
||||
|
||||
def test_get_not_found(self, client):
|
||||
fake_id = uuid.uuid4().hex
|
||||
with _PATCH_MCP_STATUS:
|
||||
r = client.get(f"/v1/api/admin/mcp-servers/{fake_id}")
|
||||
assert r.status_code == 404
|
||||
assert "not found" in r.json()["error"].lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Update
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestUpdateMcpServer:
|
||||
def test_update_name(self, client):
|
||||
created = _create_server(client, name="old-name")
|
||||
sid = created["server_id"]
|
||||
r = client.put(
|
||||
f"/v1/api/admin/mcp-servers/{sid}",
|
||||
json={"name": "new-name"},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["name"] == "new-name"
|
||||
|
||||
def test_update_transport(self, client):
|
||||
created = _create_server(
|
||||
client,
|
||||
name="update-transport",
|
||||
transport="streamable-http",
|
||||
url="http://localhost/mcp",
|
||||
)
|
||||
sid = created["server_id"]
|
||||
r = client.put(
|
||||
f"/v1/api/admin/mcp-servers/{sid}",
|
||||
json={"transport": "stdio", "command": "node"},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["transport"] == "stdio"
|
||||
|
||||
def test_update_enabled(self, client):
|
||||
created = _create_server(client, name="toggle-enabled")
|
||||
sid = created["server_id"]
|
||||
r = client.put(
|
||||
f"/v1/api/admin/mcp-servers/{sid}",
|
||||
json={"enabled": False},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["enabled"] is False
|
||||
|
||||
def test_update_not_found(self, client):
|
||||
fake_id = uuid.uuid4().hex
|
||||
r = client.put(
|
||||
f"/v1/api/admin/mcp-servers/{fake_id}",
|
||||
json={"name": "x"},
|
||||
)
|
||||
assert r.status_code == 404
|
||||
|
||||
def test_update_invalid_transport(self, client):
|
||||
created = _create_server(client, name="bad-transport-update")
|
||||
sid = created["server_id"]
|
||||
r = client.put(
|
||||
f"/v1/api/admin/mcp-servers/{sid}",
|
||||
json={"transport": "websocket"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "transport" in r.json()["error"].lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Delete
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDeleteMcpServer:
|
||||
def test_delete_existing(self, client):
|
||||
created = _create_server(client, name="del-test")
|
||||
sid = created["server_id"]
|
||||
r = client.delete(f"/v1/api/admin/mcp-servers/{sid}")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["status"] == "ok"
|
||||
|
||||
# Confirm it's gone
|
||||
with _PATCH_MCP_STATUS:
|
||||
r2 = client.get(f"/v1/api/admin/mcp-servers/{sid}")
|
||||
assert r2.status_code == 404
|
||||
|
||||
def test_delete_not_found(self, client):
|
||||
fake_id = uuid.uuid4().hex
|
||||
r = client.delete(f"/v1/api/admin/mcp-servers/{fake_id}")
|
||||
assert r.status_code == 404
|
||||
assert "not found" in r.json()["error"].lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Secret masking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSecretMasking:
|
||||
def test_list_masks_secrets(self, client):
|
||||
_create_server(
|
||||
client,
|
||||
name="secret-test",
|
||||
env={"API_KEY": "sk-real-secret-123"},
|
||||
headers={"Authorization": "Bearer tok-xyz"},
|
||||
transport="streamable-http",
|
||||
url="http://localhost/mcp",
|
||||
)
|
||||
with _PATCH_MCP_STATUS:
|
||||
r = client.get("/v1/api/admin/mcp-servers")
|
||||
assert r.status_code == 200
|
||||
server = r.json()["servers"][0]
|
||||
env = json.loads(server["env"])
|
||||
headers = json.loads(server["headers"])
|
||||
assert env["API_KEY"] == "***"
|
||||
assert headers["Authorization"] == "***"
|
||||
|
||||
def test_list_reveals_secrets(self, client):
|
||||
_create_server(
|
||||
client,
|
||||
name="reveal-test",
|
||||
env={"API_KEY": "sk-real-secret-123"},
|
||||
headers={"Authorization": "Bearer tok-xyz"},
|
||||
transport="streamable-http",
|
||||
url="http://localhost/mcp",
|
||||
)
|
||||
with _PATCH_MCP_STATUS:
|
||||
r = client.get("/v1/api/admin/mcp-servers?reveal=true")
|
||||
assert r.status_code == 200
|
||||
server = r.json()["servers"][0]
|
||||
env = json.loads(server["env"])
|
||||
headers = json.loads(server["headers"])
|
||||
assert env["API_KEY"] == "sk-real-secret-123"
|
||||
assert headers["Authorization"] == "Bearer tok-xyz"
|
||||
|
||||
def test_get_masks_secrets_by_default(self, client):
|
||||
created = _create_server(
|
||||
client,
|
||||
name="mask-get-test",
|
||||
env={"SECRET": "value"},
|
||||
)
|
||||
sid = created["server_id"]
|
||||
with _PATCH_MCP_STATUS:
|
||||
r = client.get(f"/v1/api/admin/mcp-servers/{sid}")
|
||||
assert r.status_code == 200
|
||||
env = json.loads(r.json()["env"])
|
||||
assert env["SECRET"] == "***"
|
||||
|
||||
def test_get_reveals_secrets(self, client):
|
||||
created = _create_server(
|
||||
client,
|
||||
name="reveal-get-test",
|
||||
env={"SECRET": "real-value"},
|
||||
)
|
||||
sid = created["server_id"]
|
||||
with _PATCH_MCP_STATUS:
|
||||
r = client.get(f"/v1/api/admin/mcp-servers/{sid}?reveal=true")
|
||||
assert r.status_code == 200
|
||||
env = json.loads(r.json()["env"])
|
||||
assert env["SECRET"] == "real-value"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Import
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestImportMcpConfig:
|
||||
def test_import_inline_config(self, client):
|
||||
config = {
|
||||
"mcpServers": {
|
||||
"filesystem": {
|
||||
"command": "npx",
|
||||
"args": ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"],
|
||||
},
|
||||
"remote": {
|
||||
"url": "http://remote.example.com/mcp",
|
||||
},
|
||||
},
|
||||
}
|
||||
r = client.post(
|
||||
"/v1/api/admin/mcp-servers/import",
|
||||
json={"config": config},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert "filesystem" in data["imported"]
|
||||
assert "remote" in data["imported"]
|
||||
assert data["skipped"] == []
|
||||
assert data["errors"] == []
|
||||
|
||||
def test_import_not_a_dict(self, client):
|
||||
r = client.post(
|
||||
"/v1/api/admin/mcp-servers/import",
|
||||
json={"config": "not-a-dict"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
|
||||
def test_import_skips_duplicates(self, client):
|
||||
_create_server(client, name="existing-srv")
|
||||
config = {
|
||||
"mcpServers": {
|
||||
"existing-srv": {"command": "node", "args": []},
|
||||
"new-srv": {"command": "node", "args": []},
|
||||
},
|
||||
}
|
||||
r = client.post(
|
||||
"/v1/api/admin/mcp-servers/import",
|
||||
json={"config": config},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert "new-srv" in data["imported"]
|
||||
assert "existing-srv" in data["skipped"]
|
||||
|
||||
def test_import_empty_body(self, client):
|
||||
r = client.post(
|
||||
"/v1/api/admin/mcp-servers/import",
|
||||
json={},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "config" in r.json()["error"].lower()
|
||||
|
||||
def test_import_no_mcp_servers_key(self, client):
|
||||
r = client.post(
|
||||
"/v1/api/admin/mcp-servers/import",
|
||||
json={"config": {"other": "data"}},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "mcpServers" in r.json()["error"] or "No" in r.json()["error"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Permission check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPermission:
|
||||
def test_list_without_permission(self, client_no_perm):
|
||||
with _PATCH_MCP_STATUS:
|
||||
r = client_no_perm.get("/v1/api/admin/mcp-servers")
|
||||
assert r.status_code == 403
|
||||
assert "admin.mcp" in r.json()["error"]
|
||||
|
||||
def test_create_without_permission(self, client_no_perm):
|
||||
r = client_no_perm.post(
|
||||
"/v1/api/admin/mcp-servers",
|
||||
json={"name": "test", "transport": "stdio", "command": "x"},
|
||||
)
|
||||
assert r.status_code == 403
|
||||
|
||||
def test_delete_without_permission(self, client_no_perm):
|
||||
r = client_no_perm.delete(f"/v1/api/admin/mcp-servers/{uuid.uuid4().hex}")
|
||||
assert r.status_code == 403
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests for _collect_mcp_status / _notify_nodes_mcp_reload
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _fake_request(*nodes: dict[str, Any], proxy_client: Any = None) -> MagicMock:
|
||||
"""Build a minimal mock request with collector and proxy_client."""
|
||||
collector = MagicMock()
|
||||
collector.get_nodes.return_value = (list(nodes), len(nodes))
|
||||
req = MagicMock()
|
||||
req.app.state.collector = collector
|
||||
req.app.state.proxy_client = proxy_client or AsyncMock()
|
||||
req.app.state.proxy_token_mgr = None
|
||||
req.app.state.proxy_auth_token = "tok"
|
||||
return req
|
||||
|
||||
|
||||
def _mock_resp(status_code: int = 200, json_data: Any = None) -> MagicMock:
|
||||
"""Build a mock httpx response (sync .json(), like the real thing)."""
|
||||
resp = MagicMock()
|
||||
resp.status_code = status_code
|
||||
resp.json.return_value = json_data or {}
|
||||
return resp
|
||||
|
||||
|
||||
class TestCollectMcpStatus:
|
||||
@pytest.mark.anyio
|
||||
async def test_returns_servers_on_200(self):
|
||||
resp = _mock_resp(200, {"servers": {"s1": {"status": "ok"}}})
|
||||
client = AsyncMock()
|
||||
client.get.return_value = resp
|
||||
req = _fake_request(
|
||||
{"node_id": "n1", "server_url": "http://n1:8000"},
|
||||
proxy_client=client,
|
||||
)
|
||||
result = await _collect_mcp_status(req)
|
||||
assert result == {"n1": {"s1": {"status": "ok"}}}
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_skips_non_200(self):
|
||||
client = AsyncMock()
|
||||
client.get.return_value = _mock_resp(503)
|
||||
req = _fake_request(
|
||||
{"node_id": "n1", "server_url": "http://n1:8000"},
|
||||
proxy_client=client,
|
||||
)
|
||||
result = await _collect_mcp_status(req)
|
||||
assert result == {}
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_skips_nodes_without_url(self):
|
||||
client = AsyncMock()
|
||||
req = _fake_request(
|
||||
{"node_id": "n1", "server_url": ""},
|
||||
{"node_id": "n2"},
|
||||
proxy_client=client,
|
||||
)
|
||||
result = await _collect_mcp_status(req)
|
||||
assert result == {}
|
||||
client.get.assert_not_called()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_handles_exception(self):
|
||||
client = AsyncMock()
|
||||
client.get.side_effect = ConnectionError("refused")
|
||||
req = _fake_request(
|
||||
{"node_id": "n1", "server_url": "http://n1:8000"},
|
||||
proxy_client=client,
|
||||
)
|
||||
result = await _collect_mcp_status(req)
|
||||
assert result == {}
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_empty_cluster(self):
|
||||
req = _fake_request()
|
||||
result = await _collect_mcp_status(req)
|
||||
assert result == {}
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_multiple_nodes_mixed(self):
|
||||
ok_resp = _mock_resp(200, {"servers": {"s1": {"status": "ok"}}})
|
||||
err_resp = _mock_resp(500)
|
||||
|
||||
client = AsyncMock()
|
||||
client.get.side_effect = [ok_resp, ConnectionError("down"), err_resp]
|
||||
req = _fake_request(
|
||||
{"node_id": "n1", "server_url": "http://n1:8000"},
|
||||
{"node_id": "n2", "server_url": "http://n2:8000"},
|
||||
{"node_id": "n3", "server_url": "http://n3:8000"},
|
||||
proxy_client=client,
|
||||
)
|
||||
result = await _collect_mcp_status(req)
|
||||
assert result == {"n1": {"s1": {"status": "ok"}}}
|
||||
|
||||
|
||||
class TestNotifyNodesMcpReload:
|
||||
@pytest.mark.anyio
|
||||
async def test_returns_json_on_success(self):
|
||||
client = AsyncMock()
|
||||
client.post.return_value = _mock_resp(200, {"reloaded": 3})
|
||||
req = _fake_request(
|
||||
{"node_id": "n1", "server_url": "http://n1:8000"},
|
||||
proxy_client=client,
|
||||
)
|
||||
result = await _notify_nodes_mcp_reload(req)
|
||||
assert result == {"n1": {"reloaded": 3}}
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_skips_nodes_without_url(self):
|
||||
client = AsyncMock()
|
||||
req = _fake_request(
|
||||
{"node_id": "n1", "server_url": ""},
|
||||
proxy_client=client,
|
||||
)
|
||||
result = await _notify_nodes_mcp_reload(req)
|
||||
assert result == {}
|
||||
client.post.assert_not_called()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_records_error_on_exception(self):
|
||||
client = AsyncMock()
|
||||
client.post.side_effect = ConnectionError("refused")
|
||||
req = _fake_request(
|
||||
{"node_id": "n1", "server_url": "http://n1:8000"},
|
||||
proxy_client=client,
|
||||
)
|
||||
result = await _notify_nodes_mcp_reload(req)
|
||||
assert "n1" in result
|
||||
assert "error" in result["n1"]
|
||||
assert "refused" in result["n1"]["error"]
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_empty_cluster(self):
|
||||
req = _fake_request()
|
||||
result = await _notify_nodes_mcp_reload(req)
|
||||
assert result == {}
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_multiple_nodes_mixed(self):
|
||||
client = AsyncMock()
|
||||
client.post.side_effect = [
|
||||
_mock_resp(200, {"reloaded": 2}),
|
||||
TimeoutError("timeout"),
|
||||
]
|
||||
req = _fake_request(
|
||||
{"node_id": "n1", "server_url": "http://n1:8000"},
|
||||
{"node_id": "n2", "server_url": "http://n2:8000"},
|
||||
proxy_client=client,
|
||||
)
|
||||
result = await _notify_nodes_mcp_reload(req)
|
||||
assert result["n1"] == {"reloaded": 2}
|
||||
assert "error" in result["n2"]
|
||||
@@ -0,0 +1,403 @@
|
||||
"""Tests for MCPClientManager hot-reload methods."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from turnstone.core.mcp_client import MCPClientManager
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _fake_openai_tool(name: str = "mcp__test__search") -> dict[str, Any]:
|
||||
"""Create a fake OpenAI-format tool dict."""
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": name,
|
||||
"description": "[MCP: test] Search stuff",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _fake_resource_dict(
|
||||
uri: str = "file:///README.md",
|
||||
name: str = "readme",
|
||||
server: str = "test",
|
||||
) -> dict[str, Any]:
|
||||
"""Create a fake resource dict as stored in per-server state."""
|
||||
return {
|
||||
"uri": uri,
|
||||
"name": name,
|
||||
"description": "A resource",
|
||||
"mimeType": "text/plain",
|
||||
"server": server,
|
||||
}
|
||||
|
||||
|
||||
def _fake_prompt_dict(
|
||||
name: str = "mcp__test__code_review",
|
||||
original_name: str = "code_review",
|
||||
server: str = "test",
|
||||
) -> dict[str, Any]:
|
||||
"""Create a fake prompt dict as stored in per-server state."""
|
||||
return {
|
||||
"name": name,
|
||||
"original_name": original_name,
|
||||
"server": server,
|
||||
"description": "Generate a code review",
|
||||
"arguments": [
|
||||
{"name": "language", "description": "Programming language", "required": True}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# add_server_sync
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAddServerSync:
|
||||
def test_rejects_double_underscore_name(self) -> None:
|
||||
"""Names containing __ should be rejected."""
|
||||
mgr = MCPClientManager({})
|
||||
result = mgr.add_server_sync("bad__name", {"command": "echo"})
|
||||
assert result["connected"] is False
|
||||
assert "__" in result["error"]
|
||||
assert result["tools"] == 0
|
||||
assert result["resources"] == 0
|
||||
assert result["prompts"] == 0
|
||||
|
||||
def test_fails_without_event_loop(self) -> None:
|
||||
"""Adding a server without starting the event loop should fail gracefully."""
|
||||
mgr = MCPClientManager({})
|
||||
result = mgr.add_server_sync("test", {"command": "echo"})
|
||||
assert result["connected"] is False
|
||||
assert "loop" in result["error"].lower()
|
||||
|
||||
def test_config_removed_on_failure(self) -> None:
|
||||
"""add_server_sync removes the config entry when connection fails."""
|
||||
mgr = MCPClientManager({})
|
||||
mgr.add_server_sync("new-srv", {"command": "echo"})
|
||||
# Since the loop isn't running, it fails and config is cleaned up
|
||||
assert "new-srv" not in mgr._server_configs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# remove_server_sync
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRemoveServerSync:
|
||||
def test_returns_false_for_nonexistent(self) -> None:
|
||||
"""Removing a non-connected server returns False."""
|
||||
mgr = MCPClientManager({})
|
||||
assert mgr.remove_server_sync("nonexistent") is False
|
||||
|
||||
def test_cleans_up_per_server_state(self) -> None:
|
||||
"""remove_server_sync cleans up all per-server state dicts."""
|
||||
mgr = MCPClientManager({"test": {"command": "echo"}})
|
||||
# Simulate state as if the server was connected
|
||||
mgr._per_server_tools["test"] = [_fake_openai_tool()]
|
||||
mgr._per_server_resources["test"] = [_fake_resource_dict()]
|
||||
mgr._per_server_prompts["test"] = [_fake_prompt_dict()]
|
||||
mgr._supports_list_changed["test"] = True
|
||||
mgr._supports_resources["test"] = True
|
||||
mgr._supports_resource_list_changed["test"] = True
|
||||
mgr._supports_prompts["test"] = True
|
||||
mgr._supports_prompt_list_changed["test"] = True
|
||||
mgr._rebuild_tools()
|
||||
mgr._rebuild_resources()
|
||||
mgr._rebuild_prompts()
|
||||
|
||||
# Verify preconditions
|
||||
assert len(mgr.get_tools()) == 1
|
||||
assert mgr.resource_count == 1
|
||||
assert mgr.prompt_count == 1
|
||||
|
||||
mgr.remove_server_sync("test")
|
||||
|
||||
assert len(mgr.get_tools()) == 0
|
||||
assert mgr.resource_count == 0
|
||||
assert mgr.prompt_count == 0
|
||||
assert "test" not in mgr._per_server_tools
|
||||
assert "test" not in mgr._per_server_resources
|
||||
assert "test" not in mgr._per_server_prompts
|
||||
assert "test" not in mgr._supports_list_changed
|
||||
assert "test" not in mgr._supports_resources
|
||||
assert "test" not in mgr._supports_resource_list_changed
|
||||
assert "test" not in mgr._supports_prompts
|
||||
assert "test" not in mgr._supports_prompt_list_changed
|
||||
|
||||
def test_removes_config_to_prevent_reconnect(self) -> None:
|
||||
"""remove_server_sync removes from _server_configs to prevent reconnect."""
|
||||
mgr = MCPClientManager({"test": {"command": "echo"}})
|
||||
assert "test" in mgr._server_configs
|
||||
mgr.remove_server_sync("test")
|
||||
assert "test" not in mgr._server_configs
|
||||
|
||||
def test_preserves_other_servers(self) -> None:
|
||||
"""Removing one server does not affect another server's state."""
|
||||
mgr = MCPClientManager({"srv_a": {}, "srv_b": {}})
|
||||
mgr._per_server_tools["srv_a"] = [_fake_openai_tool("mcp__srv_a__foo")]
|
||||
mgr._per_server_tools["srv_b"] = [_fake_openai_tool("mcp__srv_b__bar")]
|
||||
mgr._rebuild_tools()
|
||||
|
||||
assert len(mgr.get_tools()) == 2
|
||||
|
||||
mgr.remove_server_sync("srv_a")
|
||||
|
||||
assert len(mgr.get_tools()) == 1
|
||||
assert mgr.get_tools()[0]["function"]["name"] == "mcp__srv_b__bar"
|
||||
assert "srv_b" in mgr._server_configs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_server_status
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetServerStatus:
|
||||
def test_disconnected_server_in_config(self) -> None:
|
||||
"""Status of a configured but not connected server shows disconnected."""
|
||||
mgr = MCPClientManager({"test": {"command": "echo"}})
|
||||
status = mgr.get_server_status("test")
|
||||
assert status["connected"] is False
|
||||
assert status["tools"] == 0
|
||||
assert status["resources"] == 0
|
||||
assert status["prompts"] == 0
|
||||
assert status["error"] == ""
|
||||
|
||||
def test_connected_server_with_tools(self) -> None:
|
||||
"""Status of a connected server reports correct tool/resource/prompt counts."""
|
||||
mgr = MCPClientManager({"test": {}})
|
||||
# Simulate connected state
|
||||
mgr._sessions["test"] = object() # any truthy value
|
||||
mgr._per_server_tools["test"] = [
|
||||
_fake_openai_tool("mcp__test__a"),
|
||||
_fake_openai_tool("mcp__test__b"),
|
||||
]
|
||||
mgr._per_server_resources["test"] = [_fake_resource_dict()]
|
||||
mgr._per_server_prompts["test"] = [_fake_prompt_dict()]
|
||||
|
||||
status = mgr.get_server_status("test")
|
||||
assert status["connected"] is True
|
||||
assert status["tools"] == 2
|
||||
assert status["resources"] == 1
|
||||
assert status["prompts"] == 1
|
||||
|
||||
def test_unknown_server(self) -> None:
|
||||
"""Status of a server not in config or sessions shows disconnected."""
|
||||
mgr = MCPClientManager({})
|
||||
status = mgr.get_server_status("unknown")
|
||||
assert status["connected"] is False
|
||||
assert status["tools"] == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_all_server_status
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetAllServerStatus:
|
||||
def test_empty_manager(self) -> None:
|
||||
"""Empty manager returns empty status dict."""
|
||||
mgr = MCPClientManager({})
|
||||
assert mgr.get_all_server_status() == {}
|
||||
|
||||
def test_multiple_servers(self) -> None:
|
||||
"""Manager with configs but no connections returns status for each."""
|
||||
mgr = MCPClientManager({"alpha": {}, "bravo": {}})
|
||||
statuses = mgr.get_all_server_status()
|
||||
assert len(statuses) == 2
|
||||
assert "alpha" in statuses
|
||||
assert "bravo" in statuses
|
||||
assert statuses["alpha"]["connected"] is False
|
||||
assert statuses["bravo"]["connected"] is False
|
||||
|
||||
def test_mixed_connected_and_disconnected(self) -> None:
|
||||
"""Status correctly reflects a mix of connected and disconnected servers."""
|
||||
mgr = MCPClientManager({"up": {}, "down": {}})
|
||||
mgr._sessions["up"] = object()
|
||||
mgr._per_server_tools["up"] = [_fake_openai_tool("mcp__up__x")]
|
||||
|
||||
statuses = mgr.get_all_server_status()
|
||||
assert statuses["up"]["connected"] is True
|
||||
assert statuses["up"]["tools"] == 1
|
||||
assert statuses["down"]["connected"] is False
|
||||
assert statuses["down"]["tools"] == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Error tracking (_last_error)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestErrorTracking:
|
||||
def test_get_server_status_returns_error(self) -> None:
|
||||
"""Error stored in _last_error flows through get_server_status."""
|
||||
mgr = MCPClientManager({"test": {"command": "echo"}})
|
||||
mgr._last_error["test"] = "Connection refused"
|
||||
status = mgr.get_server_status("test")
|
||||
assert status["error"] == "Connection refused"
|
||||
assert status["connected"] is False
|
||||
|
||||
def test_no_error_by_default(self) -> None:
|
||||
"""Default error is empty string."""
|
||||
mgr = MCPClientManager({"test": {"command": "echo"}})
|
||||
status = mgr.get_server_status("test")
|
||||
assert status["error"] == ""
|
||||
|
||||
def test_error_cleared_after_pop(self) -> None:
|
||||
"""Clearing _last_error makes get_server_status return empty."""
|
||||
mgr = MCPClientManager({"test": {"command": "echo"}})
|
||||
mgr._last_error["test"] = "Connection refused"
|
||||
mgr._last_error.pop("test", None)
|
||||
status = mgr.get_server_status("test")
|
||||
assert status["error"] == ""
|
||||
|
||||
def test_error_cleared_on_remove(self) -> None:
|
||||
"""remove_server_sync cleans up _last_error entry."""
|
||||
mgr = MCPClientManager({"test": {"command": "echo"}})
|
||||
mgr._last_error["test"] = "Connection refused"
|
||||
mgr.remove_server_sync("test")
|
||||
assert "test" not in mgr._last_error
|
||||
|
||||
def test_all_server_status_includes_errors(self) -> None:
|
||||
"""get_all_server_status propagates per-server errors."""
|
||||
mgr = MCPClientManager({"alpha": {}, "bravo": {}})
|
||||
mgr._last_error["alpha"] = "Timeout"
|
||||
statuses = mgr.get_all_server_status()
|
||||
assert statuses["alpha"]["error"] == "Timeout"
|
||||
assert statuses["bravo"]["error"] == ""
|
||||
|
||||
def test_error_does_not_leak_across_servers(self) -> None:
|
||||
"""Error on one server does not affect another."""
|
||||
mgr = MCPClientManager({"a": {}, "b": {}})
|
||||
mgr._last_error["a"] = "Failed"
|
||||
assert mgr.get_server_status("b")["error"] == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# reconcile_sync
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeStorage:
|
||||
"""Minimal mock storage for reconcile tests."""
|
||||
|
||||
def __init__(self, rows: list[dict[str, Any]]) -> None:
|
||||
self._rows = rows
|
||||
|
||||
def list_mcp_servers(self, enabled_only: bool = False) -> list[dict[str, Any]]:
|
||||
if enabled_only:
|
||||
return [r for r in self._rows if r.get("enabled", True)]
|
||||
return list(self._rows)
|
||||
|
||||
|
||||
def _db_row(
|
||||
name: str,
|
||||
transport: str = "stdio",
|
||||
command: str = "echo",
|
||||
args: str = "[]",
|
||||
url: str = "",
|
||||
headers: str = "{}",
|
||||
env: str = "{}",
|
||||
enabled: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"name": name,
|
||||
"transport": transport,
|
||||
"command": command,
|
||||
"args": args,
|
||||
"url": url,
|
||||
"headers": headers,
|
||||
"env": env,
|
||||
"enabled": enabled,
|
||||
}
|
||||
|
||||
|
||||
class TestReconcileSync:
|
||||
def test_adds_new_servers(self) -> None:
|
||||
mgr = MCPClientManager({})
|
||||
storage = _FakeStorage([_db_row("new-srv")])
|
||||
# Can't actually connect (no loop), but config should be attempted
|
||||
result = mgr.reconcile_sync(storage)
|
||||
# add_server_sync fails without a loop, but the method shouldn't crash
|
||||
assert "new-srv" not in result["added"] # fails gracefully
|
||||
assert result["removed"] == []
|
||||
assert result["updated"] == []
|
||||
|
||||
def test_removes_stale_db_servers(self) -> None:
|
||||
mgr = MCPClientManager({"old-srv": {"command": "echo"}})
|
||||
mgr._db_managed.add("old-srv") # mark as DB-managed
|
||||
storage = _FakeStorage([]) # DB is empty
|
||||
result = mgr.reconcile_sync(storage)
|
||||
assert "old-srv" in result["removed"]
|
||||
assert "old-srv" not in mgr._server_configs
|
||||
|
||||
def test_preserves_config_file_servers(self) -> None:
|
||||
"""Config-file servers (not in _db_managed) survive reconcile."""
|
||||
mgr = MCPClientManager({"env-srv": {"command": "echo"}})
|
||||
# NOT in _db_managed — loaded from MCP_CONFIG env
|
||||
storage = _FakeStorage([]) # DB is empty
|
||||
result = mgr.reconcile_sync(storage)
|
||||
assert result["removed"] == []
|
||||
assert "env-srv" in mgr._server_configs # still there
|
||||
|
||||
def test_config_server_not_overwritten_by_db_name_collision(self) -> None:
|
||||
"""DB server with same name as config-file server does not replace it."""
|
||||
original_cfg = {"type": "stdio", "command": "config-echo", "args": [], "env": {}}
|
||||
mgr = MCPClientManager({"shared-name": dict(original_cfg)})
|
||||
# NOT in _db_managed — this is a config-file server
|
||||
# DB has a server with the same name but different config
|
||||
storage = _FakeStorage([_db_row("shared-name", command="db-echo")])
|
||||
result = mgr.reconcile_sync(storage)
|
||||
# Config-file server should NOT be updated
|
||||
assert result["updated"] == []
|
||||
assert "shared-name" in mgr._server_configs
|
||||
assert mgr._server_configs["shared-name"]["command"] == "config-echo"
|
||||
|
||||
def test_updates_changed_config(self) -> None:
|
||||
original_cfg = {"type": "stdio", "command": "echo", "args": [], "env": {}}
|
||||
mgr = MCPClientManager({"srv": dict(original_cfg)})
|
||||
mgr._db_managed.add("srv") # mark as DB-managed
|
||||
# DB has updated command — config differs
|
||||
storage = _FakeStorage([_db_row("srv", command="cat")])
|
||||
result = mgr.reconcile_sync(storage)
|
||||
# remove_server_sync ran (old config cleared), add_server_sync attempted
|
||||
# but fails without a running event loop — that's expected in unit tests.
|
||||
# The key assertion: the old config was evicted (not left stale).
|
||||
assert "srv" not in mgr._server_configs
|
||||
# Not in "removed" (that's for servers absent from DB)
|
||||
assert "srv" not in result["removed"]
|
||||
|
||||
def test_no_change_is_noop(self) -> None:
|
||||
cfg = {"type": "stdio", "command": "echo", "args": [], "env": {}}
|
||||
mgr = MCPClientManager({"srv": dict(cfg)})
|
||||
storage = _FakeStorage([_db_row("srv", command="echo")])
|
||||
result = mgr.reconcile_sync(storage)
|
||||
assert result["added"] == []
|
||||
assert result["removed"] == []
|
||||
assert result["updated"] == []
|
||||
# Config unchanged
|
||||
assert "srv" in mgr._server_configs
|
||||
|
||||
def test_storage_failure_graceful(self) -> None:
|
||||
mgr = MCPClientManager({"srv": {}})
|
||||
|
||||
class _BrokenStorage:
|
||||
def list_mcp_servers(self, **kw: Any) -> list[dict[str, Any]]:
|
||||
raise RuntimeError("DB down")
|
||||
|
||||
result = mgr.reconcile_sync(_BrokenStorage())
|
||||
assert result == {"added": [], "removed": [], "updated": []}
|
||||
# Existing server untouched
|
||||
assert "srv" in mgr._server_configs
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Tests for MCP server storage CRUD operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db(tmp_path):
|
||||
"""Fresh SQLite backend for each test."""
|
||||
return SQLiteBackend(str(tmp_path / "test.db"))
|
||||
|
||||
|
||||
def _make_id() -> str:
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
class TestMcpServerStorage:
|
||||
def test_create_and_get(self, db: SQLiteBackend) -> None:
|
||||
sid = _make_id()
|
||||
db.create_mcp_server(
|
||||
server_id=sid,
|
||||
name="test-server",
|
||||
transport="stdio",
|
||||
command="echo",
|
||||
args='["hello"]',
|
||||
)
|
||||
s = db.get_mcp_server(sid)
|
||||
assert s is not None
|
||||
assert s["name"] == "test-server"
|
||||
assert s["transport"] == "stdio"
|
||||
assert s["command"] == "echo"
|
||||
assert s["args"] == '["hello"]'
|
||||
assert s["enabled"] is True
|
||||
assert s["auto_approve"] is False
|
||||
|
||||
def test_get_by_name(self, db: SQLiteBackend) -> None:
|
||||
sid = _make_id()
|
||||
db.create_mcp_server(server_id=sid, name="named-srv", transport="stdio")
|
||||
s = db.get_mcp_server_by_name("named-srv")
|
||||
assert s is not None
|
||||
assert s["server_id"] == sid
|
||||
|
||||
def test_get_by_name_not_found(self, db: SQLiteBackend) -> None:
|
||||
assert db.get_mcp_server_by_name("nope") is None
|
||||
|
||||
def test_get_not_found(self, db: SQLiteBackend) -> None:
|
||||
assert db.get_mcp_server("nonexistent") is None
|
||||
|
||||
def test_list_empty(self, db: SQLiteBackend) -> None:
|
||||
assert db.list_mcp_servers() == []
|
||||
|
||||
def test_list_all(self, db: SQLiteBackend) -> None:
|
||||
db.create_mcp_server(server_id=_make_id(), name="alpha", transport="stdio")
|
||||
db.create_mcp_server(
|
||||
server_id=_make_id(), name="beta", transport="streamable-http", url="http://x"
|
||||
)
|
||||
servers = db.list_mcp_servers()
|
||||
assert len(servers) == 2
|
||||
assert servers[0]["name"] == "alpha" # ordered by name
|
||||
assert servers[1]["name"] == "beta"
|
||||
|
||||
def test_list_enabled_only(self, db: SQLiteBackend) -> None:
|
||||
sid1 = _make_id()
|
||||
sid2 = _make_id()
|
||||
db.create_mcp_server(server_id=sid1, name="enabled-srv", transport="stdio", enabled=True)
|
||||
db.create_mcp_server(server_id=sid2, name="disabled-srv", transport="stdio", enabled=False)
|
||||
enabled = db.list_mcp_servers(enabled_only=True)
|
||||
assert len(enabled) == 1
|
||||
assert enabled[0]["name"] == "enabled-srv"
|
||||
|
||||
def test_update_basic_fields(self, db: SQLiteBackend) -> None:
|
||||
sid = _make_id()
|
||||
db.create_mcp_server(server_id=sid, name="orig", transport="stdio", command="echo")
|
||||
ok = db.update_mcp_server(sid, name="renamed", command="cat")
|
||||
assert ok is True
|
||||
s = db.get_mcp_server(sid)
|
||||
assert s is not None
|
||||
assert s["name"] == "renamed"
|
||||
assert s["command"] == "cat"
|
||||
|
||||
def test_update_boolean_conversion(self, db: SQLiteBackend) -> None:
|
||||
sid = _make_id()
|
||||
db.create_mcp_server(server_id=sid, name="booltest", transport="stdio")
|
||||
db.update_mcp_server(sid, auto_approve=True, enabled=False)
|
||||
s = db.get_mcp_server(sid)
|
||||
assert s is not None
|
||||
assert s["auto_approve"] is True
|
||||
assert s["enabled"] is False
|
||||
|
||||
def test_update_not_found(self, db: SQLiteBackend) -> None:
|
||||
ok = db.update_mcp_server("nonexistent", name="x")
|
||||
assert ok is False
|
||||
|
||||
def test_update_ignores_disallowed_fields(self, db: SQLiteBackend) -> None:
|
||||
sid = _make_id()
|
||||
db.create_mcp_server(server_id=sid, name="guard", transport="stdio", created_by="admin")
|
||||
original = db.get_mcp_server(sid)
|
||||
assert original is not None
|
||||
original_created = original["created"]
|
||||
# created_by and created are not in the mutable allowlist
|
||||
db.update_mcp_server(sid, created_by="evil", created="2000-01-01T00:00:00")
|
||||
s = db.get_mcp_server(sid)
|
||||
assert s is not None
|
||||
assert s["created_by"] == "admin" # unchanged
|
||||
assert s["created"] == original_created # unchanged
|
||||
|
||||
def test_delete(self, db: SQLiteBackend) -> None:
|
||||
sid = _make_id()
|
||||
db.create_mcp_server(server_id=sid, name="delme", transport="stdio")
|
||||
ok = db.delete_mcp_server(sid)
|
||||
assert ok is True
|
||||
assert db.get_mcp_server(sid) is None
|
||||
|
||||
def test_delete_not_found(self, db: SQLiteBackend) -> None:
|
||||
ok = db.delete_mcp_server("nonexistent")
|
||||
assert ok is False
|
||||
|
||||
def test_create_duplicate_name(self, db: SQLiteBackend) -> None:
|
||||
db.create_mcp_server(server_id=_make_id(), name="unique", transport="stdio")
|
||||
# Second create with same name but different ID should be no-op (OR IGNORE)
|
||||
sid2 = _make_id()
|
||||
db.create_mcp_server(server_id=sid2, name="unique", transport="stdio")
|
||||
# OR IGNORE silently drops the conflicting insert
|
||||
assert db.get_mcp_server(sid2) is None
|
||||
|
||||
def test_create_idempotent_same_id(self, db: SQLiteBackend) -> None:
|
||||
sid = _make_id()
|
||||
db.create_mcp_server(server_id=sid, name="idem", transport="stdio", command="v1")
|
||||
db.create_mcp_server(server_id=sid, name="idem", transport="stdio", command="v2")
|
||||
s = db.get_mcp_server(sid)
|
||||
assert s is not None
|
||||
assert s["command"] == "v1" # original preserved, second ignored
|
||||
|
||||
def test_http_transport_fields(self, db: SQLiteBackend) -> None:
|
||||
sid = _make_id()
|
||||
db.create_mcp_server(
|
||||
server_id=sid,
|
||||
name="http-srv",
|
||||
transport="streamable-http",
|
||||
url="https://example.com/mcp",
|
||||
headers='{"Authorization":"Bearer xyz"}',
|
||||
)
|
||||
s = db.get_mcp_server(sid)
|
||||
assert s is not None
|
||||
assert s["transport"] == "streamable-http"
|
||||
assert s["url"] == "https://example.com/mcp"
|
||||
assert "Authorization" in s["headers"]
|
||||
@@ -0,0 +1,508 @@
|
||||
"""Tests for memory API endpoints (server + console admin)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.middleware import Middleware
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.routing import Mount, Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
from turnstone.console.server import (
|
||||
admin_delete_memory,
|
||||
admin_get_memory,
|
||||
admin_list_memories,
|
||||
admin_search_memories,
|
||||
)
|
||||
from turnstone.core.auth import AuthResult
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
from turnstone.server import (
|
||||
delete_memory_endpoint,
|
||||
list_memories,
|
||||
save_memory,
|
||||
search_memories,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth bypass middleware
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _InjectAuthMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request: Request, call_next: Any) -> Response:
|
||||
request.state.auth_result = AuthResult(
|
||||
user_id="test-user",
|
||||
scopes=frozenset({"approve"}),
|
||||
token_source="config",
|
||||
permissions=frozenset(
|
||||
{
|
||||
"read",
|
||||
"write",
|
||||
"approve",
|
||||
"admin.memories",
|
||||
}
|
||||
),
|
||||
)
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path):
|
||||
return SQLiteBackend(str(tmp_path / "test.db"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server_client(storage):
|
||||
"""TestClient wired to server memory endpoints."""
|
||||
import turnstone.core.storage._registry as reg
|
||||
|
||||
old = reg._storage
|
||||
reg._storage = storage
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Mount(
|
||||
"/v1",
|
||||
routes=[
|
||||
Route("/api/memories", list_memories),
|
||||
Route("/api/memories", save_memory, methods=["POST"]),
|
||||
Route("/api/memories/search", search_memories, methods=["POST"]),
|
||||
Route("/api/memories/{name}", delete_memory_endpoint, methods=["DELETE"]),
|
||||
],
|
||||
),
|
||||
],
|
||||
middleware=[Middleware(_InjectAuthMiddleware)],
|
||||
)
|
||||
yield TestClient(app)
|
||||
reg._storage = old
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def admin_client(storage):
|
||||
"""TestClient wired to console admin memory endpoints."""
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Mount(
|
||||
"/v1",
|
||||
routes=[
|
||||
Route("/api/admin/memories", admin_list_memories),
|
||||
Route("/api/admin/memories/search", admin_search_memories),
|
||||
Route("/api/admin/memories/{memory_id}", admin_get_memory),
|
||||
Route(
|
||||
"/api/admin/memories/{memory_id}",
|
||||
admin_delete_memory,
|
||||
methods=["DELETE"],
|
||||
),
|
||||
],
|
||||
),
|
||||
],
|
||||
middleware=[Middleware(_InjectAuthMiddleware)],
|
||||
)
|
||||
app.state.auth_storage = storage
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _seed_memory(storage, name="test_key", content="test content", **kw):
|
||||
"""Helper to insert a memory directly into storage."""
|
||||
import uuid
|
||||
|
||||
mid = kw.pop("memory_id", str(uuid.uuid4()))
|
||||
storage.create_structured_memory(
|
||||
mid,
|
||||
name,
|
||||
kw.get("description", ""),
|
||||
kw.get("mem_type", "project"),
|
||||
kw.get("scope", "global"),
|
||||
kw.get("scope_id", ""),
|
||||
content,
|
||||
)
|
||||
return mid
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Server endpoint tests
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestServerListMemories:
|
||||
def test_empty(self, server_client):
|
||||
r = server_client.get("/v1/api/memories")
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert data["memories"] == []
|
||||
assert data["total"] == 0
|
||||
|
||||
def test_with_data(self, server_client, storage):
|
||||
_seed_memory(storage, "key_a", "content a")
|
||||
_seed_memory(storage, "key_b", "content b")
|
||||
r = server_client.get("/v1/api/memories")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["total"] == 2
|
||||
|
||||
def test_filter_by_type(self, server_client, storage):
|
||||
_seed_memory(storage, "a", "x", mem_type="user")
|
||||
_seed_memory(storage, "b", "y", mem_type="project")
|
||||
r = server_client.get("/v1/api/memories?type=user")
|
||||
assert r.json()["total"] == 1
|
||||
assert r.json()["memories"][0]["name"] == "a"
|
||||
|
||||
def test_filter_by_scope(self, server_client, storage):
|
||||
_seed_memory(storage, "a", "x", scope="global")
|
||||
_seed_memory(storage, "b", "y", scope="workstream", scope_id="ws1")
|
||||
r = server_client.get("/v1/api/memories?scope=workstream&scope_id=ws1")
|
||||
assert r.json()["total"] == 1
|
||||
assert r.json()["memories"][0]["name"] == "b"
|
||||
|
||||
def test_limit(self, server_client, storage):
|
||||
for i in range(5):
|
||||
_seed_memory(storage, f"k{i}", f"v{i}")
|
||||
r = server_client.get("/v1/api/memories?limit=2")
|
||||
assert r.json()["total"] == 2
|
||||
|
||||
def test_invalid_limit(self, server_client):
|
||||
r = server_client.get("/v1/api/memories?limit=abc")
|
||||
assert r.status_code == 400
|
||||
|
||||
|
||||
class TestServerSaveMemory:
|
||||
def test_create(self, server_client):
|
||||
r = server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={"name": "my_key", "content": "my content"},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
data = r.json()
|
||||
assert data["name"] == "my_key"
|
||||
assert data["content"] == "my content"
|
||||
assert data["type"] == "project"
|
||||
assert data["scope"] == "global"
|
||||
|
||||
def test_upsert(self, server_client):
|
||||
server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={"name": "key", "content": "v1"},
|
||||
)
|
||||
r = server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={"name": "key", "content": "v2"},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["content"] == "v2"
|
||||
|
||||
def test_with_type_and_scope(self, server_client):
|
||||
r = server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={
|
||||
"name": "feedback_key",
|
||||
"content": "data",
|
||||
"type": "feedback",
|
||||
"scope": "workstream",
|
||||
"scope_id": "ws1",
|
||||
},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
assert r.json()["type"] == "feedback"
|
||||
assert r.json()["scope"] == "workstream"
|
||||
|
||||
def test_missing_name(self, server_client):
|
||||
r = server_client.post("/v1/api/memories", json={"content": "data"})
|
||||
assert r.status_code == 400
|
||||
|
||||
def test_missing_content(self, server_client):
|
||||
r = server_client.post("/v1/api/memories", json={"name": "k"})
|
||||
assert r.status_code == 400
|
||||
|
||||
def test_invalid_type(self, server_client):
|
||||
r = server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={"name": "k", "content": "c", "type": "bogus"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "invalid type" in r.json()["error"]
|
||||
|
||||
def test_invalid_scope(self, server_client):
|
||||
r = server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={"name": "k", "content": "c", "scope": "bogus"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "invalid scope" in r.json()["error"]
|
||||
|
||||
def test_content_too_large(self, server_client):
|
||||
r = server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={"name": "k", "content": "x" * 70000},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "limit" in r.json()["error"]
|
||||
|
||||
def test_name_normalisation(self, server_client):
|
||||
r = server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={"name": "My-Key Name", "content": "data"},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
assert r.json()["name"] == "my_key_name"
|
||||
|
||||
|
||||
class TestServerUserScopeSecurity:
|
||||
def test_user_scope_binds_to_auth(self, server_client):
|
||||
"""User scope auto-resolves scope_id from authenticated user."""
|
||||
r = server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={"name": "priv", "content": "secret", "scope": "user"},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
assert r.json()["scope_id"] == "test-user"
|
||||
|
||||
def test_user_scope_rejects_cross_user(self, server_client):
|
||||
"""Cannot access another user's memories via scope_id."""
|
||||
r = server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={"name": "x", "content": "y", "scope": "user", "scope_id": "other-user"},
|
||||
)
|
||||
assert r.status_code == 403
|
||||
|
||||
def test_user_scope_allows_own_scope_id(self, server_client):
|
||||
"""Passing own user_id as scope_id is allowed."""
|
||||
r = server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={"name": "x", "content": "y", "scope": "user", "scope_id": "test-user"},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
|
||||
def test_list_rejects_cross_user(self, server_client):
|
||||
r = server_client.get("/v1/api/memories?scope=user&scope_id=other-user")
|
||||
assert r.status_code == 403
|
||||
|
||||
def test_delete_rejects_cross_user(self, server_client, storage):
|
||||
_seed_memory(storage, "k", "v", scope="user", scope_id="other-user")
|
||||
r = server_client.delete("/v1/api/memories/k?scope=user&scope_id=other-user")
|
||||
assert r.status_code == 403
|
||||
|
||||
|
||||
class TestServerScopeScopeIdValidation:
|
||||
"""scope_id requires scope; global scope rejects scope_id."""
|
||||
|
||||
def test_save_global_with_scope_id_rejected(self, server_client):
|
||||
r = server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={"name": "k", "content": "c", "scope": "global", "scope_id": "ws1"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "scope_id" in r.json()["error"]
|
||||
|
||||
def test_save_workstream_without_scope_id_rejected(self, server_client):
|
||||
r = server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={"name": "k", "content": "c", "scope": "workstream"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "scope_id is required" in r.json()["error"]
|
||||
|
||||
def test_save_workstream_with_scope_id_ok(self, server_client):
|
||||
r = server_client.post(
|
||||
"/v1/api/memories",
|
||||
json={"name": "k", "content": "c", "scope": "workstream", "scope_id": "ws1"},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
|
||||
def test_list_scope_id_without_scope_rejected(self, server_client):
|
||||
r = server_client.get("/v1/api/memories?scope_id=ws1")
|
||||
assert r.status_code == 400
|
||||
assert "scope is required" in r.json()["error"]
|
||||
|
||||
def test_list_global_with_scope_id_rejected(self, server_client):
|
||||
r = server_client.get("/v1/api/memories?scope=global&scope_id=ws1")
|
||||
assert r.status_code == 400
|
||||
assert "scope_id" in r.json()["error"]
|
||||
|
||||
def test_search_scope_id_without_scope_rejected(self, server_client):
|
||||
r = server_client.post(
|
||||
"/v1/api/memories/search",
|
||||
json={"query": "test", "scope_id": "ws1"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "scope is required" in r.json()["error"]
|
||||
|
||||
def test_search_global_with_scope_id_rejected(self, server_client):
|
||||
r = server_client.post(
|
||||
"/v1/api/memories/search",
|
||||
json={"query": "test", "scope": "global", "scope_id": "ws1"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "scope_id" in r.json()["error"]
|
||||
|
||||
def test_delete_global_with_scope_id_rejected(self, server_client):
|
||||
r = server_client.delete("/v1/api/memories/k?scope=global&scope_id=ws1")
|
||||
assert r.status_code == 400
|
||||
assert "scope_id" in r.json()["error"]
|
||||
|
||||
def test_delete_workstream_without_scope_id_rejected(self, server_client):
|
||||
r = server_client.delete("/v1/api/memories/k?scope=workstream")
|
||||
assert r.status_code == 400
|
||||
assert "scope_id is required" in r.json()["error"]
|
||||
|
||||
|
||||
class TestServerSearchMemories:
|
||||
def test_search(self, server_client, storage):
|
||||
_seed_memory(storage, "db_config", "postgresql host", description="database")
|
||||
_seed_memory(storage, "api_key", "secret_value")
|
||||
r = server_client.post(
|
||||
"/v1/api/memories/search",
|
||||
json={"query": "database"},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["total"] == 1
|
||||
assert r.json()["memories"][0]["name"] == "db_config"
|
||||
|
||||
def test_no_results(self, server_client, storage):
|
||||
_seed_memory(storage, "a", "b")
|
||||
r = server_client.post(
|
||||
"/v1/api/memories/search",
|
||||
json={"query": "nonexistent_xyz"},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
assert r.json()["total"] == 0
|
||||
|
||||
def test_missing_query(self, server_client):
|
||||
r = server_client.post("/v1/api/memories/search", json={})
|
||||
assert r.status_code == 400
|
||||
|
||||
|
||||
class TestServerDeleteMemory:
|
||||
def test_delete(self, server_client, storage):
|
||||
_seed_memory(storage, "doomed")
|
||||
r = server_client.delete("/v1/api/memories/doomed")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["status"] == "ok"
|
||||
|
||||
def test_not_found(self, server_client):
|
||||
r = server_client.delete("/v1/api/memories/nope")
|
||||
assert r.status_code == 404
|
||||
|
||||
def test_delete_scoped(self, server_client, storage):
|
||||
_seed_memory(storage, "k", "data", scope="workstream", scope_id="ws1")
|
||||
# Wrong scope → not found
|
||||
r = server_client.delete("/v1/api/memories/k")
|
||||
assert r.status_code == 404
|
||||
# Correct scope → success
|
||||
r = server_client.delete("/v1/api/memories/k?scope=workstream&scope_id=ws1")
|
||||
assert r.status_code == 200
|
||||
|
||||
def test_invalid_scope(self, server_client):
|
||||
r = server_client.delete("/v1/api/memories/k?scope=bogus")
|
||||
assert r.status_code == 400
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Console admin endpoint tests
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestAdminListMemories:
|
||||
def test_empty(self, admin_client):
|
||||
r = admin_client.get("/v1/api/admin/memories")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["memories"] == []
|
||||
|
||||
def test_with_data(self, admin_client, storage):
|
||||
_seed_memory(storage, "a", "1")
|
||||
_seed_memory(storage, "b", "2")
|
||||
r = admin_client.get("/v1/api/admin/memories")
|
||||
assert r.json()["total"] == 2
|
||||
|
||||
def test_filter(self, admin_client, storage):
|
||||
_seed_memory(storage, "a", "1", mem_type="user")
|
||||
_seed_memory(storage, "b", "2", mem_type="project")
|
||||
r = admin_client.get("/v1/api/admin/memories?type=user")
|
||||
assert r.json()["total"] == 1
|
||||
|
||||
|
||||
class TestAdminScopeScopeIdValidation:
|
||||
"""Console admin: scope_id requires scope; global scope rejects scope_id."""
|
||||
|
||||
def test_list_scope_id_without_scope_rejected(self, admin_client):
|
||||
r = admin_client.get("/v1/api/admin/memories?scope_id=ws1")
|
||||
assert r.status_code == 400
|
||||
assert "scope is required" in r.json()["error"]
|
||||
|
||||
def test_list_global_with_scope_id_rejected(self, admin_client):
|
||||
r = admin_client.get("/v1/api/admin/memories?scope=global&scope_id=ws1")
|
||||
assert r.status_code == 400
|
||||
assert "scope_id" in r.json()["error"]
|
||||
|
||||
def test_search_scope_id_without_scope_rejected(self, admin_client):
|
||||
r = admin_client.get("/v1/api/admin/memories/search?q=test&scope_id=ws1")
|
||||
assert r.status_code == 400
|
||||
assert "scope is required" in r.json()["error"]
|
||||
|
||||
def test_search_global_with_scope_id_rejected(self, admin_client):
|
||||
r = admin_client.get("/v1/api/admin/memories/search?q=test&scope=global&scope_id=ws1")
|
||||
assert r.status_code == 400
|
||||
assert "scope_id" in r.json()["error"]
|
||||
|
||||
|
||||
class TestAdminSearchMemories:
|
||||
def test_search(self, admin_client, storage):
|
||||
_seed_memory(storage, "db_config", "pg host", description="database")
|
||||
_seed_memory(storage, "other", "unrelated")
|
||||
r = admin_client.get("/v1/api/admin/memories/search?q=database")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["total"] == 1
|
||||
|
||||
def test_missing_query(self, admin_client):
|
||||
r = admin_client.get("/v1/api/admin/memories/search")
|
||||
assert r.status_code == 400
|
||||
|
||||
|
||||
class TestAdminGetMemory:
|
||||
def test_found(self, admin_client, storage):
|
||||
mid = _seed_memory(storage, "k", "content")
|
||||
r = admin_client.get(f"/v1/api/admin/memories/{mid}")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["name"] == "k"
|
||||
|
||||
def test_not_found(self, admin_client):
|
||||
r = admin_client.get("/v1/api/admin/memories/nonexistent-id")
|
||||
assert r.status_code == 404
|
||||
|
||||
|
||||
class TestAdminDeleteMemory:
|
||||
def test_delete(self, admin_client, storage):
|
||||
mid = _seed_memory(storage, "doomed", "data")
|
||||
r = admin_client.delete(f"/v1/api/admin/memories/{mid}")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["status"] == "ok"
|
||||
# Verify it's gone
|
||||
assert storage.get_structured_memory(mid) is None
|
||||
|
||||
def test_not_found(self, admin_client):
|
||||
r = admin_client.delete("/v1/api/admin/memories/nonexistent-id")
|
||||
assert r.status_code == 404
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Storage: delete_structured_memory_by_id
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestDeleteByIdStorage:
|
||||
def test_delete_existing(self, storage):
|
||||
storage.create_structured_memory("m1", "k", "d", "project", "global", "", "data")
|
||||
assert storage.delete_structured_memory_by_id("m1")
|
||||
assert storage.get_structured_memory("m1") is None
|
||||
|
||||
def test_delete_nonexistent(self, storage):
|
||||
assert not storage.delete_structured_memory_by_id("nope")
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Tests for turnstone.core.memory_relevance — scoring, formatting, context extraction."""
|
||||
|
||||
from turnstone.core.memory_relevance import (
|
||||
build_memory_context,
|
||||
extract_recent_context,
|
||||
score_memories,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# score_memories
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestScoreMemories:
|
||||
def test_empty_memories(self):
|
||||
assert score_memories([], "query") == []
|
||||
|
||||
def test_empty_query_returns_recent(self):
|
||||
mems = [
|
||||
{"name": "a", "description": "", "content": "alpha"},
|
||||
{"name": "b", "description": "", "content": "beta"},
|
||||
{"name": "c", "description": "", "content": "gamma"},
|
||||
]
|
||||
result = score_memories(mems, "", k=2)
|
||||
assert len(result) == 2
|
||||
assert result[0]["name"] == "a"
|
||||
|
||||
def test_whitespace_query_returns_recent(self):
|
||||
mems = [{"name": "a", "description": "", "content": "alpha"}]
|
||||
assert score_memories(mems, " ", k=5) == mems
|
||||
|
||||
def test_relevance_ranking(self):
|
||||
mems = [
|
||||
{"name": "cooking", "description": "recipes", "content": "pasta sauce tomato"},
|
||||
{"name": "python", "description": "programming", "content": "python file io disk"},
|
||||
{
|
||||
"name": "disk_io",
|
||||
"description": "file operations",
|
||||
"content": "read write file disk",
|
||||
},
|
||||
]
|
||||
result = score_memories(mems, "file disk", k=2)
|
||||
names = [m["name"] for m in result]
|
||||
assert "disk_io" in names
|
||||
assert "python" in names
|
||||
|
||||
def test_k_limits_results(self):
|
||||
mems = [{"name": f"m{i}", "description": "", "content": f"word{i}"} for i in range(10)]
|
||||
result = score_memories(mems, "word0 word1 word2", k=2)
|
||||
assert len(result) <= 2
|
||||
|
||||
def test_no_match_returns_empty(self):
|
||||
mems = [{"name": "a", "description": "", "content": "hello world"}]
|
||||
result = score_memories(mems, "zzzznotfound")
|
||||
assert result == []
|
||||
|
||||
def test_uses_name_for_scoring(self):
|
||||
mems = [
|
||||
{"name": "database_config", "description": "", "content": "host=localhost"},
|
||||
{"name": "unrelated", "description": "", "content": "nothing here"},
|
||||
]
|
||||
result = score_memories(mems, "database", k=1)
|
||||
assert len(result) == 1
|
||||
assert result[0]["name"] == "database_config"
|
||||
|
||||
def test_uses_description_for_scoring(self):
|
||||
mems = [
|
||||
{"name": "x", "description": "postgresql connection settings", "content": "host=db"},
|
||||
{"name": "y", "description": "unrelated", "content": "nothing"},
|
||||
]
|
||||
result = score_memories(mems, "postgresql", k=1)
|
||||
assert result[0]["name"] == "x"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# build_memory_context
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBuildMemoryContext:
|
||||
def test_empty_memories(self):
|
||||
assert build_memory_context([]) == ""
|
||||
|
||||
def test_single_memory(self):
|
||||
mems = [{"name": "test", "type": "project", "scope": "global", "content": "hello"}]
|
||||
ctx = build_memory_context(mems)
|
||||
assert "<memories>" in ctx
|
||||
assert "</memories>" in ctx
|
||||
assert 'name="test"' in ctx
|
||||
assert "hello" in ctx
|
||||
|
||||
def test_html_escaping(self):
|
||||
mems = [
|
||||
{
|
||||
"name": "a<b",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"content": "x & y",
|
||||
"description": 'say "hi"',
|
||||
}
|
||||
]
|
||||
ctx = build_memory_context(mems)
|
||||
assert "<" in ctx
|
||||
assert "&" in ctx
|
||||
assert """ in ctx
|
||||
|
||||
def test_truncates_long_content(self):
|
||||
mems = [
|
||||
{
|
||||
"name": "long",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"content": "x" * 600,
|
||||
}
|
||||
]
|
||||
ctx = build_memory_context(mems)
|
||||
assert "..." in ctx
|
||||
# Content should be truncated to 500 chars + "..."
|
||||
assert "x" * 501 not in ctx
|
||||
|
||||
def test_description_attribute(self):
|
||||
mems = [
|
||||
{
|
||||
"name": "test",
|
||||
"type": "project",
|
||||
"scope": "global",
|
||||
"content": "data",
|
||||
"description": "some desc",
|
||||
}
|
||||
]
|
||||
ctx = build_memory_context(mems)
|
||||
assert 'description="some desc"' in ctx
|
||||
|
||||
def test_no_description_attribute_when_empty(self):
|
||||
mems = [{"name": "test", "type": "project", "scope": "global", "content": "data"}]
|
||||
ctx = build_memory_context(mems)
|
||||
assert "description=" not in ctx
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# extract_recent_context
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractRecentContext:
|
||||
def test_extracts_user_messages(self):
|
||||
msgs = [
|
||||
{"role": "user", "content": "hello"},
|
||||
{"role": "assistant", "content": "hi"},
|
||||
{"role": "user", "content": "world"},
|
||||
]
|
||||
ctx = extract_recent_context(msgs, max_messages=2)
|
||||
assert "world" in ctx
|
||||
assert "hello" in ctx
|
||||
|
||||
def test_skips_non_user(self):
|
||||
msgs = [
|
||||
{"role": "assistant", "content": "ignored"},
|
||||
{"role": "user", "content": "included"},
|
||||
]
|
||||
ctx = extract_recent_context(msgs, max_messages=5)
|
||||
assert "included" in ctx
|
||||
assert "ignored" not in ctx
|
||||
|
||||
def test_respects_max_messages(self):
|
||||
msgs = [
|
||||
{"role": "user", "content": "first"},
|
||||
{"role": "user", "content": "second"},
|
||||
{"role": "user", "content": "third"},
|
||||
]
|
||||
ctx = extract_recent_context(msgs, max_messages=1)
|
||||
assert "third" in ctx
|
||||
assert "first" not in ctx
|
||||
|
||||
def test_handles_list_content(self):
|
||||
msgs = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "multi-part"},
|
||||
{"type": "image_url", "image_url": {"url": "http://example.com"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
ctx = extract_recent_context(msgs, max_messages=1)
|
||||
assert "multi-part" in ctx
|
||||
|
||||
def test_handles_string_parts_in_list(self):
|
||||
msgs = [{"role": "user", "content": ["plain string part"]}]
|
||||
ctx = extract_recent_context(msgs, max_messages=1)
|
||||
assert "plain string part" in ctx
|
||||
|
||||
def test_empty_messages(self):
|
||||
assert extract_recent_context([]) == ""
|
||||
@@ -0,0 +1,267 @@
|
||||
"""Tests for turnstone.core.metacognition — detection, nudging, formatting."""
|
||||
|
||||
from turnstone.core.metacognition import (
|
||||
NUDGE_COMPLETION,
|
||||
NUDGE_CORRECTION,
|
||||
NUDGE_DENIAL,
|
||||
NUDGE_RESUME,
|
||||
NUDGE_START,
|
||||
detect_completion,
|
||||
detect_correction,
|
||||
format_nudge,
|
||||
should_nudge,
|
||||
)
|
||||
|
||||
|
||||
class TestDetectCorrection:
|
||||
"""Strong patterns always fire; weak 'no <word>' uses allowlist."""
|
||||
|
||||
# -- strong patterns (always fire) --
|
||||
|
||||
def test_no_comma(self):
|
||||
assert detect_correction("no, that's wrong") is True
|
||||
|
||||
def test_no_period(self):
|
||||
assert detect_correction("no. do it differently") is True
|
||||
|
||||
def test_dont(self):
|
||||
assert detect_correction("don't use tabs") is True
|
||||
|
||||
def test_stop(self):
|
||||
assert detect_correction("stop adding comments") is True
|
||||
|
||||
def test_actually(self):
|
||||
assert detect_correction("actually, use pytest instead") is True
|
||||
|
||||
def test_instead(self):
|
||||
assert detect_correction("instead, try this approach") is True
|
||||
|
||||
def test_wrong(self):
|
||||
assert detect_correction("wrong, the port is 8080") is True
|
||||
|
||||
def test_i_said(self):
|
||||
assert detect_correction("I said use snake_case") is True
|
||||
|
||||
def test_i_meant(self):
|
||||
assert detect_correction("I meant the other file") is True
|
||||
|
||||
def test_please_dont(self):
|
||||
assert detect_correction("please don't mock the database") is True
|
||||
|
||||
# -- weak pattern: "no" + allowlisted context word --
|
||||
|
||||
def test_no_space(self):
|
||||
assert detect_correction("no I meant the other one") is True
|
||||
|
||||
def test_no_that(self):
|
||||
assert detect_correction("no that's wrong") is True
|
||||
|
||||
def test_no_it(self):
|
||||
assert detect_correction("no it should be different") is True
|
||||
|
||||
def test_no_the(self):
|
||||
assert detect_correction("no the other one") is True
|
||||
|
||||
def test_no_not(self):
|
||||
assert detect_correction("no not that file") is True
|
||||
|
||||
def test_no_you(self):
|
||||
assert detect_correction("no you should use pytest") is True
|
||||
|
||||
# -- negatives: "no <word>" not in allowlist --
|
||||
|
||||
def test_negative_no_problem(self):
|
||||
assert detect_correction("no problem") is False
|
||||
|
||||
def test_negative_no_worries(self):
|
||||
assert detect_correction("no worries") is False
|
||||
|
||||
def test_negative_no_rush(self):
|
||||
assert detect_correction("no rush") is False
|
||||
|
||||
def test_negative_no_one(self):
|
||||
assert detect_correction("no one knows") is False
|
||||
|
||||
def test_negative_no_thanks(self):
|
||||
assert detect_correction("no thanks") is False
|
||||
|
||||
def test_negative_no_doubt(self):
|
||||
assert detect_correction("no doubt about it") is False
|
||||
|
||||
def test_negative_no_idea(self):
|
||||
assert detect_correction("no idea what you mean") is False
|
||||
|
||||
def test_negative_no_kidding(self):
|
||||
assert detect_correction("no kidding") is False
|
||||
|
||||
def test_negative_no_luck(self):
|
||||
assert detect_correction("no luck finding the bug") is False
|
||||
|
||||
# -- negatives: unrelated messages --
|
||||
|
||||
def test_negative_notice(self):
|
||||
assert detect_correction("I noticed the test passes") is False
|
||||
|
||||
def test_negative_nobody(self):
|
||||
assert detect_correction("nobody knows the answer") is False
|
||||
|
||||
def test_negative_innovation(self):
|
||||
assert detect_correction("innovation in AI is exciting") is False
|
||||
|
||||
def test_negative_normal(self):
|
||||
assert detect_correction("can you refactor this function?") is False
|
||||
|
||||
def test_negative_empty(self):
|
||||
assert detect_correction("") is False
|
||||
|
||||
def test_negative_note(self):
|
||||
assert detect_correction("note that this requires Python 3.11") is False
|
||||
|
||||
def test_negative_nonstop(self):
|
||||
assert detect_correction("nonstop improvements to the codebase") is False
|
||||
|
||||
|
||||
class TestDetectCompletion:
|
||||
"""Strong patterns always fire; weak patterns gated by length + continuation."""
|
||||
|
||||
# -- strong patterns (always fire) --
|
||||
|
||||
def test_thats_all(self):
|
||||
assert detect_completion("that's all for now") is True
|
||||
|
||||
def test_lgtm(self):
|
||||
assert detect_completion("lgtm") is True
|
||||
|
||||
# -- weak patterns: short message, no continuation --
|
||||
|
||||
def test_thanks(self):
|
||||
assert detect_completion("thanks, that's perfect") is True
|
||||
|
||||
def test_thanks_standalone(self):
|
||||
assert detect_completion("thanks") is True
|
||||
|
||||
def test_thanks_exclaim(self):
|
||||
assert detect_completion("thanks!") is True
|
||||
|
||||
def test_looks_good(self):
|
||||
assert detect_completion("looks good to me") is True
|
||||
|
||||
def test_perfect(self):
|
||||
assert detect_completion("perfect") is True
|
||||
|
||||
def test_done(self):
|
||||
assert detect_completion("done") is True
|
||||
|
||||
def test_great_job(self):
|
||||
assert detect_completion("great job") is True
|
||||
|
||||
def test_that_works(self):
|
||||
assert detect_completion("that works") is True
|
||||
|
||||
# -- negatives: "thanks for" is acknowledgment --
|
||||
|
||||
def test_negative_thanks_for(self):
|
||||
assert detect_completion("thanks for the update") is False
|
||||
|
||||
def test_negative_thanks_for_looking(self):
|
||||
assert detect_completion("thanks for looking into this") is False
|
||||
|
||||
# -- negatives: continuation markers suppress weak patterns --
|
||||
|
||||
def test_negative_thanks_but(self):
|
||||
assert detect_completion("thanks but can you also add tests") is False
|
||||
|
||||
def test_negative_thanks_though(self):
|
||||
assert detect_completion("thanks though I have one more question") is False
|
||||
|
||||
def test_negative_looks_good_but(self):
|
||||
assert detect_completion("looks good but can you also add validation") is False
|
||||
|
||||
def test_negative_perfect_now(self):
|
||||
assert detect_completion("perfect, now add error handling") is False
|
||||
|
||||
def test_negative_done_can_you(self):
|
||||
assert detect_completion("done with that, can you start on the tests?") is False
|
||||
|
||||
def test_negative_question_mark(self):
|
||||
assert detect_completion("can you add error handling?") is False
|
||||
|
||||
# -- negatives: long messages suppress weak patterns --
|
||||
|
||||
def test_negative_thanks_long(self):
|
||||
msg = "thanks, this is really helpful — I was also wondering about the deployment pipeline and whether we need to update the CI config"
|
||||
assert detect_completion(msg) is False
|
||||
|
||||
def test_negative_looks_good_long(self):
|
||||
msg = "looks good overall, there are a few things I'd like to tweak though — the error messages could be more descriptive and the retry logic needs a backoff"
|
||||
assert detect_completion(msg) is False
|
||||
|
||||
# -- negatives: unrelated --
|
||||
|
||||
def test_negative_empty(self):
|
||||
assert detect_completion("") is False
|
||||
|
||||
|
||||
class TestShouldNudge:
|
||||
def test_basic_fires(self):
|
||||
state: dict[str, float] = {}
|
||||
assert should_nudge("correction", state, message_count=3, memory_count=0) is True
|
||||
|
||||
def test_cooldown(self):
|
||||
state: dict[str, float] = {}
|
||||
should_nudge("correction", state, message_count=3, memory_count=0)
|
||||
assert should_nudge("correction", state, message_count=3, memory_count=0) is False
|
||||
|
||||
def test_different_types_independent(self):
|
||||
state: dict[str, float] = {}
|
||||
should_nudge("correction", state, message_count=3, memory_count=0)
|
||||
assert should_nudge("denial", state, message_count=3, memory_count=0) is True
|
||||
|
||||
def test_no_nudge_first_message(self):
|
||||
state: dict[str, float] = {}
|
||||
assert should_nudge("correction", state, message_count=1, memory_count=0) is False
|
||||
|
||||
def test_resume_requires_memories(self):
|
||||
state: dict[str, float] = {}
|
||||
assert should_nudge("resume", state, message_count=5, memory_count=0) is False
|
||||
assert should_nudge("resume", state, message_count=5, memory_count=3) is True
|
||||
|
||||
def test_resume_allowed_on_first_message(self):
|
||||
state: dict[str, float] = {}
|
||||
assert should_nudge("resume", state, message_count=1, memory_count=3) is True
|
||||
|
||||
def test_start_fires_on_first_message_with_memories(self):
|
||||
state: dict[str, float] = {}
|
||||
assert should_nudge("start", state, message_count=1, memory_count=3) is True
|
||||
|
||||
def test_start_requires_memories(self):
|
||||
state: dict[str, float] = {}
|
||||
assert should_nudge("start", state, message_count=1, memory_count=0) is False
|
||||
|
||||
def test_start_only_on_first_message(self):
|
||||
state: dict[str, float] = {}
|
||||
assert should_nudge("start", state, message_count=2, memory_count=3) is False
|
||||
|
||||
def test_invalid_type(self):
|
||||
state: dict[str, float] = {}
|
||||
assert should_nudge("invalid", state, message_count=3, memory_count=0) is False
|
||||
|
||||
|
||||
class TestFormatNudge:
|
||||
def test_correction(self):
|
||||
assert format_nudge("correction") == NUDGE_CORRECTION
|
||||
|
||||
def test_denial(self):
|
||||
assert format_nudge("denial") == NUDGE_DENIAL
|
||||
|
||||
def test_resume(self):
|
||||
assert format_nudge("resume") == NUDGE_RESUME
|
||||
|
||||
def test_completion(self):
|
||||
assert format_nudge("completion") == NUDGE_COMPLETION
|
||||
|
||||
def test_start(self):
|
||||
assert format_nudge("start") == NUDGE_START
|
||||
|
||||
def test_invalid(self):
|
||||
assert format_nudge("invalid") == ""
|
||||
@@ -28,7 +28,7 @@ class TestModelConfig:
|
||||
)
|
||||
assert cfg.alias == "local"
|
||||
assert cfg.model == "qwen3-32b"
|
||||
assert cfg.context_window == 131072 # default
|
||||
assert cfg.context_window == 32768 # default
|
||||
|
||||
def test_custom_context_window(self) -> None:
|
||||
cfg = ModelConfig(
|
||||
@@ -530,7 +530,9 @@ class TestWorkstreamModelParam:
|
||||
|
||||
captured_alias = None
|
||||
|
||||
def factory(ui: Any, model_alias: str | None = None, ws_id: str | None = None) -> Any:
|
||||
def factory(
|
||||
ui: Any, model_alias: str | None = None, ws_id: str | None = None, **kwargs: Any
|
||||
) -> Any:
|
||||
nonlocal captured_alias
|
||||
captured_alias = model_alias
|
||||
mock_session = MagicMock()
|
||||
@@ -544,7 +546,9 @@ class TestWorkstreamModelParam:
|
||||
def test_create_without_model(self) -> None:
|
||||
captured_alias = None
|
||||
|
||||
def factory(ui: Any, model_alias: str | None = None, ws_id: str | None = None) -> Any:
|
||||
def factory(
|
||||
ui: Any, model_alias: str | None = None, ws_id: str | None = None, **kwargs: Any
|
||||
) -> Any:
|
||||
nonlocal captured_alias
|
||||
captured_alias = model_alias
|
||||
mock_session = MagicMock()
|
||||
|
||||
@@ -0,0 +1,967 @@
|
||||
"""Tests for turnstone.core.oidc — OIDC authentication support."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import urllib.parse
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import jwt as pyjwt
|
||||
import pytest
|
||||
|
||||
from turnstone.core.oidc import (
|
||||
OIDCConfig,
|
||||
OIDCError,
|
||||
apply_role_mapping,
|
||||
build_authorize_url,
|
||||
discover_oidc,
|
||||
generate_pkce_pair,
|
||||
load_oidc_config,
|
||||
provision_oidc_user,
|
||||
validate_id_token,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_config(**overrides) -> OIDCConfig:
|
||||
"""Build a test OIDCConfig with sensible defaults."""
|
||||
defaults = {
|
||||
"enabled": True,
|
||||
"issuer": "https://idp.example.com",
|
||||
"client_id": "my-client",
|
||||
"client_secret": "my-secret",
|
||||
"scopes": "openid email profile",
|
||||
"provider_name": "TestIDP",
|
||||
"role_claim": "",
|
||||
"role_map": {},
|
||||
"password_enabled": True,
|
||||
"authorization_endpoint": "https://idp.example.com/authorize",
|
||||
"token_endpoint": "https://idp.example.com/token",
|
||||
"userinfo_endpoint": "https://idp.example.com/userinfo",
|
||||
"jwks_uri": "https://idp.example.com/.well-known/jwks.json",
|
||||
}
|
||||
defaults.update(overrides)
|
||||
return OIDCConfig(**defaults)
|
||||
|
||||
|
||||
def _mock_storage(**overrides):
|
||||
"""Build a MagicMock with sensible storage defaults."""
|
||||
s = MagicMock()
|
||||
s.get_oidc_identity.return_value = overrides.get("identity")
|
||||
s.get_user.return_value = overrides.get("user")
|
||||
s.get_user_by_username.return_value = overrides.get("user_by_username")
|
||||
s.get_role.return_value = overrides.get("role")
|
||||
return s
|
||||
|
||||
|
||||
def _mock_async_client(mock_get):
|
||||
"""Build a patched httpx.AsyncClient context manager for async tests."""
|
||||
|
||||
class _AsyncCtx:
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
pass
|
||||
|
||||
async def get(self, url):
|
||||
return await mock_get(url)
|
||||
|
||||
return _AsyncCtx()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config Loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLoadOIDCConfig:
|
||||
def test_load_oidc_config_from_env(self, monkeypatch):
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_SCOPES", "openid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_PROVIDER_NAME", "Okta")
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.enabled is True
|
||||
assert cfg.issuer == "https://auth.example.com"
|
||||
assert cfg.client_id == "cid"
|
||||
assert cfg.client_secret == "csecret"
|
||||
assert cfg.scopes == "openid"
|
||||
assert cfg.provider_name == "Okta"
|
||||
|
||||
def test_load_oidc_config_disabled_when_missing(self, monkeypatch):
|
||||
monkeypatch.delenv("TURNSTONE_OIDC_ISSUER", raising=False)
|
||||
monkeypatch.delenv("TURNSTONE_OIDC_CLIENT_ID", raising=False)
|
||||
monkeypatch.delenv("TURNSTONE_OIDC_CLIENT_SECRET", raising=False)
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.enabled is False
|
||||
|
||||
def test_load_oidc_config_partial_env(self, monkeypatch):
|
||||
"""Only issuer set, no client_id -> enabled=False."""
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.delenv("TURNSTONE_OIDC_CLIENT_ID", raising=False)
|
||||
monkeypatch.delenv("TURNSTONE_OIDC_CLIENT_SECRET", raising=False)
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.enabled is False
|
||||
assert cfg.issuer == "https://auth.example.com"
|
||||
assert cfg.client_id == ""
|
||||
|
||||
def test_load_oidc_config_role_map_parsing(self, monkeypatch):
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ROLE_CLAIM", "roles")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ROLE_MAP", "admin:builtin-admin,eng:builtin-operator")
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.role_claim == "roles"
|
||||
assert cfg.role_map == {"admin": "builtin-admin", "eng": "builtin-operator"}
|
||||
|
||||
def test_load_oidc_config_password_enabled_false(self, monkeypatch):
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_PASSWORD_ENABLED", "false")
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.enabled is True
|
||||
assert cfg.password_enabled is False
|
||||
|
||||
def test_load_oidc_config_password_enabled_true(self, monkeypatch):
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_PASSWORD_ENABLED", "true")
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.password_enabled is True
|
||||
|
||||
def test_load_oidc_config_role_map_empty_entries(self, monkeypatch):
|
||||
"""Role map with empty/whitespace entries should be silently skipped."""
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ROLE_MAP", "admin:builtin-admin, , :, foo:")
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.role_map == {"admin": "builtin-admin"}
|
||||
|
||||
def test_load_oidc_config_defaults(self, monkeypatch):
|
||||
"""Defaults for scopes and provider_name when not set."""
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.delenv("TURNSTONE_OIDC_SCOPES", raising=False)
|
||||
monkeypatch.delenv("TURNSTONE_OIDC_PROVIDER_NAME", raising=False)
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.scopes == "openid email profile"
|
||||
assert cfg.provider_name == "SSO"
|
||||
|
||||
def test_load_oidc_config_redirect_base_from_env(self, monkeypatch):
|
||||
"""TURNSTONE_OIDC_REDIRECT_BASE populates redirect_base."""
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_REDIRECT_BASE", "https://app.example.com")
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.redirect_base == "https://app.example.com"
|
||||
|
||||
def test_load_oidc_config_redirect_base_strips_trailing_slash(self, monkeypatch):
|
||||
"""Trailing slashes are stripped from redirect_base."""
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_REDIRECT_BASE", "https://app.example.com/")
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.redirect_base == "https://app.example.com"
|
||||
|
||||
def test_load_oidc_config_redirect_base_default_empty(self, monkeypatch):
|
||||
"""redirect_base defaults to empty string when not set."""
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.delenv("TURNSTONE_OIDC_REDIRECT_BASE", raising=False)
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.redirect_base == ""
|
||||
|
||||
def test_load_oidc_config_redirect_base_rejects_path(self, monkeypatch):
|
||||
"""redirect_base with a path component is rejected (falls back to empty)."""
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_REDIRECT_BASE", "https://app.example.com/subpath")
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.redirect_base == ""
|
||||
|
||||
def test_load_oidc_config_redirect_base_rejects_no_scheme(self, monkeypatch):
|
||||
"""redirect_base without a scheme is rejected."""
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_REDIRECT_BASE", "app.example.com")
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.redirect_base == ""
|
||||
|
||||
def test_load_oidc_config_redirect_base_rejects_userinfo(self, monkeypatch):
|
||||
"""redirect_base with userinfo (user:pass@host) is rejected."""
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_REDIRECT_BASE", "https://user:pass@app.example.com")
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.redirect_base == ""
|
||||
|
||||
def test_load_oidc_config_redirect_base_rejects_invalid_port(self, monkeypatch):
|
||||
"""redirect_base with non-numeric port is rejected."""
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_REDIRECT_BASE", "https://app.example.com:abc")
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.redirect_base == ""
|
||||
|
||||
def test_load_oidc_config_redirect_base_rejects_missing_hostname(self, monkeypatch):
|
||||
"""redirect_base without a hostname is rejected."""
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_REDIRECT_BASE", "https://")
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.redirect_base == ""
|
||||
|
||||
def test_load_oidc_config_redirect_base_allows_http(self, monkeypatch):
|
||||
"""http:// redirect_base is allowed (with warning) for local dev."""
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_ISSUER", "https://auth.example.com")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_ID", "cid")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_CLIENT_SECRET", "csecret")
|
||||
monkeypatch.setenv("TURNSTONE_OIDC_REDIRECT_BASE", "http://localhost:8000")
|
||||
|
||||
with patch("turnstone.core.config.load_config", return_value={}):
|
||||
cfg = load_oidc_config()
|
||||
|
||||
assert cfg.redirect_base == "http://localhost:8000"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Redirect URI Builder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBuildOIDCRedirectURI:
|
||||
"""Tests for ``_build_oidc_redirect_uri`` in auth.py."""
|
||||
|
||||
def _make_request(self, host="app.example.com", scheme="https", forwarded_proto=""):
|
||||
"""Build a minimal mock Starlette Request."""
|
||||
req = MagicMock()
|
||||
headers = {"host": host}
|
||||
if forwarded_proto:
|
||||
headers["x-forwarded-proto"] = forwarded_proto
|
||||
req.headers = headers
|
||||
req.url.scheme = scheme
|
||||
return req
|
||||
|
||||
def test_pinned_redirect_base(self):
|
||||
"""When redirect_base is set, Host header is ignored."""
|
||||
from turnstone.core.auth import _build_oidc_redirect_uri
|
||||
|
||||
config = _make_config(redirect_base="https://public.example.com")
|
||||
req = self._make_request(host="internal-host:8080", scheme="http")
|
||||
result = _build_oidc_redirect_uri(req, config)
|
||||
assert result == "https://public.example.com/v1/api/auth/oidc/callback"
|
||||
|
||||
def test_fallback_to_host_header(self):
|
||||
"""When redirect_base is empty, redirect URI uses Host header."""
|
||||
from turnstone.core.auth import _build_oidc_redirect_uri
|
||||
|
||||
config = _make_config(redirect_base="")
|
||||
req = self._make_request(host="app.example.com", scheme="https")
|
||||
result = _build_oidc_redirect_uri(req, config)
|
||||
assert result == "https://app.example.com/v1/api/auth/oidc/callback"
|
||||
|
||||
def test_fallback_x_forwarded_proto(self):
|
||||
"""When redirect_base is empty and X-Forwarded-Proto is https, scheme is https."""
|
||||
from turnstone.core.auth import _build_oidc_redirect_uri
|
||||
|
||||
config = _make_config(redirect_base="")
|
||||
req = self._make_request(host="app.example.com", scheme="http", forwarded_proto="https")
|
||||
result = _build_oidc_redirect_uri(req, config)
|
||||
assert result == "https://app.example.com/v1/api/auth/oidc/callback"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PKCE
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPKCE:
|
||||
def test_generate_pkce_pair(self):
|
||||
verifier, challenge = generate_pkce_pair()
|
||||
|
||||
# Verifier should be URL-safe base64
|
||||
assert isinstance(verifier, str)
|
||||
assert len(verifier) > 40 # 48 bytes -> ~64 chars
|
||||
|
||||
# Challenge should be base64url SHA-256 of verifier
|
||||
expected_digest = hashlib.sha256(verifier.encode("ascii")).digest()
|
||||
expected_challenge = base64.urlsafe_b64encode(expected_digest).rstrip(b"=").decode("ascii")
|
||||
assert challenge == expected_challenge
|
||||
|
||||
def test_pkce_challenge_matches_verifier(self):
|
||||
"""Manually compute challenge and verify it matches."""
|
||||
verifier, challenge = generate_pkce_pair()
|
||||
digest = hashlib.sha256(verifier.encode("ascii")).digest()
|
||||
manual_challenge = base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii")
|
||||
assert challenge == manual_challenge
|
||||
|
||||
def test_pkce_pair_uniqueness(self):
|
||||
"""Each call should produce a unique pair."""
|
||||
v1, c1 = generate_pkce_pair()
|
||||
v2, c2 = generate_pkce_pair()
|
||||
assert v1 != v2
|
||||
assert c1 != c2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Authorization URL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBuildAuthorizeURL:
|
||||
def test_build_authorize_url_contains_required_params(self):
|
||||
config = _make_config()
|
||||
verifier, _ = generate_pkce_pair()
|
||||
url = build_authorize_url(
|
||||
config=config,
|
||||
redirect_uri="https://app.example.com/callback",
|
||||
state="test-state",
|
||||
nonce="test-nonce",
|
||||
code_verifier=verifier,
|
||||
)
|
||||
|
||||
assert url.startswith("https://idp.example.com/authorize?")
|
||||
assert "response_type=code" in url
|
||||
assert "client_id=my-client" in url
|
||||
assert "redirect_uri=" in url
|
||||
assert "scope=openid" in url
|
||||
assert "state=test-state" in url
|
||||
assert "nonce=test-nonce" in url
|
||||
assert "code_challenge=" in url
|
||||
assert "code_challenge_method=S256" in url
|
||||
|
||||
def test_build_authorize_url_pkce(self):
|
||||
"""code_challenge in URL should be correct S256 of the verifier."""
|
||||
config = _make_config()
|
||||
verifier, _ = generate_pkce_pair()
|
||||
|
||||
url = build_authorize_url(
|
||||
config=config,
|
||||
redirect_uri="https://app.example.com/callback",
|
||||
state="s",
|
||||
nonce="n",
|
||||
code_verifier=verifier,
|
||||
)
|
||||
|
||||
# Extract code_challenge from URL
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
params = urllib.parse.parse_qs(parsed.query)
|
||||
actual_challenge = params["code_challenge"][0]
|
||||
|
||||
# Compute expected challenge
|
||||
digest = hashlib.sha256(verifier.encode("ascii")).digest()
|
||||
expected = base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii")
|
||||
assert actual_challenge == expected
|
||||
|
||||
def test_build_authorize_url_redirect_uri_encoded(self):
|
||||
config = _make_config()
|
||||
verifier, _ = generate_pkce_pair()
|
||||
redirect = "https://app.example.com/callback?extra=1"
|
||||
|
||||
url = build_authorize_url(
|
||||
config=config,
|
||||
redirect_uri=redirect,
|
||||
state="s",
|
||||
nonce="n",
|
||||
code_verifier=verifier,
|
||||
)
|
||||
|
||||
# The redirect_uri should be URL-encoded
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
params = urllib.parse.parse_qs(parsed.query)
|
||||
assert params["redirect_uri"][0] == redirect
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ID Token Validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestValidateIDToken:
|
||||
_FAKE_JWKS = {"keys": [{"kid": "key1", "kty": "RSA", "n": "abc", "e": "AQAB"}]}
|
||||
|
||||
def test_validate_id_token_nonce_mismatch(self):
|
||||
"""Nonce mismatch should raise OIDCError."""
|
||||
config = _make_config()
|
||||
|
||||
mock_pyjwk = MagicMock()
|
||||
mock_pyjwk.return_value.key = "fake-key"
|
||||
|
||||
with (
|
||||
patch("jwt.get_unverified_header", return_value={"kid": "key1", "alg": "RS256"}),
|
||||
patch("jwt.PyJWK", mock_pyjwk),
|
||||
patch("jwt.decode", return_value={"sub": "user1", "nonce": "wrong-nonce"}),
|
||||
pytest.raises(OIDCError, match="nonce mismatch"),
|
||||
):
|
||||
validate_id_token(
|
||||
raw_token="fake.jwt.token",
|
||||
jwks_data=self._FAKE_JWKS,
|
||||
config=config,
|
||||
nonce="expected-nonce",
|
||||
)
|
||||
|
||||
def test_validate_id_token_success(self):
|
||||
"""Successful validation returns decoded claims."""
|
||||
config = _make_config()
|
||||
|
||||
mock_pyjwk = MagicMock()
|
||||
mock_pyjwk.return_value.key = "fake-key"
|
||||
|
||||
expected_claims = {
|
||||
"sub": "user1",
|
||||
"email": "user@example.com",
|
||||
"nonce": "test-nonce",
|
||||
}
|
||||
|
||||
with (
|
||||
patch("jwt.get_unverified_header", return_value={"kid": "key1", "alg": "RS256"}),
|
||||
patch("jwt.PyJWK", mock_pyjwk),
|
||||
patch("jwt.decode", return_value=expected_claims) as mock_decode,
|
||||
):
|
||||
claims = validate_id_token(
|
||||
raw_token="fake.jwt.token",
|
||||
jwks_data=self._FAKE_JWKS,
|
||||
config=config,
|
||||
nonce="test-nonce",
|
||||
)
|
||||
|
||||
assert claims == expected_claims
|
||||
mock_decode.assert_called_once_with(
|
||||
"fake.jwt.token",
|
||||
"fake-key",
|
||||
algorithms=[
|
||||
"RS256",
|
||||
"RS384",
|
||||
"RS512",
|
||||
"ES256",
|
||||
"ES384",
|
||||
"ES512",
|
||||
"PS256",
|
||||
"PS384",
|
||||
"PS512",
|
||||
],
|
||||
audience="my-client",
|
||||
issuer="https://idp.example.com",
|
||||
)
|
||||
|
||||
def test_validate_id_token_kid_not_found(self):
|
||||
"""Unknown kid raises OIDCError with descriptive message."""
|
||||
config = _make_config()
|
||||
jwks_data = {"keys": [{"kid": "other-key", "kty": "RSA"}]}
|
||||
|
||||
with (
|
||||
patch("jwt.get_unverified_header", return_value={"kid": "unknown", "alg": "RS256"}),
|
||||
pytest.raises(OIDCError, match="not found in JWKS"),
|
||||
):
|
||||
validate_id_token(
|
||||
raw_token="bad.token",
|
||||
jwks_data=jwks_data,
|
||||
config=config,
|
||||
nonce="n",
|
||||
)
|
||||
|
||||
def test_validate_id_token_invalid_jwt(self):
|
||||
"""Invalid JWT raises OIDCError."""
|
||||
config = _make_config()
|
||||
|
||||
mock_pyjwk = MagicMock()
|
||||
mock_pyjwk.return_value.key = "fake-key"
|
||||
|
||||
with (
|
||||
patch("jwt.get_unverified_header", return_value={"kid": "key1", "alg": "RS256"}),
|
||||
patch("jwt.PyJWK", mock_pyjwk := MagicMock(return_value=MagicMock(key="fake-key"))),
|
||||
patch("jwt.decode", side_effect=pyjwt.InvalidTokenError("expired")),
|
||||
pytest.raises(OIDCError, match="ID token validation failed"),
|
||||
):
|
||||
validate_id_token(
|
||||
raw_token="expired.token",
|
||||
jwks_data=self._FAKE_JWKS,
|
||||
config=config,
|
||||
nonce="n",
|
||||
)
|
||||
|
||||
def test_validate_id_token_invalid_header(self):
|
||||
"""Malformed token header raises OIDCError."""
|
||||
config = _make_config()
|
||||
|
||||
with (
|
||||
patch("jwt.get_unverified_header", side_effect=pyjwt.DecodeError("bad header")),
|
||||
pytest.raises(OIDCError, match="Invalid ID token header"),
|
||||
):
|
||||
validate_id_token(
|
||||
raw_token="garbage",
|
||||
jwks_data=self._FAKE_JWKS,
|
||||
config=config,
|
||||
nonce="n",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# User Provisioning
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestProvisionOIDCUser:
|
||||
def test_provision_oidc_user_existing(self):
|
||||
"""Existing identity -> returns existing user, updates last_login."""
|
||||
config = _make_config()
|
||||
existing_user = {
|
||||
"user_id": "u1",
|
||||
"username": "alice",
|
||||
"display_name": "Alice",
|
||||
"password_hash": "!oidc",
|
||||
}
|
||||
existing_identity = {
|
||||
"issuer": "https://idp.example.com",
|
||||
"subject": "sub-123",
|
||||
"user_id": "u1",
|
||||
"email": "alice@example.com",
|
||||
"created": "2024-01-01T00:00:00",
|
||||
"last_login": "2024-01-01T00:00:00",
|
||||
}
|
||||
storage = _mock_storage(identity=existing_identity, user=existing_user)
|
||||
|
||||
claims = {"sub": "sub-123", "email": "alice@example.com", "name": "Alice"}
|
||||
user = provision_oidc_user(storage, config, claims)
|
||||
|
||||
assert user["user_id"] == "u1"
|
||||
assert user["username"] == "alice"
|
||||
storage.update_oidc_identity_login.assert_called_once()
|
||||
# Should not create a new user
|
||||
storage.create_user.assert_not_called()
|
||||
storage.create_oidc_identity.assert_not_called()
|
||||
|
||||
def test_provision_oidc_user_new(self):
|
||||
"""No identity -> creates user + identity."""
|
||||
config = _make_config()
|
||||
storage = _mock_storage()
|
||||
|
||||
# After create_user, get_user should return the new user
|
||||
new_user = {
|
||||
"user_id": "u-new",
|
||||
"username": "bob",
|
||||
"display_name": "Bob",
|
||||
"password_hash": "!oidc",
|
||||
}
|
||||
storage.get_user.return_value = new_user
|
||||
|
||||
claims = {"sub": "sub-456", "preferred_username": "bob", "email": "bob@example.com"}
|
||||
|
||||
with patch("turnstone.core.oidc.uuid") as mock_uuid:
|
||||
mock_uuid.uuid4.return_value = MagicMock(hex="u-new-hex-00000000000000000000")
|
||||
user = provision_oidc_user(storage, config, claims)
|
||||
|
||||
assert user["username"] == "bob"
|
||||
storage.create_user.assert_called_once()
|
||||
storage.create_oidc_identity.assert_called_once()
|
||||
# Verify create_oidc_identity was called with correct issuer and sub
|
||||
call_args = storage.create_oidc_identity.call_args
|
||||
assert call_args[0][0] == "https://idp.example.com" # issuer
|
||||
assert call_args[0][1] == "sub-456" # subject
|
||||
|
||||
def test_provision_oidc_user_username_dedup(self):
|
||||
"""First username taken -> appends suffix."""
|
||||
config = _make_config()
|
||||
storage = _mock_storage()
|
||||
|
||||
# First call: username "bob" exists; second call: "bob2" doesn't exist
|
||||
storage.get_user_by_username.side_effect = [
|
||||
{"user_id": "u-other", "username": "bob"}, # "bob" taken
|
||||
None, # "bob2" available
|
||||
]
|
||||
new_user = {
|
||||
"user_id": "u-new",
|
||||
"username": "bob2",
|
||||
"display_name": "Bob",
|
||||
"password_hash": "!oidc",
|
||||
}
|
||||
storage.get_user.return_value = new_user
|
||||
|
||||
claims = {"sub": "sub-789", "preferred_username": "bob", "email": "bob@example.com"}
|
||||
user = provision_oidc_user(storage, config, claims)
|
||||
|
||||
assert user["username"] == "bob2"
|
||||
# create_user should have been called with "bob2" as username
|
||||
call_args = storage.create_user.call_args
|
||||
assert call_args[0][1] == "bob2"
|
||||
|
||||
def test_provision_oidc_user_email_prefix(self):
|
||||
"""No preferred_username -> uses email prefix."""
|
||||
config = _make_config()
|
||||
storage = _mock_storage()
|
||||
|
||||
new_user = {
|
||||
"user_id": "u-new",
|
||||
"username": "charlie",
|
||||
"display_name": "charlie@example.com",
|
||||
"password_hash": "!oidc",
|
||||
}
|
||||
storage.get_user.return_value = new_user
|
||||
|
||||
claims = {"sub": "sub-abc", "email": "charlie@example.com"}
|
||||
provision_oidc_user(storage, config, claims)
|
||||
|
||||
# create_user should have been called with "charlie" (email prefix)
|
||||
call_args = storage.create_user.call_args
|
||||
assert call_args[0][1] == "charlie"
|
||||
|
||||
def test_provision_oidc_user_missing_user_raises(self):
|
||||
"""Identity references missing user -> raises OIDCError."""
|
||||
config = _make_config()
|
||||
existing_identity = {
|
||||
"issuer": "https://idp.example.com",
|
||||
"subject": "sub-orphan",
|
||||
"user_id": "u-gone",
|
||||
"email": "gone@example.com",
|
||||
"created": "2024-01-01T00:00:00",
|
||||
"last_login": "2024-01-01T00:00:00",
|
||||
}
|
||||
storage = _mock_storage(identity=existing_identity, user=None)
|
||||
|
||||
claims = {"sub": "sub-orphan", "email": "gone@example.com"}
|
||||
with pytest.raises(OIDCError, match="missing user"):
|
||||
provision_oidc_user(storage, config, claims)
|
||||
|
||||
def test_provision_oidc_user_fallback_username(self):
|
||||
"""No preferred_username and no email -> falls back to 'user'."""
|
||||
config = _make_config()
|
||||
storage = _mock_storage()
|
||||
new_user = {
|
||||
"user_id": "u-new",
|
||||
"username": "user",
|
||||
"display_name": "",
|
||||
"password_hash": "!oidc",
|
||||
}
|
||||
storage.get_user.return_value = new_user
|
||||
|
||||
claims = {"sub": "sub-noemail"}
|
||||
provision_oidc_user(storage, config, claims)
|
||||
|
||||
call_args = storage.create_user.call_args
|
||||
assert call_args[0][1] == "user"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Role Mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestApplyRoleMapping:
|
||||
def test_apply_role_mapping_basic(self):
|
||||
"""Maps claim value to role."""
|
||||
config = _make_config(
|
||||
role_claim="groups",
|
||||
role_map={"admin": "builtin-admin"},
|
||||
)
|
||||
storage = _mock_storage(role={"role_id": "builtin-admin", "name": "Admin"})
|
||||
|
||||
claims = {"sub": "u1", "groups": "admin"}
|
||||
apply_role_mapping(storage, "u1", claims, config)
|
||||
|
||||
storage.assign_role.assert_called_once_with("u1", "builtin-admin", "oidc")
|
||||
|
||||
def test_apply_role_mapping_list_claim(self):
|
||||
"""Claim is a list of strings -> maps each."""
|
||||
config = _make_config(
|
||||
role_claim="roles",
|
||||
role_map={"admin": "builtin-admin", "editor": "builtin-operator"},
|
||||
)
|
||||
storage = _mock_storage()
|
||||
# get_role returns non-None for both roles
|
||||
storage.get_role.return_value = {"role_id": "some-role"}
|
||||
|
||||
claims = {"sub": "u1", "roles": ["admin", "editor"]}
|
||||
apply_role_mapping(storage, "u1", claims, config)
|
||||
|
||||
assert storage.assign_role.call_count == 2
|
||||
|
||||
def test_apply_role_mapping_no_config(self):
|
||||
"""No role_claim configured -> no-op."""
|
||||
config = _make_config(role_claim="", role_map={})
|
||||
storage = _mock_storage()
|
||||
|
||||
claims = {"sub": "u1", "roles": "admin"}
|
||||
apply_role_mapping(storage, "u1", claims, config)
|
||||
|
||||
storage.assign_role.assert_not_called()
|
||||
|
||||
def test_apply_role_mapping_unknown_role(self):
|
||||
"""Claim maps to nonexistent role -> skipped."""
|
||||
config = _make_config(
|
||||
role_claim="groups",
|
||||
role_map={"admin": "nonexistent-role"},
|
||||
)
|
||||
storage = _mock_storage(role=None) # role doesn't exist
|
||||
|
||||
claims = {"sub": "u1", "groups": "admin"}
|
||||
apply_role_mapping(storage, "u1", claims, config)
|
||||
|
||||
storage.assign_role.assert_not_called()
|
||||
|
||||
def test_apply_role_mapping_no_matching_claim_value(self):
|
||||
"""Claim value not in role_map -> no assignment."""
|
||||
config = _make_config(
|
||||
role_claim="groups",
|
||||
role_map={"admin": "builtin-admin"},
|
||||
)
|
||||
storage = _mock_storage()
|
||||
|
||||
claims = {"sub": "u1", "groups": "viewer"} # "viewer" not in role_map
|
||||
apply_role_mapping(storage, "u1", claims, config)
|
||||
|
||||
storage.assign_role.assert_not_called()
|
||||
|
||||
def test_apply_role_mapping_claim_missing(self):
|
||||
"""Claim key not present in claims -> no-op."""
|
||||
config = _make_config(
|
||||
role_claim="groups",
|
||||
role_map={"admin": "builtin-admin"},
|
||||
)
|
||||
storage = _mock_storage()
|
||||
|
||||
claims = {"sub": "u1"} # no "groups" key
|
||||
apply_role_mapping(storage, "u1", claims, config)
|
||||
|
||||
storage.assign_role.assert_not_called()
|
||||
|
||||
def test_apply_role_mapping_no_role_map(self):
|
||||
"""role_claim set but role_map empty -> no-op (early return)."""
|
||||
config = _make_config(role_claim="groups", role_map={})
|
||||
storage = _mock_storage()
|
||||
|
||||
claims = {"sub": "u1", "groups": "admin"}
|
||||
apply_role_mapping(storage, "u1", claims, config)
|
||||
|
||||
storage.assign_role.assert_not_called()
|
||||
|
||||
def test_apply_role_mapping_revokes_stale_oidc_roles(self):
|
||||
"""Roles previously assigned by OIDC but no longer in claims are revoked."""
|
||||
config = _make_config(
|
||||
role_claim="groups",
|
||||
role_map={"admin": "builtin-admin", "eng": "builtin-operator"},
|
||||
)
|
||||
storage = _mock_storage()
|
||||
storage.get_role.return_value = {"role_id": "some-role"}
|
||||
# User currently has admin (via OIDC) and a manual role
|
||||
storage.list_user_roles.return_value = [
|
||||
{"role_id": "builtin-admin", "assigned_by": "oidc"},
|
||||
{"role_id": "custom-role", "assigned_by": "admin-ui"},
|
||||
]
|
||||
|
||||
# IdP now only says "eng", not "admin"
|
||||
claims = {"sub": "u1", "groups": ["eng"]}
|
||||
apply_role_mapping(storage, "u1", claims, config)
|
||||
|
||||
# builtin-admin should be revoked (OIDC-assigned, no longer in claims)
|
||||
storage.unassign_role.assert_called_once_with("u1", "builtin-admin")
|
||||
# custom-role should NOT be revoked (not assigned by OIDC)
|
||||
|
||||
def test_apply_role_mapping_preserves_manual_roles(self):
|
||||
"""Manually assigned roles are never revoked by OIDC sync."""
|
||||
config = _make_config(
|
||||
role_claim="groups",
|
||||
role_map={"admin": "builtin-admin"},
|
||||
)
|
||||
storage = _mock_storage()
|
||||
storage.get_role.return_value = {"role_id": "some-role"}
|
||||
storage.list_user_roles.return_value = [
|
||||
{"role_id": "builtin-admin", "assigned_by": "admin-ui"},
|
||||
]
|
||||
|
||||
# Claims have no groups at all
|
||||
claims = {"sub": "u1"}
|
||||
apply_role_mapping(storage, "u1", claims, config)
|
||||
|
||||
# Manual admin role must NOT be revoked
|
||||
storage.unassign_role.assert_not_called()
|
||||
|
||||
def test_apply_role_mapping_revokes_all_oidc_roles_when_claim_absent(self):
|
||||
"""When the claim is absent from the token, all OIDC-assigned roles are revoked."""
|
||||
config = _make_config(
|
||||
role_claim="groups",
|
||||
role_map={"admin": "builtin-admin"},
|
||||
)
|
||||
storage = _mock_storage()
|
||||
storage.get_role.return_value = {"role_id": "some-role"}
|
||||
storage.list_user_roles.return_value = [
|
||||
{"role_id": "builtin-admin", "assigned_by": "oidc"},
|
||||
{"role_id": "builtin-operator", "assigned_by": "oidc"},
|
||||
]
|
||||
|
||||
claims = {"sub": "u1"} # no "groups" key
|
||||
apply_role_mapping(storage, "u1", claims, config)
|
||||
|
||||
assert storage.unassign_role.call_count == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Discovery (async)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDiscoverOIDC:
|
||||
def test_discover_oidc_success(self):
|
||||
"""Mock httpx response, verify endpoints populated."""
|
||||
config = _make_config(
|
||||
authorization_endpoint="",
|
||||
token_endpoint="",
|
||||
userinfo_endpoint="",
|
||||
jwks_uri="",
|
||||
)
|
||||
|
||||
discovery_doc = {
|
||||
"authorization_endpoint": "https://idp.example.com/authorize",
|
||||
"token_endpoint": "https://idp.example.com/token",
|
||||
"userinfo_endpoint": "https://idp.example.com/userinfo",
|
||||
"jwks_uri": "https://idp.example.com/.well-known/jwks.json",
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = discovery_doc
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
|
||||
async def _run():
|
||||
client = _mock_async_client(lambda url: _async_return(mock_response))
|
||||
with patch("httpx.AsyncClient", return_value=client):
|
||||
result = await discover_oidc(config)
|
||||
|
||||
assert result.authorization_endpoint == "https://idp.example.com/authorize"
|
||||
assert result.token_endpoint == "https://idp.example.com/token"
|
||||
assert result.userinfo_endpoint == "https://idp.example.com/userinfo"
|
||||
assert result.jwks_uri == "https://idp.example.com/.well-known/jwks.json"
|
||||
assert result.enabled is True
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
def test_discover_oidc_failure(self):
|
||||
"""Mock httpx error -> enabled=False returned."""
|
||||
config = _make_config(
|
||||
authorization_endpoint="",
|
||||
token_endpoint="",
|
||||
userinfo_endpoint="",
|
||||
jwks_uri="",
|
||||
)
|
||||
|
||||
async def _failing_get(url):
|
||||
raise httpx.ConnectError("connection refused")
|
||||
|
||||
async def _run():
|
||||
client = _mock_async_client(_failing_get)
|
||||
with patch("httpx.AsyncClient", return_value=client):
|
||||
result = await discover_oidc(config)
|
||||
|
||||
assert result.enabled is False
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
def test_discover_oidc_no_issuer(self):
|
||||
"""Empty issuer -> enabled=False."""
|
||||
config = _make_config(issuer="")
|
||||
|
||||
async def _run():
|
||||
result = await discover_oidc(config)
|
||||
assert result.enabled is False
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
def test_discover_oidc_missing_required_endpoints(self):
|
||||
"""Discovery doc missing authorization_endpoint -> enabled=False."""
|
||||
config = _make_config(
|
||||
authorization_endpoint="",
|
||||
token_endpoint="",
|
||||
userinfo_endpoint="",
|
||||
jwks_uri="",
|
||||
)
|
||||
|
||||
# Document missing authorization_endpoint
|
||||
discovery_doc = {
|
||||
"token_endpoint": "https://idp.example.com/token",
|
||||
"jwks_uri": "https://idp.example.com/.well-known/jwks.json",
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = discovery_doc
|
||||
mock_response.raise_for_status = MagicMock()
|
||||
|
||||
async def _run():
|
||||
client = _mock_async_client(lambda url: _async_return(mock_response))
|
||||
with patch("httpx.AsyncClient", return_value=client):
|
||||
result = await discover_oidc(config)
|
||||
|
||||
assert result.enabled is False
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
async def _async_return(value):
|
||||
"""Helper: return a value from an async function."""
|
||||
return value
|
||||
@@ -0,0 +1,581 @@
|
||||
"""Integration tests for OIDC HTTP handlers (authorize, callback, admin endpoints).
|
||||
|
||||
Uses Starlette TestClient with real SQLiteBackend storage. External OIDC
|
||||
functions (exchange_code, validate_id_token, etc.) are mocked — the focus is
|
||||
on the HTTP handler logic, request/response wiring, and storage side-effects.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import urllib.parse
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.middleware import Middleware
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.routing import Mount, Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
from turnstone.console.server import (
|
||||
admin_delete_oidc_identity,
|
||||
admin_list_oidc_identities,
|
||||
)
|
||||
from turnstone.core.auth import (
|
||||
AuthResult,
|
||||
LoginRateLimiter,
|
||||
handle_oidc_authorize,
|
||||
handle_oidc_callback,
|
||||
)
|
||||
from turnstone.core.oidc import OIDCConfig, OIDCError
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_oidc_config(**overrides: Any) -> OIDCConfig:
|
||||
"""Build a test OIDCConfig with sensible defaults."""
|
||||
defaults: dict[str, Any] = {
|
||||
"enabled": True,
|
||||
"issuer": "https://idp.example.com",
|
||||
"client_id": "my-client",
|
||||
"client_secret": "my-secret",
|
||||
"scopes": "openid email profile",
|
||||
"provider_name": "TestIDP",
|
||||
"role_claim": "",
|
||||
"role_map": {},
|
||||
"password_enabled": True,
|
||||
"authorization_endpoint": "https://idp.example.com/authorize",
|
||||
"token_endpoint": "https://idp.example.com/token",
|
||||
"userinfo_endpoint": "https://idp.example.com/userinfo",
|
||||
"jwks_uri": "https://idp.example.com/.well-known/jwks.json",
|
||||
}
|
||||
defaults.update(overrides)
|
||||
return OIDCConfig(**defaults)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Thin handler wrappers — match the pattern used in server.py / console
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _oidc_authorize(request: Request) -> Response:
|
||||
return await handle_oidc_authorize(request, "test-audience")
|
||||
|
||||
|
||||
async def _oidc_callback(request: Request) -> Response:
|
||||
return await handle_oidc_callback(request, "test-audience")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth bypass middleware for admin endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _InjectAuthMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request: Request, call_next: Any) -> Response:
|
||||
request.state.auth_result = AuthResult(
|
||||
user_id="test-admin",
|
||||
scopes=frozenset({"approve"}),
|
||||
token_source="config",
|
||||
permissions=frozenset(
|
||||
{
|
||||
"read",
|
||||
"write",
|
||||
"approve",
|
||||
"admin.users",
|
||||
}
|
||||
),
|
||||
)
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path: Any) -> SQLiteBackend:
|
||||
"""Fresh SQLite backend with a seeded admin user."""
|
||||
backend = SQLiteBackend(str(tmp_path / "test.db"))
|
||||
backend.create_user("test-admin", "testadmin", "Test Admin", "hash")
|
||||
return backend
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def oidc_config() -> OIDCConfig:
|
||||
return _make_oidc_config()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def authorize_client(storage: SQLiteBackend, oidc_config: OIDCConfig) -> TestClient:
|
||||
"""TestClient wired to the OIDC authorize + callback handlers."""
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Mount(
|
||||
"/v1",
|
||||
routes=[
|
||||
Route("/api/auth/oidc/authorize", _oidc_authorize),
|
||||
Route("/api/auth/oidc/callback", _oidc_callback),
|
||||
],
|
||||
),
|
||||
],
|
||||
)
|
||||
app.state.oidc_config = oidc_config
|
||||
app.state.auth_storage = storage
|
||||
app.state.jwt_secret = "test-jwt-secret"
|
||||
app.state.jwks_data = {"keys": []}
|
||||
app.state.login_limiter = None
|
||||
return TestClient(app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def admin_client(storage: SQLiteBackend) -> TestClient:
|
||||
"""TestClient wired to the admin OIDC identity endpoints."""
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Mount(
|
||||
"/v1",
|
||||
routes=[
|
||||
Route(
|
||||
"/api/admin/users/{user_id}/oidc-identities",
|
||||
admin_list_oidc_identities,
|
||||
),
|
||||
Route(
|
||||
"/api/admin/oidc-identities",
|
||||
admin_delete_oidc_identity,
|
||||
methods=["DELETE"],
|
||||
),
|
||||
],
|
||||
),
|
||||
],
|
||||
middleware=[Middleware(_InjectAuthMiddleware)],
|
||||
)
|
||||
app.state.auth_storage = storage
|
||||
return TestClient(app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /authorize tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOIDCAuthorize:
|
||||
"""Tests for GET /v1/api/auth/oidc/authorize."""
|
||||
|
||||
def test_happy_path_redirects_to_idp(self, authorize_client: TestClient) -> None:
|
||||
resp = authorize_client.get("/v1/api/auth/oidc/authorize", follow_redirects=False)
|
||||
assert resp.status_code == 302
|
||||
location = resp.headers["location"]
|
||||
assert location.startswith("https://idp.example.com/authorize?")
|
||||
parsed = urllib.parse.urlparse(location)
|
||||
params = urllib.parse.parse_qs(parsed.query)
|
||||
assert params["response_type"] == ["code"]
|
||||
assert params["client_id"] == ["my-client"]
|
||||
assert params["scope"] == ["openid email profile"]
|
||||
assert "state" in params
|
||||
assert "nonce" in params
|
||||
assert "code_challenge" in params
|
||||
assert params["code_challenge_method"] == ["S256"]
|
||||
|
||||
def test_oidc_not_configured_returns_404(
|
||||
self,
|
||||
storage: SQLiteBackend,
|
||||
) -> None:
|
||||
app = Starlette(
|
||||
routes=[Mount("/v1", routes=[Route("/api/auth/oidc/authorize", _oidc_authorize)])]
|
||||
)
|
||||
app.state.auth_storage = storage
|
||||
# No oidc_config at all
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
resp = client.get("/v1/api/auth/oidc/authorize")
|
||||
assert resp.status_code == 404
|
||||
assert resp.json()["error"] == "OIDC not configured"
|
||||
|
||||
def test_oidc_not_enabled_returns_404(
|
||||
self,
|
||||
storage: SQLiteBackend,
|
||||
) -> None:
|
||||
app = Starlette(
|
||||
routes=[Mount("/v1", routes=[Route("/api/auth/oidc/authorize", _oidc_authorize)])]
|
||||
)
|
||||
app.state.oidc_config = _make_oidc_config(enabled=False)
|
||||
app.state.auth_storage = storage
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
resp = client.get("/v1/api/auth/oidc/authorize")
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_no_storage_returns_503(self) -> None:
|
||||
app = Starlette(
|
||||
routes=[Mount("/v1", routes=[Route("/api/auth/oidc/authorize", _oidc_authorize)])]
|
||||
)
|
||||
app.state.oidc_config = _make_oidc_config()
|
||||
app.state.login_limiter = None
|
||||
# No auth_storage
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
resp = client.get("/v1/api/auth/oidc/authorize")
|
||||
assert resp.status_code == 503
|
||||
|
||||
def test_no_users_returns_403(self, tmp_path: Any) -> None:
|
||||
backend = SQLiteBackend(str(tmp_path / "empty.db"))
|
||||
app = Starlette(
|
||||
routes=[Mount("/v1", routes=[Route("/api/auth/oidc/authorize", _oidc_authorize)])]
|
||||
)
|
||||
app.state.oidc_config = _make_oidc_config()
|
||||
app.state.auth_storage = backend
|
||||
app.state.login_limiter = None
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
resp = client.get("/v1/api/auth/oidc/authorize")
|
||||
assert resp.status_code == 403
|
||||
assert "setup" in resp.json()["error"].lower()
|
||||
|
||||
def test_pending_state_persisted(
|
||||
self,
|
||||
authorize_client: TestClient,
|
||||
storage: SQLiteBackend,
|
||||
) -> None:
|
||||
resp = authorize_client.get("/v1/api/auth/oidc/authorize", follow_redirects=False)
|
||||
assert resp.status_code == 302
|
||||
location = resp.headers["location"]
|
||||
parsed = urllib.parse.urlparse(location)
|
||||
params = urllib.parse.parse_qs(parsed.query)
|
||||
state = params["state"][0]
|
||||
# The pending state should be retrievable from storage
|
||||
pending = storage.pop_oidc_pending_state(state, max_age_seconds=300)
|
||||
assert pending is not None
|
||||
assert pending["audience"] == "test-audience"
|
||||
assert pending["nonce"] != ""
|
||||
assert pending["code_verifier"] != ""
|
||||
|
||||
def test_rate_limited_redirects_with_error(self, storage: SQLiteBackend) -> None:
|
||||
app = Starlette(
|
||||
routes=[Mount("/v1", routes=[Route("/api/auth/oidc/authorize", _oidc_authorize)])]
|
||||
)
|
||||
app.state.oidc_config = _make_oidc_config()
|
||||
app.state.auth_storage = storage
|
||||
limiter = LoginRateLimiter(max_attempts=1, window_seconds=300)
|
||||
# Exhaust the rate limit
|
||||
limiter.record("ip:testclient")
|
||||
app.state.login_limiter = limiter
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
resp = client.get("/v1/api/auth/oidc/authorize", follow_redirects=False)
|
||||
assert resp.status_code == 302
|
||||
assert "oidc_error" in resp.headers["location"]
|
||||
assert "Too+many" in resp.headers["location"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /callback tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOIDCCallback:
|
||||
"""Tests for GET /v1/api/auth/oidc/callback."""
|
||||
|
||||
def _seed_pending_state(
|
||||
self,
|
||||
storage: SQLiteBackend,
|
||||
state: str = "valid-state",
|
||||
nonce: str = "test-nonce",
|
||||
code_verifier: str = "test-verifier",
|
||||
audience: str = "test-audience",
|
||||
) -> None:
|
||||
storage.create_oidc_pending_state(state, nonce, code_verifier, audience)
|
||||
|
||||
@patch("turnstone.core.oidc.provision_oidc_user")
|
||||
@patch("turnstone.core.oidc.validate_id_token")
|
||||
@patch("turnstone.core.oidc.exchange_code", new_callable=AsyncMock)
|
||||
def test_happy_path(
|
||||
self,
|
||||
mock_exchange: AsyncMock,
|
||||
mock_validate: Any,
|
||||
mock_provision: Any,
|
||||
authorize_client: TestClient,
|
||||
storage: SQLiteBackend,
|
||||
) -> None:
|
||||
self._seed_pending_state(storage)
|
||||
mock_exchange.return_value = {"id_token": "fake.jwt.token", "access_token": "at"}
|
||||
mock_validate.return_value = {
|
||||
"sub": "user123",
|
||||
"email": "u@example.com",
|
||||
"nonce": "test-nonce",
|
||||
}
|
||||
mock_provision.return_value = {"user_id": "test-admin", "username": "testadmin"}
|
||||
|
||||
resp = authorize_client.get(
|
||||
"/v1/api/auth/oidc/callback?code=authcode&state=valid-state",
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert resp.status_code == 302
|
||||
assert "oidc_success=1" in resp.headers["location"]
|
||||
assert "set-cookie" in resp.headers
|
||||
assert "turnstone_auth=" in resp.headers["set-cookie"]
|
||||
|
||||
def test_oidc_not_configured_returns_404(self, storage: SQLiteBackend) -> None:
|
||||
app = Starlette(
|
||||
routes=[Mount("/v1", routes=[Route("/api/auth/oidc/callback", _oidc_callback)])]
|
||||
)
|
||||
app.state.auth_storage = storage
|
||||
# No oidc_config
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
resp = client.get("/v1/api/auth/oidc/callback?code=x&state=y")
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_idp_error_param_redirects(
|
||||
self,
|
||||
authorize_client: TestClient,
|
||||
) -> None:
|
||||
resp = authorize_client.get(
|
||||
"/v1/api/auth/oidc/callback?error=access_denied&error_description=User+cancelled",
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert resp.status_code == 302
|
||||
location = resp.headers["location"]
|
||||
assert "oidc_error" in location
|
||||
assert "User" in urllib.parse.unquote(location)
|
||||
|
||||
def test_invalid_state_redirects_expired(
|
||||
self,
|
||||
authorize_client: TestClient,
|
||||
) -> None:
|
||||
resp = authorize_client.get(
|
||||
"/v1/api/auth/oidc/callback?code=authcode&state=nonexistent",
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert resp.status_code == 302
|
||||
assert "Login+session+expired" in resp.headers["location"]
|
||||
|
||||
def test_expired_state_redirects(
|
||||
self,
|
||||
authorize_client: TestClient,
|
||||
storage: SQLiteBackend,
|
||||
) -> None:
|
||||
# Insert state then backdate created_at via raw SQL so that
|
||||
# pop_oidc_pending_state's max_age_seconds=300 check rejects it.
|
||||
self._seed_pending_state(storage, state="old-state")
|
||||
import sqlalchemy as sa
|
||||
|
||||
with storage._engine.connect() as conn:
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"UPDATE oidc_pending_states SET created_at = '2020-01-01T00:00:00' "
|
||||
"WHERE state = 'old-state'"
|
||||
)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
resp = authorize_client.get(
|
||||
"/v1/api/auth/oidc/callback?code=authcode&state=old-state",
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert resp.status_code == 302
|
||||
assert "Login+session+expired" in resp.headers["location"]
|
||||
|
||||
@patch("turnstone.core.oidc.exchange_code", new_callable=AsyncMock)
|
||||
def test_code_exchange_failure(
|
||||
self,
|
||||
mock_exchange: AsyncMock,
|
||||
authorize_client: TestClient,
|
||||
storage: SQLiteBackend,
|
||||
) -> None:
|
||||
self._seed_pending_state(storage)
|
||||
mock_exchange.side_effect = OIDCError("Token endpoint error")
|
||||
|
||||
resp = authorize_client.get(
|
||||
"/v1/api/auth/oidc/callback?code=badcode&state=valid-state",
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert resp.status_code == 302
|
||||
assert "Authentication+failed" in resp.headers["location"]
|
||||
|
||||
@patch("turnstone.core.oidc.validate_id_token")
|
||||
@patch("turnstone.core.oidc.exchange_code", new_callable=AsyncMock)
|
||||
def test_token_validation_failure(
|
||||
self,
|
||||
mock_exchange: AsyncMock,
|
||||
mock_validate: Any,
|
||||
authorize_client: TestClient,
|
||||
storage: SQLiteBackend,
|
||||
) -> None:
|
||||
self._seed_pending_state(storage)
|
||||
mock_exchange.return_value = {"id_token": "fake.jwt.token"}
|
||||
mock_validate.side_effect = OIDCError("Signature invalid")
|
||||
|
||||
resp = authorize_client.get(
|
||||
"/v1/api/auth/oidc/callback?code=authcode&state=valid-state",
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert resp.status_code == 302
|
||||
assert "Authentication+failed" in resp.headers["location"]
|
||||
|
||||
@patch("turnstone.core.oidc.provision_oidc_user")
|
||||
@patch("turnstone.core.oidc.validate_id_token")
|
||||
@patch("turnstone.core.oidc.fetch_jwks", new_callable=AsyncMock)
|
||||
@patch("turnstone.core.oidc.exchange_code", new_callable=AsyncMock)
|
||||
def test_jwks_key_rotation_retry(
|
||||
self,
|
||||
mock_exchange: AsyncMock,
|
||||
mock_fetch_jwks: AsyncMock,
|
||||
mock_validate: Any,
|
||||
mock_provision: Any,
|
||||
authorize_client: TestClient,
|
||||
storage: SQLiteBackend,
|
||||
) -> None:
|
||||
"""First validate raises 'kid not found in JWKS', fetch_jwks retried, second validate succeeds."""
|
||||
self._seed_pending_state(storage)
|
||||
mock_exchange.return_value = {"id_token": "fake.jwt.token"}
|
||||
|
||||
# First call raises kid-not-found; second call (after JWKS refresh) succeeds
|
||||
mock_validate.side_effect = [
|
||||
OIDCError("Signing key 'new-kid' not found in JWKS"),
|
||||
{"sub": "user123", "email": "u@example.com", "nonce": "test-nonce"},
|
||||
]
|
||||
mock_fetch_jwks.return_value = {"keys": [{"kid": "new-kid", "kty": "RSA"}]}
|
||||
mock_provision.return_value = {"user_id": "test-admin", "username": "testadmin"}
|
||||
|
||||
resp = authorize_client.get(
|
||||
"/v1/api/auth/oidc/callback?code=authcode&state=valid-state",
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert resp.status_code == 302
|
||||
assert "oidc_success=1" in resp.headers["location"]
|
||||
mock_fetch_jwks.assert_called_once()
|
||||
assert mock_validate.call_count == 2
|
||||
|
||||
@patch("turnstone.core.oidc.provision_oidc_user")
|
||||
@patch("turnstone.core.oidc.validate_id_token")
|
||||
@patch("turnstone.core.oidc.exchange_code", new_callable=AsyncMock)
|
||||
def test_no_users_after_oidc_success_redirects_setup(
|
||||
self,
|
||||
mock_exchange: AsyncMock,
|
||||
mock_validate: Any,
|
||||
mock_provision: Any,
|
||||
tmp_path: Any,
|
||||
) -> None:
|
||||
"""When OIDC succeeds but no users exist (edge case), redirect with setup error."""
|
||||
# Use a fresh empty-user storage
|
||||
backend = SQLiteBackend(str(tmp_path / "empty.db"))
|
||||
app = Starlette(
|
||||
routes=[Mount("/v1", routes=[Route("/api/auth/oidc/callback", _oidc_callback)])]
|
||||
)
|
||||
app.state.oidc_config = _make_oidc_config()
|
||||
app.state.auth_storage = backend
|
||||
app.state.jwt_secret = "secret"
|
||||
app.state.jwks_data = {"keys": []}
|
||||
app.state.login_limiter = None
|
||||
|
||||
# Seed a pending state in the empty database
|
||||
backend.create_oidc_pending_state("state1", "nonce1", "verifier1", "test-audience")
|
||||
|
||||
mock_exchange.return_value = {"id_token": "fake.jwt.token"}
|
||||
mock_validate.return_value = {"sub": "u1", "email": "u@example.com", "nonce": "nonce1"}
|
||||
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
resp = client.get(
|
||||
"/v1/api/auth/oidc/callback?code=authcode&state=state1",
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert resp.status_code == 302
|
||||
assert "Initial+setup+required" in resp.headers["location"]
|
||||
|
||||
def test_rate_limited_redirects_with_error(self, storage: SQLiteBackend) -> None:
|
||||
app = Starlette(
|
||||
routes=[Mount("/v1", routes=[Route("/api/auth/oidc/callback", _oidc_callback)])]
|
||||
)
|
||||
app.state.oidc_config = _make_oidc_config()
|
||||
app.state.auth_storage = storage
|
||||
app.state.jwt_secret = "secret"
|
||||
app.state.jwks_data = {"keys": []}
|
||||
limiter = LoginRateLimiter(max_attempts=1, window_seconds=300)
|
||||
limiter.record("ip:testclient")
|
||||
app.state.login_limiter = limiter
|
||||
|
||||
client = TestClient(app, raise_server_exceptions=False)
|
||||
resp = client.get(
|
||||
"/v1/api/auth/oidc/callback?code=authcode&state=x",
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert resp.status_code == 302
|
||||
assert "oidc_error" in resp.headers["location"]
|
||||
assert "Too+many" in resp.headers["location"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Admin OIDC identity endpoint tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAdminOIDCIdentities:
|
||||
"""Tests for admin OIDC identity management endpoints."""
|
||||
|
||||
def test_list_identities(
|
||||
self,
|
||||
admin_client: TestClient,
|
||||
storage: SQLiteBackend,
|
||||
) -> None:
|
||||
storage.create_oidc_identity(
|
||||
"https://idp.example.com",
|
||||
"sub-123",
|
||||
"test-admin",
|
||||
"admin@example.com",
|
||||
)
|
||||
resp = admin_client.get("/v1/api/admin/users/test-admin/oidc-identities")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert len(data["oidc_identities"]) == 1
|
||||
identity = data["oidc_identities"][0]
|
||||
assert identity["issuer"] == "https://idp.example.com"
|
||||
assert identity["subject"] == "sub-123"
|
||||
assert identity["user_id"] == "test-admin"
|
||||
assert identity["email"] == "admin@example.com"
|
||||
|
||||
def test_list_empty(self, admin_client: TestClient) -> None:
|
||||
resp = admin_client.get("/v1/api/admin/users/test-admin/oidc-identities")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["oidc_identities"] == []
|
||||
|
||||
def test_delete_identity(
|
||||
self,
|
||||
admin_client: TestClient,
|
||||
storage: SQLiteBackend,
|
||||
) -> None:
|
||||
storage.create_oidc_identity(
|
||||
"https://idp.example.com",
|
||||
"sub-456",
|
||||
"test-admin",
|
||||
"admin@example.com",
|
||||
)
|
||||
resp = admin_client.delete(
|
||||
"/v1/api/admin/oidc-identities?issuer=https://idp.example.com&subject=sub-456",
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["status"] == "ok"
|
||||
# Verify it's gone
|
||||
assert storage.get_oidc_identity("https://idp.example.com", "sub-456") is None
|
||||
|
||||
def test_delete_nonexistent_returns_404(self, admin_client: TestClient) -> None:
|
||||
resp = admin_client.delete(
|
||||
"/v1/api/admin/oidc-identities?issuer=https://no.such&subject=nope",
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
assert "not found" in resp.json()["error"].lower()
|
||||
|
||||
def test_delete_missing_params_returns_400(self, admin_client: TestClient) -> None:
|
||||
# Missing subject
|
||||
resp = admin_client.delete(
|
||||
"/v1/api/admin/oidc-identities?issuer=https://idp.example.com",
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "required" in resp.json()["error"].lower()
|
||||
|
||||
# Missing both
|
||||
resp = admin_client.delete("/v1/api/admin/oidc-identities")
|
||||
assert resp.status_code == 400
|
||||
@@ -0,0 +1,315 @@
|
||||
"""Tests for OIDC identity and pending state storage CRUD (SQLite backend)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db(tmp_path):
|
||||
"""Create a fresh SQLite backend for each test."""
|
||||
return SQLiteBackend(str(tmp_path / "test.db"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OIDC Identity CRUD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOIDCIdentityCRUD:
|
||||
def test_create_and_get_oidc_identity(self, db):
|
||||
db.create_oidc_identity("https://idp.example.com", "sub-123", "u1", "alice@example.com")
|
||||
identity = db.get_oidc_identity("https://idp.example.com", "sub-123")
|
||||
assert identity is not None
|
||||
assert identity["issuer"] == "https://idp.example.com"
|
||||
assert identity["subject"] == "sub-123"
|
||||
assert identity["user_id"] == "u1"
|
||||
assert identity["email"] == "alice@example.com"
|
||||
assert identity["created"] != ""
|
||||
assert identity["last_login"] != ""
|
||||
|
||||
def test_get_oidc_identity_not_found(self, db):
|
||||
assert db.get_oidc_identity("https://unknown.example.com", "sub-999") is None
|
||||
|
||||
def test_create_oidc_identity_idempotent(self, db):
|
||||
"""Creating twice with same (issuer, subject) does not error (OR IGNORE)."""
|
||||
db.create_oidc_identity("https://idp.example.com", "sub-123", "u1", "alice@example.com")
|
||||
db.create_oidc_identity("https://idp.example.com", "sub-123", "u2", "bob@example.com")
|
||||
|
||||
identity = db.get_oidc_identity("https://idp.example.com", "sub-123")
|
||||
assert identity is not None
|
||||
# OR IGNORE preserves the first insert
|
||||
assert identity["user_id"] == "u1"
|
||||
assert identity["email"] == "alice@example.com"
|
||||
|
||||
def test_update_oidc_identity_login(self, db):
|
||||
db.create_oidc_identity("https://idp.example.com", "sub-123", "u1", "alice@example.com")
|
||||
|
||||
before = db.get_oidc_identity("https://idp.example.com", "sub-123")
|
||||
assert before is not None
|
||||
original_login = before["last_login"]
|
||||
|
||||
# Small sleep to ensure timestamp differs
|
||||
time.sleep(0.05)
|
||||
|
||||
result = db.update_oidc_identity_login("https://idp.example.com", "sub-123")
|
||||
assert result is True
|
||||
|
||||
after = db.get_oidc_identity("https://idp.example.com", "sub-123")
|
||||
assert after is not None
|
||||
assert after["last_login"] >= original_login
|
||||
|
||||
def test_update_oidc_identity_login_nonexistent(self, db):
|
||||
result = db.update_oidc_identity_login("https://idp.example.com", "sub-999")
|
||||
assert result is False
|
||||
|
||||
def test_list_oidc_identities_for_user(self, db):
|
||||
"""Two identities for same user, list returns both."""
|
||||
db.create_oidc_identity("https://idp1.example.com", "sub-A", "u1", "alice@idp1.com")
|
||||
db.create_oidc_identity("https://idp2.example.com", "sub-B", "u1", "alice@idp2.com")
|
||||
|
||||
identities = db.list_oidc_identities_for_user("u1")
|
||||
assert len(identities) == 2
|
||||
issuers = {i["issuer"] for i in identities}
|
||||
assert issuers == {"https://idp1.example.com", "https://idp2.example.com"}
|
||||
|
||||
def test_list_oidc_identities_for_user_empty(self, db):
|
||||
assert db.list_oidc_identities_for_user("u-none") == []
|
||||
|
||||
def test_list_oidc_identities_excludes_other_users(self, db):
|
||||
db.create_oidc_identity("https://idp.example.com", "sub-1", "u1", "alice@example.com")
|
||||
db.create_oidc_identity("https://idp.example.com", "sub-2", "u2", "bob@example.com")
|
||||
|
||||
identities = db.list_oidc_identities_for_user("u1")
|
||||
assert len(identities) == 1
|
||||
assert identities[0]["user_id"] == "u1"
|
||||
|
||||
def test_delete_oidc_identity(self, db):
|
||||
db.create_oidc_identity("https://idp.example.com", "sub-123", "u1", "alice@example.com")
|
||||
assert db.delete_oidc_identity("https://idp.example.com", "sub-123") is True
|
||||
assert db.get_oidc_identity("https://idp.example.com", "sub-123") is None
|
||||
|
||||
def test_delete_oidc_identity_nonexistent(self, db):
|
||||
assert db.delete_oidc_identity("https://idp.example.com", "sub-999") is False
|
||||
|
||||
def test_delete_oidc_identity_only_deletes_target(self, db):
|
||||
"""Deleting one identity does not affect others."""
|
||||
db.create_oidc_identity("https://idp.example.com", "sub-1", "u1", "a@example.com")
|
||||
db.create_oidc_identity("https://idp.example.com", "sub-2", "u1", "b@example.com")
|
||||
|
||||
db.delete_oidc_identity("https://idp.example.com", "sub-1")
|
||||
|
||||
assert db.get_oidc_identity("https://idp.example.com", "sub-1") is None
|
||||
assert db.get_oidc_identity("https://idp.example.com", "sub-2") is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OIDC Pending State
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOIDCPendingState:
|
||||
def test_create_and_pop_pending_state(self, db):
|
||||
db.create_oidc_pending_state(
|
||||
state="state-abc",
|
||||
nonce="nonce-xyz",
|
||||
code_verifier="verifier-123",
|
||||
audience="server",
|
||||
)
|
||||
|
||||
result = db.pop_oidc_pending_state("state-abc")
|
||||
assert result is not None
|
||||
assert result["state"] == "state-abc"
|
||||
assert result["nonce"] == "nonce-xyz"
|
||||
assert result["code_verifier"] == "verifier-123"
|
||||
assert result["audience"] == "server"
|
||||
assert result["created_at"] != ""
|
||||
|
||||
def test_pop_pending_state_not_found(self, db):
|
||||
assert db.pop_oidc_pending_state("nonexistent-state") is None
|
||||
|
||||
def test_pop_pending_state_expired(self, db):
|
||||
"""Create with old timestamp, pop returns None."""
|
||||
# Insert a row with an old created_at timestamp directly
|
||||
import sqlalchemy as sa
|
||||
|
||||
from turnstone.core.storage._schema import oidc_pending_states
|
||||
|
||||
with db._engine.connect() as conn:
|
||||
conn.execute(
|
||||
sa.insert(oidc_pending_states),
|
||||
{
|
||||
"state": "state-old",
|
||||
"nonce": "nonce-old",
|
||||
"code_verifier": "verifier-old",
|
||||
"audience": "server",
|
||||
"created_at": "2020-01-01T00:00:00",
|
||||
},
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
# Default max_age_seconds=300, so a 2020 timestamp is expired
|
||||
result = db.pop_oidc_pending_state("state-old")
|
||||
assert result is None
|
||||
|
||||
def test_pop_pending_state_consumed(self, db):
|
||||
"""Pop twice -> second returns None (one-time use)."""
|
||||
db.create_oidc_pending_state(
|
||||
state="state-once",
|
||||
nonce="nonce-1",
|
||||
code_verifier="verifier-1",
|
||||
audience="server",
|
||||
)
|
||||
|
||||
first = db.pop_oidc_pending_state("state-once")
|
||||
assert first is not None
|
||||
|
||||
second = db.pop_oidc_pending_state("state-once")
|
||||
assert second is None
|
||||
|
||||
def test_pop_pending_state_custom_max_age(self, db):
|
||||
"""Custom max_age_seconds allows longer-lived states."""
|
||||
db.create_oidc_pending_state(
|
||||
state="state-long",
|
||||
nonce="nonce-long",
|
||||
code_verifier="verifier-long",
|
||||
audience="server",
|
||||
)
|
||||
|
||||
# With very short max_age, it might still be valid since we just created it
|
||||
result = db.pop_oidc_pending_state("state-long", max_age_seconds=600)
|
||||
assert result is not None
|
||||
|
||||
def test_create_pending_state_duplicate_raises(self, db):
|
||||
"""Duplicate state insertion raises IntegrityError (no silent drop)."""
|
||||
import sqlalchemy.exc
|
||||
|
||||
db.create_oidc_pending_state("state-dup", "nonce-1", "verifier-1", "server")
|
||||
with pytest.raises(sqlalchemy.exc.IntegrityError):
|
||||
db.create_oidc_pending_state("state-dup", "nonce-2", "verifier-2", "server")
|
||||
|
||||
def test_cleanup_expired_states(self, db):
|
||||
"""Create expired + fresh, cleanup removes only expired."""
|
||||
import sqlalchemy as sa
|
||||
|
||||
from turnstone.core.storage._schema import oidc_pending_states
|
||||
|
||||
# Insert an expired state directly with old timestamp
|
||||
with db._engine.connect() as conn:
|
||||
conn.execute(
|
||||
sa.insert(oidc_pending_states),
|
||||
{
|
||||
"state": "state-expired",
|
||||
"nonce": "nonce-old",
|
||||
"code_verifier": "verifier-old",
|
||||
"audience": "server",
|
||||
"created_at": "2020-01-01T00:00:00",
|
||||
},
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
# Insert a fresh state via normal API
|
||||
db.create_oidc_pending_state("state-fresh", "nonce-new", "verifier-new", "server")
|
||||
|
||||
# Cleanup with default 300s max age
|
||||
deleted = db.cleanup_expired_oidc_states()
|
||||
assert deleted == 1
|
||||
|
||||
# Fresh state should still exist
|
||||
result = db.pop_oidc_pending_state("state-fresh")
|
||||
assert result is not None
|
||||
|
||||
def test_cleanup_expired_states_none_expired(self, db):
|
||||
"""Cleanup with no expired states returns 0."""
|
||||
db.create_oidc_pending_state("state-1", "nonce-1", "verifier-1", "server")
|
||||
deleted = db.cleanup_expired_oidc_states()
|
||||
assert deleted == 0
|
||||
|
||||
def test_cleanup_expired_states_all_expired(self, db):
|
||||
"""Cleanup with all expired states removes all."""
|
||||
import sqlalchemy as sa
|
||||
|
||||
from turnstone.core.storage._schema import oidc_pending_states
|
||||
|
||||
with db._engine.connect() as conn:
|
||||
for i in range(3):
|
||||
conn.execute(
|
||||
sa.insert(oidc_pending_states),
|
||||
{
|
||||
"state": f"state-{i}",
|
||||
"nonce": f"nonce-{i}",
|
||||
"code_verifier": f"verifier-{i}",
|
||||
"audience": "server",
|
||||
"created_at": "2020-01-01T00:00:00",
|
||||
},
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
deleted = db.cleanup_expired_oidc_states()
|
||||
assert deleted == 3
|
||||
|
||||
def test_cleanup_expired_states_custom_max_age(self, db):
|
||||
"""Custom max_age_seconds affects what counts as expired."""
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from turnstone.core.storage._schema import oidc_pending_states
|
||||
|
||||
# Insert a state created 60 seconds ago
|
||||
old_ts = (datetime.now(UTC) - timedelta(seconds=60)).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
with db._engine.connect() as conn:
|
||||
conn.execute(
|
||||
sa.insert(oidc_pending_states),
|
||||
{
|
||||
"state": "state-1",
|
||||
"nonce": "nonce-1",
|
||||
"code_verifier": "verifier-1",
|
||||
"audience": "server",
|
||||
"created_at": old_ts,
|
||||
},
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
# With default max_age=300s the 60s-old state is NOT expired
|
||||
deleted = db.cleanup_expired_oidc_states(max_age_seconds=300)
|
||||
assert deleted == 0
|
||||
|
||||
# With max_age=30s the 60s-old state IS expired
|
||||
deleted = db.cleanup_expired_oidc_states(max_age_seconds=30)
|
||||
assert deleted == 1
|
||||
|
||||
def test_pop_expired_cleans_up_row(self, db):
|
||||
"""Popping an expired state should delete the row (not leave orphan)."""
|
||||
import sqlalchemy as sa
|
||||
|
||||
from turnstone.core.storage._schema import oidc_pending_states
|
||||
|
||||
with db._engine.connect() as conn:
|
||||
conn.execute(
|
||||
sa.insert(oidc_pending_states),
|
||||
{
|
||||
"state": "state-cleanup",
|
||||
"nonce": "nonce-c",
|
||||
"code_verifier": "verifier-c",
|
||||
"audience": "server",
|
||||
"created_at": "2020-01-01T00:00:00",
|
||||
},
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
# Pop returns None (expired)
|
||||
assert db.pop_oidc_pending_state("state-cleanup") is None
|
||||
|
||||
# Row should be gone (cleaned up even though expired)
|
||||
with db._engine.connect() as conn:
|
||||
count = conn.execute(
|
||||
sa.select(sa.func.count())
|
||||
.select_from(oidc_pending_states)
|
||||
.where(oidc_pending_states.c.state == "state-cleanup")
|
||||
).scalar()
|
||||
assert count == 0
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from turnstone.core.session import ChatSession, _render_template
|
||||
@@ -391,3 +392,131 @@ class TestMCPTemplates:
|
||||
session = _make_session(template="mcp__server__code")
|
||||
content = _sys_content(session)
|
||||
assert "MCP_EXPLICIT" in content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Resume with deleted template
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestResumeDeletedTemplate:
|
||||
def test_resume_with_deleted_template_degrades_gracefully(self, tmp_db, capsys):
|
||||
from turnstone.core.memory import save_message
|
||||
from turnstone.core.storage import get_storage
|
||||
|
||||
db = get_storage()
|
||||
_create_template(db, "t1", "ephemeral-tpl", "EPHEMERAL_CONTENT", is_default=False)
|
||||
|
||||
# Create session with template, save a message so resume has history
|
||||
session1 = _make_session(template="ephemeral-tpl")
|
||||
ws_id = session1.ws_id
|
||||
save_message(ws_id, "user", "hello")
|
||||
assert "EPHEMERAL_CONTENT" in _sys_content(session1)
|
||||
|
||||
# Delete the template from storage
|
||||
db.delete_prompt_template("t1")
|
||||
|
||||
# Resume into a new session
|
||||
session2 = _make_session()
|
||||
resumed = session2.resume(ws_id)
|
||||
|
||||
assert resumed
|
||||
assert session2._template_name == "ephemeral-tpl"
|
||||
assert session2._template_content is None
|
||||
# System message should not contain the deleted template content
|
||||
content = _sys_content(session2)
|
||||
assert "EPHEMERAL_CONTENT" not in content
|
||||
# Warning should be logged via structlog
|
||||
captured = capsys.readouterr()
|
||||
assert "not_found" in captured.out or "not_found" in captured.err
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Threading safety
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTemplateFactoryPassthrough:
|
||||
def test_template_passed_through_workstream_create(self, tmp_db):
|
||||
"""WorkstreamManager.create(template=...) propagates to session factory."""
|
||||
from turnstone.core.storage import get_storage
|
||||
from turnstone.core.workstream import WorkstreamManager
|
||||
|
||||
db = get_storage()
|
||||
_create_template(db, "t1", "factory-tpl", "FACTORY_CONTENT", is_default=False)
|
||||
|
||||
captured_template = None
|
||||
|
||||
def factory(ui, model_alias=None, ws_id=None, *, template=None):
|
||||
nonlocal captured_template
|
||||
captured_template = template
|
||||
return _make_session(template=template)
|
||||
|
||||
mgr = WorkstreamManager(factory)
|
||||
ws = mgr.create(name="test", template="factory-tpl")
|
||||
assert captured_template == "factory-tpl"
|
||||
assert ws.session is not None
|
||||
assert ws.session._template_name == "factory-tpl"
|
||||
assert "FACTORY_CONTENT" in _sys_content(ws.session)
|
||||
|
||||
def test_template_none_uses_defaults(self, tmp_db):
|
||||
"""WorkstreamManager.create() without template passes None."""
|
||||
captured_template = "sentinel"
|
||||
|
||||
def factory(ui, model_alias=None, ws_id=None, *, template=None):
|
||||
nonlocal captured_template
|
||||
captured_template = template
|
||||
return _make_session(template=template)
|
||||
|
||||
from turnstone.core.workstream import WorkstreamManager
|
||||
|
||||
mgr = WorkstreamManager(factory)
|
||||
mgr.create(name="test")
|
||||
assert captured_template is None
|
||||
|
||||
|
||||
class TestTemplateThreadSafety:
|
||||
def test_concurrent_template_and_system_message_init(self, tmp_db):
|
||||
from turnstone.core.storage import get_storage
|
||||
|
||||
db = get_storage()
|
||||
_create_template(db, "t1", "thread-tpl", "THREAD_TEMPLATE", is_default=False)
|
||||
|
||||
session = _make_session(template="thread-tpl")
|
||||
errors: list[Exception] = []
|
||||
stop = threading.Event()
|
||||
iterations = 200
|
||||
|
||||
def init_loop():
|
||||
"""Simulate MCP callback repeatedly calling _init_system_messages."""
|
||||
try:
|
||||
for _ in range(iterations):
|
||||
if stop.is_set():
|
||||
break
|
||||
session._init_system_messages()
|
||||
# system_messages must always be a valid list
|
||||
msgs = session.system_messages
|
||||
assert isinstance(msgs, list)
|
||||
assert len(msgs) > 0
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
t = threading.Thread(target=init_loop, daemon=True)
|
||||
t.start()
|
||||
|
||||
# Main thread toggles template on/off
|
||||
try:
|
||||
for i in range(iterations):
|
||||
if i % 2 == 0:
|
||||
session.set_template("thread-tpl")
|
||||
else:
|
||||
session.set_template(None)
|
||||
finally:
|
||||
stop.set()
|
||||
t.join(timeout=5)
|
||||
|
||||
assert not errors, f"Thread raised: {errors}"
|
||||
# Final state: system_messages is a valid list
|
||||
msgs = session.system_messages
|
||||
assert isinstance(msgs, list)
|
||||
assert len(msgs) > 0
|
||||
|
||||
@@ -338,6 +338,28 @@ class TestPlanExec:
|
||||
# Last user message in second call is the coaching message
|
||||
assert "did not follow" in captured_messages[1][-1]["content"]
|
||||
|
||||
def test_plan_includes_template_content(self, tmp_db, tmp_path, monkeypatch):
|
||||
"""Plan agent system message includes template guardrails."""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
session = _make_session()
|
||||
session._template_content = "SAFETY: Do not produce harmful plans."
|
||||
_, _, messages = self._run_plan(session, "build something")
|
||||
sys_content = messages[0]["content"]
|
||||
assert "SAFETY: Do not produce harmful plans." in sys_content
|
||||
assert ChatSession._PLAN_IDENTITY in sys_content
|
||||
# Template appears before plan identity
|
||||
tpl_pos = sys_content.index("SAFETY:")
|
||||
identity_pos = sys_content.index(ChatSession._PLAN_IDENTITY)
|
||||
assert tpl_pos < identity_pos
|
||||
|
||||
def test_plan_no_template_is_identity_only(self, tmp_db, tmp_path, monkeypatch):
|
||||
"""Without templates, plan system message is exactly _PLAN_IDENTITY."""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
session = _make_session()
|
||||
assert session._template_content is None
|
||||
_, _, messages = self._run_plan(session, "build something")
|
||||
assert messages[0]["content"] == ChatSession._PLAN_IDENTITY
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plan validation
|
||||
@@ -557,6 +579,27 @@ class TestPlanRefinement:
|
||||
assert msgs[3]["role"] == "user"
|
||||
assert "add tests too" in msgs[3]["content"]
|
||||
|
||||
def test_refine_plan_includes_template_content(self, tmp_db, tmp_path, monkeypatch):
|
||||
"""_refine_plan system message includes template guardrails."""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
session = _make_session()
|
||||
session._template_content = "SAFETY: guardrails here"
|
||||
captured = {}
|
||||
|
||||
def fake_run_agent(messages, **kwargs):
|
||||
captured["messages"] = list(messages)
|
||||
return self.GOOD_PLAN
|
||||
|
||||
with patch.object(session, "_run_agent", side_effect=fake_run_agent):
|
||||
session._refine_plan(self.GOOD_PLAN, "add auth", "add tests too")
|
||||
|
||||
sys_content = captured["messages"][0]["content"]
|
||||
assert "SAFETY: guardrails here" in sys_content
|
||||
assert ChatSession._PLAN_IDENTITY in sys_content
|
||||
tpl_pos = sys_content.index("SAFETY:")
|
||||
identity_pos = sys_content.index(ChatSession._PLAN_IDENTITY)
|
||||
assert tpl_pos < identity_pos
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Vision / image support
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Tests for system settings admin API endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
from starlette.applications import Starlette
|
||||
from starlette.middleware import Middleware
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.routing import Mount, Route
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
from turnstone.console.server import (
|
||||
admin_delete_setting,
|
||||
admin_list_settings,
|
||||
admin_settings_schema,
|
||||
admin_update_setting,
|
||||
)
|
||||
from turnstone.core.auth import AuthResult
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth bypass middleware
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _InjectAuthMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request: Request, call_next: Any) -> Response:
|
||||
request.state.auth_result = AuthResult(
|
||||
user_id="test-user",
|
||||
scopes=frozenset({"approve"}),
|
||||
token_source="config",
|
||||
permissions=frozenset(
|
||||
{
|
||||
"read",
|
||||
"write",
|
||||
"approve",
|
||||
"admin.settings",
|
||||
}
|
||||
),
|
||||
)
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path):
|
||||
return SQLiteBackend(str(tmp_path / "test.db"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(storage):
|
||||
"""TestClient wired to console admin settings endpoints."""
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Mount(
|
||||
"/v1",
|
||||
routes=[
|
||||
Route("/api/admin/settings", admin_list_settings),
|
||||
Route("/api/admin/settings/schema", admin_settings_schema),
|
||||
Route(
|
||||
"/api/admin/settings/{key:path}",
|
||||
admin_update_setting,
|
||||
methods=["PUT"],
|
||||
),
|
||||
Route(
|
||||
"/api/admin/settings/{key:path}",
|
||||
admin_delete_setting,
|
||||
methods=["DELETE"],
|
||||
),
|
||||
],
|
||||
),
|
||||
],
|
||||
middleware=[Middleware(_InjectAuthMiddleware)],
|
||||
)
|
||||
app.state.auth_storage = storage
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# List settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestListSettings:
|
||||
def test_returns_all_registry_entries(self, client):
|
||||
from turnstone.core.settings_registry import SETTINGS
|
||||
|
||||
r = client.get("/v1/api/admin/settings")
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert len(data["settings"]) == len(SETTINGS)
|
||||
# Every entry has source "default" when nothing stored
|
||||
for entry in data["settings"]:
|
||||
assert entry["source"] == "default"
|
||||
|
||||
def test_stored_value_shows_source_storage(self, client, storage):
|
||||
from turnstone.core.settings_registry import serialize_value
|
||||
|
||||
storage.upsert_system_setting(
|
||||
key="tools.timeout",
|
||||
value=serialize_value(60),
|
||||
node_id="",
|
||||
is_secret=False,
|
||||
changed_by="admin",
|
||||
)
|
||||
r = client.get("/v1/api/admin/settings")
|
||||
assert r.status_code == 200
|
||||
by_key = {s["key"]: s for s in r.json()["settings"]}
|
||||
assert by_key["tools.timeout"]["source"] == "storage"
|
||||
assert by_key["tools.timeout"]["value"] == 60
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Update setting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestUpdateSetting:
|
||||
def test_update_valid(self, client):
|
||||
r = client.put(
|
||||
"/v1/api/admin/settings/tools.timeout",
|
||||
json={"value": 30},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert data["key"] == "tools.timeout"
|
||||
assert data["value"] == 30
|
||||
assert data["source"] == "storage"
|
||||
|
||||
def test_update_invalid_key(self, client):
|
||||
r = client.put(
|
||||
"/v1/api/admin/settings/bogus.nonexistent",
|
||||
json={"value": "x"},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "Unknown setting" in r.json()["error"]
|
||||
|
||||
def test_update_invalid_value_out_of_range(self, client):
|
||||
r = client.put(
|
||||
"/v1/api/admin/settings/tools.timeout",
|
||||
json={"value": 0},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "minimum" in r.json()["error"]
|
||||
|
||||
def test_update_then_list_shows_storage(self, client):
|
||||
client.put(
|
||||
"/v1/api/admin/settings/tools.timeout",
|
||||
json={"value": 42},
|
||||
)
|
||||
r = client.get("/v1/api/admin/settings")
|
||||
by_key = {s["key"]: s for s in r.json()["settings"]}
|
||||
assert by_key["tools.timeout"]["source"] == "storage"
|
||||
assert by_key["tools.timeout"]["value"] == 42
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Delete setting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDeleteSetting:
|
||||
def test_delete_stored(self, client):
|
||||
# First store a value
|
||||
client.put(
|
||||
"/v1/api/admin/settings/tools.timeout",
|
||||
json={"value": 30},
|
||||
)
|
||||
# Delete it
|
||||
r = client.delete("/v1/api/admin/settings/tools.timeout")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["status"] == "ok"
|
||||
|
||||
def test_delete_then_list_shows_default(self, client):
|
||||
client.put(
|
||||
"/v1/api/admin/settings/tools.timeout",
|
||||
json={"value": 30},
|
||||
)
|
||||
client.delete("/v1/api/admin/settings/tools.timeout")
|
||||
r = client.get("/v1/api/admin/settings")
|
||||
by_key = {s["key"]: s for s in r.json()["settings"]}
|
||||
assert by_key["tools.timeout"]["source"] == "default"
|
||||
|
||||
def test_delete_non_existent(self, client):
|
||||
r = client.delete("/v1/api/admin/settings/tools.timeout")
|
||||
assert r.status_code == 404
|
||||
assert "not found" in r.json()["error"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schema endpoint
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSettingsSchema:
|
||||
def test_returns_registry(self, client):
|
||||
from turnstone.core.settings_registry import SETTINGS
|
||||
|
||||
r = client.get("/v1/api/admin/settings/schema")
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert len(data["schema"]) == len(SETTINGS)
|
||||
# Spot-check a few fields
|
||||
by_key = {s["key"]: s for s in data["schema"]}
|
||||
timeout = by_key["tools.timeout"]
|
||||
assert timeout["type"] == "int"
|
||||
assert timeout["default"] == 120
|
||||
assert timeout["min_value"] == 1
|
||||
assert timeout["max_value"] == 3600
|
||||
assert timeout["description"]
|
||||
|
||||
def test_choices_present(self, client):
|
||||
r = client.get("/v1/api/admin/settings/schema")
|
||||
by_key = {s["key"]: s for s in r.json()["schema"]}
|
||||
assert by_key["tools.search"]["choices"] == ["auto", "on", "off"]
|
||||
|
||||
def test_secret_flag(self, client):
|
||||
r = client.get("/v1/api/admin/settings/schema")
|
||||
by_key = {s["key"]: s for s in r.json()["schema"]}
|
||||
assert by_key["judge.api_key"]["is_secret"] is True
|
||||
assert by_key["tools.timeout"]["is_secret"] is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Secret masking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSecretMasking:
|
||||
def test_secret_masked_in_list(self, client, storage):
|
||||
from turnstone.core.settings_registry import serialize_value
|
||||
|
||||
storage.upsert_system_setting(
|
||||
key="judge.api_key",
|
||||
value=serialize_value("sk-real-secret"),
|
||||
node_id="",
|
||||
is_secret=True,
|
||||
changed_by="admin",
|
||||
)
|
||||
r = client.get("/v1/api/admin/settings")
|
||||
by_key = {s["key"]: s for s in r.json()["settings"]}
|
||||
assert by_key["judge.api_key"]["value"] == "***"
|
||||
|
||||
def test_secret_write_blocked(self, client):
|
||||
"""Secret settings cannot be modified via API."""
|
||||
r = client.put(
|
||||
"/v1/api/admin/settings/judge.api_key",
|
||||
json={"value": "sk-secret-123"},
|
||||
)
|
||||
assert r.status_code == 403
|
||||
assert "config.toml" in r.json()["error"]
|
||||
|
||||
def test_secret_shows_managed_label(self, client):
|
||||
"""Secret settings show a label instead of a value."""
|
||||
r = client.get("/v1/api/admin/settings")
|
||||
by_key = {s["key"]: s for s in r.json()["settings"]}
|
||||
assert "managed via" in by_key["judge.api_key"]["value"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Audit trail (verify endpoint returns 200, confirming record_audit call)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAuditTrail:
|
||||
def test_update_returns_200(self, client):
|
||||
"""Update succeeds — audit recording did not raise."""
|
||||
r = client.put(
|
||||
"/v1/api/admin/settings/tools.timeout",
|
||||
json={"value": 45},
|
||||
)
|
||||
assert r.status_code == 200
|
||||
|
||||
def test_delete_returns_200(self, client):
|
||||
"""Delete succeeds — audit recording did not raise."""
|
||||
client.put(
|
||||
"/v1/api/admin/settings/tools.timeout",
|
||||
json={"value": 45},
|
||||
)
|
||||
r = client.delete("/v1/api/admin/settings/tools.timeout")
|
||||
assert r.status_code == 200
|
||||
@@ -0,0 +1,171 @@
|
||||
"""Tests for settings registry validation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.settings_registry import (
|
||||
BOOTSTRAP_SECTIONS,
|
||||
SETTINGS,
|
||||
deserialize_value,
|
||||
serialize_value,
|
||||
validate_key,
|
||||
validate_value,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# validate_key
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestValidateKey:
|
||||
def test_known_key(self):
|
||||
defn = validate_key("memory.relevance_k")
|
||||
assert defn.key == "memory.relevance_k"
|
||||
assert defn.type == "int"
|
||||
|
||||
def test_unknown_key(self):
|
||||
with pytest.raises(ValueError, match="Unknown setting"):
|
||||
validate_key("nonexistent.key")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# validate_value — type coercion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestValidateValueCoercion:
|
||||
def test_int(self):
|
||||
assert validate_value("tools.timeout", "60") == 60
|
||||
assert validate_value("tools.timeout", 60) == 60
|
||||
assert isinstance(validate_value("tools.timeout", "60"), int)
|
||||
|
||||
def test_float(self):
|
||||
assert validate_value("model.temperature", "0.7") == 0.7
|
||||
assert validate_value("model.temperature", 1.5) == 1.5
|
||||
assert isinstance(validate_value("model.temperature", "0.7"), float)
|
||||
|
||||
def test_bool_native(self):
|
||||
assert validate_value("tools.skip_permissions", True) is True
|
||||
assert validate_value("tools.skip_permissions", False) is False
|
||||
|
||||
def test_bool_string_true(self):
|
||||
for s in ("true", "True", "1", "yes"):
|
||||
assert validate_value("tools.skip_permissions", s) is True
|
||||
|
||||
def test_bool_string_false(self):
|
||||
for s in ("false", "False", "0", "no"):
|
||||
assert validate_value("tools.skip_permissions", s) is False
|
||||
|
||||
def test_bool_garbage_string(self):
|
||||
with pytest.raises(ValueError, match="Cannot convert"):
|
||||
validate_value("tools.skip_permissions", "banana")
|
||||
|
||||
def test_none_rejected_for_numeric(self):
|
||||
"""None is not a valid value for numeric settings."""
|
||||
with pytest.raises((ValueError, TypeError)):
|
||||
validate_value("model.temperature", None)
|
||||
with pytest.raises((ValueError, TypeError)):
|
||||
validate_value("tools.timeout", None)
|
||||
|
||||
def test_str(self):
|
||||
assert validate_value("model.name", "gpt-5") == "gpt-5"
|
||||
assert validate_value("session.instructions", "be nice") == "be nice"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# validate_value — range constraints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestValidateValueRange:
|
||||
def test_min_value(self):
|
||||
with pytest.raises(ValueError, match="minimum"):
|
||||
validate_value("tools.timeout", 0) # min_value=1
|
||||
|
||||
def test_max_value(self):
|
||||
with pytest.raises(ValueError, match="maximum"):
|
||||
validate_value("tools.timeout", 9999) # max_value=3600
|
||||
|
||||
def test_min_value_float(self):
|
||||
with pytest.raises(ValueError, match="minimum"):
|
||||
validate_value("model.temperature", -0.1) # min_value=0.0
|
||||
|
||||
def test_max_value_float(self):
|
||||
with pytest.raises(ValueError, match="maximum"):
|
||||
validate_value("model.temperature", 2.1) # max_value=2.0
|
||||
|
||||
def test_boundary_ok(self):
|
||||
# Exact boundary values should pass
|
||||
assert validate_value("tools.timeout", 1) == 1
|
||||
assert validate_value("tools.timeout", 3600) == 3600
|
||||
assert validate_value("model.temperature", 0.0) == 0.0
|
||||
assert validate_value("model.temperature", 2.0) == 2.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# validate_value — choices
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestValidateValueChoices:
|
||||
def test_valid_choice(self):
|
||||
assert validate_value("tools.search", "auto") == "auto"
|
||||
assert validate_value("tools.search", "on") == "on"
|
||||
assert validate_value("tools.search", "off") == "off"
|
||||
|
||||
def test_invalid_choice(self):
|
||||
with pytest.raises(ValueError, match="not in"):
|
||||
validate_value("tools.search", "maybe")
|
||||
|
||||
def test_reasoning_effort_choices(self):
|
||||
for ch in ("", "none", "low", "medium", "high", "max"):
|
||||
assert validate_value("model.reasoning_effort", ch) == ch
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# serialize / deserialize round-trip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSerializeDeserialize:
|
||||
def test_int_round_trip(self):
|
||||
v = 42
|
||||
assert deserialize_value("tools.timeout", serialize_value(v)) == v
|
||||
|
||||
def test_float_round_trip(self):
|
||||
v = 0.75
|
||||
assert deserialize_value("model.temperature", serialize_value(v)) == v
|
||||
|
||||
def test_bool_round_trip(self):
|
||||
for v in (True, False):
|
||||
assert deserialize_value("tools.skip_permissions", serialize_value(v)) is v
|
||||
|
||||
def test_str_round_trip(self):
|
||||
v = "hello world"
|
||||
assert deserialize_value("model.name", serialize_value(v)) == v
|
||||
|
||||
def test_str_round_trip_empty(self):
|
||||
assert deserialize_value("model.name", serialize_value("")) == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry integrity
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRegistryIntegrity:
|
||||
def test_all_keys_have_valid_types(self):
|
||||
valid_types = {"int", "float", "str", "bool"}
|
||||
for key, defn in SETTINGS.items():
|
||||
assert defn.type in valid_types, f"{key} has invalid type {defn.type!r}"
|
||||
|
||||
def test_no_bootstrap_section_keys(self):
|
||||
for key, defn in SETTINGS.items():
|
||||
assert defn.section not in BOOTSTRAP_SECTIONS, (
|
||||
f"{key} in bootstrap section {defn.section!r}"
|
||||
)
|
||||
|
||||
def test_all_entries_have_descriptions(self):
|
||||
for key, defn in SETTINGS.items():
|
||||
assert defn.description, f"{key} has empty description"
|
||||
@@ -215,46 +215,6 @@ class TestWorkstreamMetadata:
|
||||
assert backend.get_workstream_display_name("s1") == "Alias"
|
||||
|
||||
|
||||
# -- Key-value store -----------------------------------------------------------
|
||||
|
||||
|
||||
class TestKVStore:
|
||||
def test_set_and_get(self, backend):
|
||||
assert backend.kv_set("key1", "value1") is None # no previous
|
||||
assert backend.kv_get("key1") == "value1"
|
||||
|
||||
def test_set_returns_old_value(self, backend):
|
||||
backend.kv_set("key1", "v1")
|
||||
old = backend.kv_set("key1", "v2")
|
||||
assert old == "v1"
|
||||
assert backend.kv_get("key1") == "v2"
|
||||
|
||||
def test_delete(self, backend):
|
||||
backend.kv_set("key1", "v1")
|
||||
assert backend.kv_delete("key1")
|
||||
assert backend.kv_get("key1") is None
|
||||
|
||||
def test_delete_nonexistent(self, backend):
|
||||
assert not backend.kv_delete("nope")
|
||||
|
||||
def test_list(self, backend):
|
||||
backend.kv_set("b", "2")
|
||||
backend.kv_set("a", "1")
|
||||
assert backend.kv_list() == [("a", "1"), ("b", "2")]
|
||||
|
||||
def test_search(self, backend):
|
||||
backend.kv_set("project_name", "turnstone")
|
||||
backend.kv_set("version", "0.3")
|
||||
results = backend.kv_search("turnstone")
|
||||
assert len(results) == 1
|
||||
assert results[0] == ("project_name", "turnstone")
|
||||
|
||||
def test_search_empty_lists_all(self, backend):
|
||||
backend.kv_set("a", "1")
|
||||
backend.kv_set("b", "2")
|
||||
assert len(backend.kv_search("")) == 2
|
||||
|
||||
|
||||
# -- Conversation search -------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Tests for turnstone.core.memory — structured memory facade functions."""
|
||||
|
||||
from turnstone.core.memory import (
|
||||
count_structured_memories,
|
||||
delete_structured_memory,
|
||||
list_structured_memories,
|
||||
normalize_key,
|
||||
save_structured_memory,
|
||||
search_structured_memories,
|
||||
)
|
||||
|
||||
|
||||
class TestSaveStructuredMemory:
|
||||
def test_save_new(self, tmp_db):
|
||||
mid, old = save_structured_memory("test_key", "hello world")
|
||||
assert mid != ""
|
||||
assert old is None
|
||||
|
||||
def test_save_upsert(self, tmp_db):
|
||||
save_structured_memory("test_key", "first")
|
||||
mid, old = save_structured_memory("test_key", "second")
|
||||
assert old == "first"
|
||||
assert mid != ""
|
||||
|
||||
def test_save_normalizes_key(self, tmp_db):
|
||||
save_structured_memory("My-Key", "value")
|
||||
mems = list_structured_memories()
|
||||
assert any(m["name"] == "my_key" for m in mems)
|
||||
|
||||
def test_save_with_type_and_scope(self, tmp_db):
|
||||
save_structured_memory("k", "v", mem_type="user", scope="workstream", scope_id="ws1")
|
||||
mems = list_structured_memories(scope="workstream", scope_id="ws1")
|
||||
assert len(mems) == 1
|
||||
assert mems[0]["type"] == "user"
|
||||
|
||||
|
||||
class TestDeleteStructuredMemory:
|
||||
def test_delete_existing(self, tmp_db):
|
||||
save_structured_memory("mykey", "val")
|
||||
assert delete_structured_memory("mykey")
|
||||
|
||||
def test_delete_nonexistent(self, tmp_db):
|
||||
assert not delete_structured_memory("nope")
|
||||
|
||||
def test_delete_normalizes_key(self, tmp_db):
|
||||
save_structured_memory("my_key", "val")
|
||||
assert delete_structured_memory("My-Key")
|
||||
|
||||
|
||||
class TestListStructuredMemories:
|
||||
def test_list_empty(self, tmp_db):
|
||||
assert list_structured_memories() == []
|
||||
|
||||
def test_list_returns_saved(self, tmp_db):
|
||||
save_structured_memory("a", "alpha")
|
||||
save_structured_memory("b", "beta")
|
||||
mems = list_structured_memories()
|
||||
assert len(mems) == 2
|
||||
|
||||
|
||||
class TestSearchStructuredMemories:
|
||||
def test_search_finds_match(self, tmp_db):
|
||||
save_structured_memory("db_host", "localhost", description="database hostname")
|
||||
save_structured_memory("api_url", "http://example.com")
|
||||
results = search_structured_memories("database")
|
||||
assert len(results) >= 1
|
||||
assert any(r["name"] == "db_host" for r in results)
|
||||
|
||||
|
||||
class TestCountStructuredMemories:
|
||||
def test_count_zero(self, tmp_db):
|
||||
assert count_structured_memories() == 0
|
||||
|
||||
def test_count_after_save(self, tmp_db):
|
||||
save_structured_memory("a", "1")
|
||||
save_structured_memory("b", "2")
|
||||
assert count_structured_memories() == 2
|
||||
|
||||
|
||||
class TestNormalizeKey:
|
||||
def test_basic(self):
|
||||
assert normalize_key("My-Key Name") == "my_key_name"
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Tests for structured memory storage backend operations."""
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def backend(tmp_path):
|
||||
return SQLiteBackend(str(tmp_path / "test.db"))
|
||||
|
||||
|
||||
class TestCreateAndGet:
|
||||
def test_create_and_get_by_id(self, backend):
|
||||
backend.create_structured_memory("m1", "test_key", "desc", "project", "global", "", "data")
|
||||
mem = backend.get_structured_memory("m1")
|
||||
assert mem is not None
|
||||
assert mem["name"] == "test_key"
|
||||
assert mem["content"] == "data"
|
||||
assert mem["type"] == "project"
|
||||
|
||||
def test_get_nonexistent(self, backend):
|
||||
assert backend.get_structured_memory("nope") is None
|
||||
|
||||
def test_get_by_name(self, backend):
|
||||
backend.create_structured_memory("m1", "mykey", "d", "project", "global", "", "val")
|
||||
mem = backend.get_structured_memory_by_name("mykey", "global", "")
|
||||
assert mem is not None
|
||||
assert mem["memory_id"] == "m1"
|
||||
|
||||
def test_get_by_name_scoped(self, backend):
|
||||
backend.create_structured_memory("m1", "key", "d", "project", "global", "", "g")
|
||||
backend.create_structured_memory("m2", "key", "d", "project", "workstream", "ws1", "w")
|
||||
g = backend.get_structured_memory_by_name("key", "global", "")
|
||||
w = backend.get_structured_memory_by_name("key", "workstream", "ws1")
|
||||
assert g["content"] == "g"
|
||||
assert w["content"] == "w"
|
||||
|
||||
|
||||
class TestUpdate:
|
||||
def test_update_content(self, backend):
|
||||
backend.create_structured_memory("m1", "k", "d", "project", "global", "", "old")
|
||||
assert backend.update_structured_memory("m1", content="new")
|
||||
mem = backend.get_structured_memory("m1")
|
||||
assert mem["content"] == "new"
|
||||
|
||||
def test_update_nonexistent(self, backend):
|
||||
assert not backend.update_structured_memory("nope", content="x")
|
||||
|
||||
def test_update_no_fields(self, backend):
|
||||
backend.create_structured_memory("m1", "k", "d", "project", "global", "", "data")
|
||||
assert not backend.update_structured_memory("m1", bogus="val")
|
||||
|
||||
def test_update_bumps_timestamp(self, backend):
|
||||
backend.create_structured_memory("m1", "k", "d", "project", "global", "", "data")
|
||||
old = backend.get_structured_memory("m1")["updated"]
|
||||
import time
|
||||
|
||||
time.sleep(0.01)
|
||||
backend.update_structured_memory("m1", content="new")
|
||||
new = backend.get_structured_memory("m1")["updated"]
|
||||
assert new >= old
|
||||
|
||||
|
||||
class TestDelete:
|
||||
def test_delete_existing(self, backend):
|
||||
backend.create_structured_memory("m1", "k", "d", "project", "global", "", "data")
|
||||
assert backend.delete_structured_memory("k", "global", "")
|
||||
assert backend.get_structured_memory("m1") is None
|
||||
|
||||
def test_delete_nonexistent(self, backend):
|
||||
assert not backend.delete_structured_memory("nope", "global", "")
|
||||
|
||||
def test_delete_scoped(self, backend):
|
||||
backend.create_structured_memory("m1", "k", "d", "project", "workstream", "ws1", "data")
|
||||
assert not backend.delete_structured_memory("k", "global", "")
|
||||
assert backend.delete_structured_memory("k", "workstream", "ws1")
|
||||
|
||||
|
||||
class TestList:
|
||||
def test_list_all(self, backend):
|
||||
backend.create_structured_memory("m1", "a", "", "project", "global", "", "1")
|
||||
backend.create_structured_memory("m2", "b", "", "user", "global", "", "2")
|
||||
mems = backend.list_structured_memories()
|
||||
assert len(mems) == 2
|
||||
|
||||
def test_list_by_type(self, backend):
|
||||
backend.create_structured_memory("m1", "a", "", "project", "global", "", "1")
|
||||
backend.create_structured_memory("m2", "b", "", "user", "global", "", "2")
|
||||
mems = backend.list_structured_memories(mem_type="user")
|
||||
assert len(mems) == 1
|
||||
assert mems[0]["name"] == "b"
|
||||
|
||||
def test_list_by_scope(self, backend):
|
||||
backend.create_structured_memory("m1", "a", "", "project", "global", "", "1")
|
||||
backend.create_structured_memory("m2", "b", "", "project", "workstream", "ws1", "2")
|
||||
mems = backend.list_structured_memories(scope="workstream")
|
||||
assert len(mems) == 1
|
||||
|
||||
def test_list_respects_limit(self, backend):
|
||||
for i in range(10):
|
||||
backend.create_structured_memory(f"m{i}", f"k{i}", "", "project", "global", "", f"{i}")
|
||||
mems = backend.list_structured_memories(limit=3)
|
||||
assert len(mems) == 3
|
||||
|
||||
|
||||
class TestSearch:
|
||||
def test_search_by_name(self, backend):
|
||||
backend.create_structured_memory("m1", "database_config", "", "project", "global", "", "pg")
|
||||
backend.create_structured_memory("m2", "api_key", "", "project", "global", "", "secret")
|
||||
results = backend.search_structured_memories("database")
|
||||
assert len(results) == 1
|
||||
assert results[0]["name"] == "database_config"
|
||||
|
||||
def test_search_by_content(self, backend):
|
||||
backend.create_structured_memory("m1", "a", "", "project", "global", "", "postgresql host")
|
||||
results = backend.search_structured_memories("postgresql")
|
||||
assert len(results) == 1
|
||||
|
||||
def test_search_empty_lists_all(self, backend):
|
||||
backend.create_structured_memory("m1", "a", "", "project", "global", "", "1")
|
||||
backend.create_structured_memory("m2", "b", "", "project", "global", "", "2")
|
||||
results = backend.search_structured_memories("")
|
||||
assert len(results) == 2
|
||||
|
||||
|
||||
class TestCount:
|
||||
def test_count_all(self, backend):
|
||||
backend.create_structured_memory("m1", "a", "", "project", "global", "", "1")
|
||||
backend.create_structured_memory("m2", "b", "", "project", "global", "", "2")
|
||||
assert backend.count_structured_memories() == 2
|
||||
|
||||
def test_count_by_scope(self, backend):
|
||||
backend.create_structured_memory("m1", "a", "", "project", "global", "", "1")
|
||||
backend.create_structured_memory("m2", "b", "", "project", "workstream", "ws1", "2")
|
||||
assert backend.count_structured_memories(scope="global") == 1
|
||||
assert backend.count_structured_memories(scope="workstream") == 1
|
||||
@@ -72,7 +72,7 @@ class TestToolsMetadata:
|
||||
"""Validate the metadata extracted from JSON files."""
|
||||
|
||||
def test_tool_count(self):
|
||||
assert len(TOOLS) == 18
|
||||
assert len(TOOLS) == 17
|
||||
|
||||
def test_agent_tools_count(self):
|
||||
assert len(AGENT_TOOLS) == 9
|
||||
@@ -106,9 +106,8 @@ class TestToolsMetadata:
|
||||
"web_search": "query",
|
||||
"task": "prompt",
|
||||
"create_plan": "goal",
|
||||
"remember": "key",
|
||||
"memory": "name",
|
||||
"recall": "query",
|
||||
"forget": "key",
|
||||
"notify": "message",
|
||||
"watch": "command",
|
||||
"read_resource": "uri",
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Tests for WebUI content accumulation — server-side single source of truth."""
|
||||
|
||||
import queue
|
||||
|
||||
import pytest
|
||||
|
||||
from turnstone.server import WebUI
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_global_queue():
|
||||
"""Ensure WebUI._global_queue is set for tests and cleaned up after."""
|
||||
WebUI._global_queue = queue.Queue()
|
||||
yield
|
||||
WebUI._global_queue = None
|
||||
|
||||
|
||||
def _make_ui() -> WebUI:
|
||||
"""Create a WebUI with a global queue for capturing broadcast events."""
|
||||
return WebUI(ws_id="ws-test")
|
||||
|
||||
|
||||
def _drain_global() -> list[dict]:
|
||||
"""Drain all events from the global queue."""
|
||||
events = []
|
||||
assert WebUI._global_queue is not None
|
||||
while not WebUI._global_queue.empty():
|
||||
events.append(WebUI._global_queue.get_nowait())
|
||||
return events
|
||||
|
||||
|
||||
class TestContentAccumulation:
|
||||
"""WebUI should accumulate content tokens and include in idle broadcast."""
|
||||
|
||||
def test_content_token_accumulates(self):
|
||||
"""on_content_token should append to _ws_turn_content."""
|
||||
ui = _make_ui()
|
||||
ui.on_content_token("Hello ")
|
||||
ui.on_content_token("world")
|
||||
assert ui._ws_turn_content == ["Hello ", "world"]
|
||||
|
||||
def test_idle_broadcast_includes_content(self):
|
||||
"""_broadcast_state('idle') should include joined content and reset."""
|
||||
ui = _make_ui()
|
||||
ui.on_content_token("Hello ")
|
||||
ui.on_content_token("world")
|
||||
ui._broadcast_state("idle")
|
||||
|
||||
events = _drain_global()
|
||||
idle_events = [e for e in events if e.get("state") == "idle"]
|
||||
assert len(idle_events) == 1
|
||||
assert idle_events[0]["content"] == "Hello world"
|
||||
# Accumulator should be reset
|
||||
assert ui._ws_turn_content == []
|
||||
assert ui._ws_turn_content_size == 0
|
||||
|
||||
def test_error_broadcast_resets_without_content(self):
|
||||
"""_broadcast_state('error') should reset accumulator without content in event."""
|
||||
ui = _make_ui()
|
||||
ui.on_content_token("partial")
|
||||
ui._broadcast_state("error")
|
||||
|
||||
events = _drain_global()
|
||||
error_events = [e for e in events if e.get("state") == "error"]
|
||||
assert len(error_events) == 1
|
||||
assert "content" not in error_events[0]
|
||||
assert ui._ws_turn_content == []
|
||||
assert ui._ws_turn_content_size == 0
|
||||
|
||||
def test_thinking_broadcast_does_not_touch_accumulator(self):
|
||||
"""_broadcast_state('thinking') should not affect the accumulator."""
|
||||
ui = _make_ui()
|
||||
ui.on_content_token("in progress")
|
||||
ui._broadcast_state("thinking")
|
||||
|
||||
assert ui._ws_turn_content == ["in progress"]
|
||||
events = _drain_global()
|
||||
thinking_events = [e for e in events if e.get("state") == "thinking"]
|
||||
assert len(thinking_events) == 1
|
||||
assert "content" not in thinking_events[0]
|
||||
|
||||
def test_multi_round_accumulation(self):
|
||||
"""Content from multiple streaming rounds accumulates before idle."""
|
||||
ui = _make_ui()
|
||||
# Round 1
|
||||
ui.on_content_token("I'll check ")
|
||||
ui.on_content_token("that. ")
|
||||
# Round 2 (after tool execution)
|
||||
ui.on_content_token("Here's ")
|
||||
ui.on_content_token("the result.")
|
||||
ui._broadcast_state("idle")
|
||||
|
||||
events = _drain_global()
|
||||
idle_events = [e for e in events if e.get("state") == "idle"]
|
||||
assert len(idle_events) == 1
|
||||
assert idle_events[0]["content"] == "I'll check that. Here's the result."
|
||||
|
||||
def test_empty_content_on_idle_without_tokens(self):
|
||||
"""idle with no content tokens should include empty content string."""
|
||||
ui = _make_ui()
|
||||
ui._broadcast_state("idle")
|
||||
|
||||
events = _drain_global()
|
||||
idle_events = [e for e in events if e.get("state") == "idle"]
|
||||
assert len(idle_events) == 1
|
||||
assert idle_events[0]["content"] == ""
|
||||
|
||||
def test_cancellation_preserves_partial_content(self):
|
||||
"""Partial content accumulated before cancel should appear in idle event."""
|
||||
ui = _make_ui()
|
||||
ui.on_content_token("I'll ")
|
||||
ui.on_content_token("start by...")
|
||||
# Cancellation triggers idle broadcast with partial content
|
||||
ui._broadcast_state("idle")
|
||||
|
||||
events = _drain_global()
|
||||
idle_events = [e for e in events if e.get("state") == "idle"]
|
||||
assert len(idle_events) == 1
|
||||
assert idle_events[0]["content"] == "I'll start by..."
|
||||
|
||||
def test_consecutive_turns_isolated(self):
|
||||
"""Content from turn 1 should not leak into turn 2."""
|
||||
ui = _make_ui()
|
||||
# Turn 1
|
||||
ui.on_content_token("first response")
|
||||
ui._broadcast_state("idle")
|
||||
_drain_global()
|
||||
|
||||
# Turn 2
|
||||
ui.on_content_token("second response")
|
||||
ui._broadcast_state("idle")
|
||||
|
||||
events = _drain_global()
|
||||
idle_events = [e for e in events if e.get("state") == "idle"]
|
||||
assert len(idle_events) == 1
|
||||
assert idle_events[0]["content"] == "second response"
|
||||
|
||||
def test_content_cap_prevents_unbounded_growth(self):
|
||||
"""Content exceeding the cap should stop accumulating."""
|
||||
from turnstone.server import _MAX_TURN_CONTENT_CHARS
|
||||
|
||||
ui = _make_ui()
|
||||
# Fill to capacity
|
||||
chunk = "x" * 1024
|
||||
for _ in range(_MAX_TURN_CONTENT_CHARS // 1024 + 10):
|
||||
ui.on_content_token(chunk)
|
||||
|
||||
assert ui._ws_turn_content_size <= _MAX_TURN_CONTENT_CHARS + 1024
|
||||
ui._broadcast_state("idle")
|
||||
|
||||
events = _drain_global()
|
||||
idle_events = [e for e in events if e.get("state") == "idle"]
|
||||
assert len(idle_events) == 1
|
||||
# Content should be capped, not contain everything
|
||||
assert len(idle_events[0]["content"]) <= _MAX_TURN_CONTENT_CHARS + 1024
|
||||
@@ -20,7 +20,7 @@ class FakeSession:
|
||||
self.messages = []
|
||||
|
||||
|
||||
def _fake_factory(ui, model_alias=None, ws_id=None):
|
||||
def _fake_factory(ui, model_alias=None, ws_id=None, **kwargs):
|
||||
return FakeSession()
|
||||
|
||||
|
||||
|
||||
@@ -333,6 +333,381 @@ def test_webui_auto_approve_tools_empty_no_effect():
|
||||
webui._approval_event.wait.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-tool "always approve" — interactive "Always" adds to auto_approve_tools
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_server_always_approve_adds_tool_names():
|
||||
"""POST /approve with always=True adds pending tool names to auto_approve_tools."""
|
||||
webui = WebUI(ws_id="ws-1")
|
||||
webui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [
|
||||
{"func_name": "bash", "needs_approval": True, "preview": "ls"},
|
||||
{"func_name": "read_file", "needs_approval": False, "preview": "/tmp"},
|
||||
],
|
||||
}
|
||||
items = webui._pending_approval.get("items", [])
|
||||
tool_names = {
|
||||
it.get("approval_label", "") or it.get("func_name", "")
|
||||
for it in items
|
||||
if it.get("needs_approval") and it.get("func_name")
|
||||
}
|
||||
tool_names.discard("")
|
||||
tool_names.discard("__budget_override__")
|
||||
webui.auto_approve_tools.update(tool_names)
|
||||
|
||||
assert webui.auto_approve_tools == {"bash"}
|
||||
assert webui.auto_approve is False # blanket flag NOT set
|
||||
|
||||
|
||||
def test_server_always_approve_uses_approval_label():
|
||||
"""When approval_label differs from func_name, approval_label is stored."""
|
||||
webui = WebUI(ws_id="ws-1")
|
||||
webui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [
|
||||
{
|
||||
"func_name": "use_prompt",
|
||||
"approval_label": "mcp__git__commit_msg",
|
||||
"needs_approval": True,
|
||||
"preview": "",
|
||||
},
|
||||
],
|
||||
}
|
||||
items = webui._pending_approval.get("items", [])
|
||||
tool_names = {
|
||||
it.get("approval_label", "") or it.get("func_name", "")
|
||||
for it in items
|
||||
if it.get("needs_approval") and it.get("func_name")
|
||||
}
|
||||
tool_names.discard("")
|
||||
tool_names.discard("__budget_override__")
|
||||
webui.auto_approve_tools.update(tool_names)
|
||||
|
||||
assert "mcp__git__commit_msg" in webui.auto_approve_tools
|
||||
assert "use_prompt" not in webui.auto_approve_tools
|
||||
|
||||
|
||||
def test_server_always_approve_excludes_budget_override():
|
||||
"""__budget_override__ should never be added to auto_approve_tools."""
|
||||
webui = WebUI(ws_id="ws-1")
|
||||
webui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [
|
||||
{"func_name": "__budget_override__", "needs_approval": True, "preview": ""},
|
||||
{"func_name": "bash", "needs_approval": True, "preview": "ls"},
|
||||
],
|
||||
}
|
||||
items = webui._pending_approval.get("items", [])
|
||||
tool_names = {
|
||||
it.get("approval_label", "") or it.get("func_name", "")
|
||||
for it in items
|
||||
if it.get("needs_approval") and it.get("func_name")
|
||||
}
|
||||
tool_names.discard("")
|
||||
tool_names.discard("__budget_override__")
|
||||
webui.auto_approve_tools.update(tool_names)
|
||||
|
||||
assert "__budget_override__" not in webui.auto_approve_tools
|
||||
assert webui.auto_approve_tools == {"bash"}
|
||||
|
||||
|
||||
def test_server_always_approve_accumulates():
|
||||
"""Successive 'always' approvals accumulate tool names."""
|
||||
webui = WebUI(ws_id="ws-1")
|
||||
|
||||
# First always-approve: bash
|
||||
webui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [{"func_name": "bash", "needs_approval": True, "preview": "ls"}],
|
||||
}
|
||||
items = webui._pending_approval["items"]
|
||||
names = {
|
||||
it.get("approval_label", "") or it["func_name"] for it in items if it.get("needs_approval")
|
||||
}
|
||||
names.discard("__budget_override__")
|
||||
webui.auto_approve_tools.update(names)
|
||||
|
||||
# Second always-approve: write_file
|
||||
webui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [{"func_name": "write_file", "needs_approval": True, "preview": ""}],
|
||||
}
|
||||
items = webui._pending_approval["items"]
|
||||
names = {
|
||||
it.get("approval_label", "") or it["func_name"] for it in items if it.get("needs_approval")
|
||||
}
|
||||
names.discard("__budget_override__")
|
||||
webui.auto_approve_tools.update(names)
|
||||
|
||||
assert webui.auto_approve_tools == {"bash", "write_file"}
|
||||
|
||||
|
||||
def test_server_always_approve_no_pending_is_noop():
|
||||
"""If _pending_approval is None, always=True does nothing."""
|
||||
webui = WebUI(ws_id="ws-1")
|
||||
webui._pending_approval = None
|
||||
# The guard `if always and approved and ui._pending_approval:` prevents action
|
||||
assert webui.auto_approve_tools == set()
|
||||
assert webui.auto_approve is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI per-tool "always approve"
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cli_always_adds_tool_names():
|
||||
"""CLI 'a' adds pending tool names to auto_approve_tools, not blanket flag."""
|
||||
from turnstone.cli import TerminalUI
|
||||
|
||||
ui = TerminalUI()
|
||||
items = [
|
||||
{"func_name": "bash", "header": "bash: ls", "needs_approval": True, "preview": "ls"},
|
||||
]
|
||||
with patch("builtins.input", return_value="a"):
|
||||
approved, _ = ui.approve_tools(items)
|
||||
assert approved is True
|
||||
assert ui.auto_approve is False
|
||||
assert "bash" in ui.auto_approve_tools
|
||||
|
||||
|
||||
def test_cli_per_tool_auto_approves_subsequent():
|
||||
"""After 'always' for bash, subsequent bash calls auto-approve silently."""
|
||||
from turnstone.cli import TerminalUI
|
||||
|
||||
ui = TerminalUI()
|
||||
ui.auto_approve_tools = {"bash"}
|
||||
items = [
|
||||
{"func_name": "bash", "header": "bash: ls", "needs_approval": True, "preview": "ls"},
|
||||
]
|
||||
# Should auto-approve without prompting
|
||||
approved, _ = ui.approve_tools(items)
|
||||
assert approved is True
|
||||
|
||||
|
||||
def test_cli_per_tool_does_not_approve_unknown():
|
||||
"""Per-tool set for bash does NOT auto-approve write_file."""
|
||||
from turnstone.cli import TerminalUI
|
||||
|
||||
ui = TerminalUI()
|
||||
ui.auto_approve_tools = {"bash"}
|
||||
items = [
|
||||
{
|
||||
"func_name": "write_file",
|
||||
"header": "write_file: /tmp/x",
|
||||
"needs_approval": True,
|
||||
"preview": "",
|
||||
},
|
||||
]
|
||||
with patch("builtins.input", return_value="n"):
|
||||
approved, _ = ui.approve_tools(items)
|
||||
assert approved is False
|
||||
|
||||
|
||||
def test_cli_always_excludes_budget_override():
|
||||
"""CLI 'always' should not add __budget_override__ to auto_approve_tools."""
|
||||
from turnstone.cli import TerminalUI
|
||||
|
||||
ui = TerminalUI()
|
||||
items = [
|
||||
{
|
||||
"func_name": "__budget_override__",
|
||||
"header": "budget",
|
||||
"needs_approval": True,
|
||||
"preview": "",
|
||||
},
|
||||
]
|
||||
with patch("builtins.input", return_value="a"):
|
||||
approved, _ = ui.approve_tools(items)
|
||||
assert approved is True
|
||||
assert "__budget_override__" not in ui.auto_approve_tools
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bridge per-tool "always approve"
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_bridge_always_adds_to_approve_tools():
|
||||
"""Bridge 'always' adds tool names to _ws_approve_tools, not _ws_auto_approve."""
|
||||
import threading
|
||||
|
||||
from turnstone.mq.bridge import DEFAULT_SAFE_TOOLS, Bridge
|
||||
|
||||
bridge = Bridge.__new__(Bridge)
|
||||
bridge._lock = threading.Lock()
|
||||
bridge._ws_auto_approve = {}
|
||||
bridge._ws_approve_tools = {}
|
||||
|
||||
ws_id = "ws-1"
|
||||
items = [
|
||||
{"func_name": "bash", "needs_approval": True},
|
||||
{"func_name": "read_file", "needs_approval": False},
|
||||
]
|
||||
|
||||
# Simulate the always-approve extraction logic from _wait_approval
|
||||
tool_names = {
|
||||
it.get("func_name", "") for it in items if it.get("needs_approval") and it.get("func_name")
|
||||
}
|
||||
tool_names.discard("")
|
||||
tool_names.discard("__budget_override__")
|
||||
if tool_names:
|
||||
with bridge._lock:
|
||||
existing = bridge._ws_approve_tools.get(ws_id, set(DEFAULT_SAFE_TOOLS))
|
||||
bridge._ws_approve_tools[ws_id] = existing | tool_names
|
||||
|
||||
# bash added, and DEFAULT_SAFE_TOOLS preserved
|
||||
assert "bash" in bridge._ws_approve_tools[ws_id]
|
||||
for name in DEFAULT_SAFE_TOOLS:
|
||||
assert name in bridge._ws_approve_tools[ws_id]
|
||||
assert ws_id not in bridge._ws_auto_approve
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration tests — POST /v1/api/approve with always=True
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestApproveEndpointAlways:
|
||||
"""Integration tests for the approve handler's per-tool 'always' logic."""
|
||||
|
||||
@staticmethod
|
||||
def _make_client(webui):
|
||||
import queue
|
||||
import threading
|
||||
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
from turnstone.core.auth import AuthConfig
|
||||
from turnstone.server import create_app
|
||||
|
||||
mock_ws = MagicMock()
|
||||
mock_ws.ui = webui
|
||||
|
||||
mock_mgr = MagicMock()
|
||||
mock_mgr.get.return_value = mock_ws
|
||||
mock_mgr.list_all.return_value = []
|
||||
|
||||
app = create_app(
|
||||
workstreams=mock_mgr,
|
||||
global_queue=queue.Queue(),
|
||||
global_listeners=[],
|
||||
global_listeners_lock=threading.Lock(),
|
||||
skip_permissions=False,
|
||||
auth_config=AuthConfig(),
|
||||
)
|
||||
return TestClient(app, raise_server_exceptions=False)
|
||||
|
||||
def test_always_adds_tool_to_auto_approve_tools(self):
|
||||
webui = WebUI(ws_id="ws-1")
|
||||
webui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [
|
||||
{"func_name": "bash", "needs_approval": True, "preview": "ls"},
|
||||
],
|
||||
}
|
||||
client = self._make_client(webui)
|
||||
resp = client.post(
|
||||
"/v1/api/approve",
|
||||
json={"approved": True, "always": True, "ws_id": "ws-1"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert "bash" in webui.auto_approve_tools
|
||||
assert webui.auto_approve is False
|
||||
|
||||
def test_always_uses_approval_label_over_func_name(self):
|
||||
webui = WebUI(ws_id="ws-1")
|
||||
webui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [
|
||||
{
|
||||
"func_name": "use_prompt",
|
||||
"approval_label": "mcp__git__commit_msg",
|
||||
"needs_approval": True,
|
||||
"preview": "",
|
||||
},
|
||||
],
|
||||
}
|
||||
client = self._make_client(webui)
|
||||
resp = client.post(
|
||||
"/v1/api/approve",
|
||||
json={"approved": True, "always": True, "ws_id": "ws-1"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert "mcp__git__commit_msg" in webui.auto_approve_tools
|
||||
assert "use_prompt" not in webui.auto_approve_tools
|
||||
|
||||
def test_always_excludes_budget_override(self):
|
||||
webui = WebUI(ws_id="ws-1")
|
||||
webui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [
|
||||
{"func_name": "__budget_override__", "needs_approval": True, "preview": ""},
|
||||
{"func_name": "bash", "needs_approval": True, "preview": "ls"},
|
||||
],
|
||||
}
|
||||
client = self._make_client(webui)
|
||||
resp = client.post(
|
||||
"/v1/api/approve",
|
||||
json={"approved": True, "always": True, "ws_id": "ws-1"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert "__budget_override__" not in webui.auto_approve_tools
|
||||
assert "bash" in webui.auto_approve_tools
|
||||
|
||||
def test_always_skips_non_pending_items(self):
|
||||
webui = WebUI(ws_id="ws-1")
|
||||
webui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [
|
||||
{"func_name": "bash", "needs_approval": True, "preview": "ls"},
|
||||
{"func_name": "read_file", "needs_approval": False, "preview": "/tmp"},
|
||||
],
|
||||
}
|
||||
client = self._make_client(webui)
|
||||
resp = client.post(
|
||||
"/v1/api/approve",
|
||||
json={"approved": True, "always": True, "ws_id": "ws-1"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert webui.auto_approve_tools == {"bash"}
|
||||
|
||||
def test_always_false_does_not_add_tools(self):
|
||||
webui = WebUI(ws_id="ws-1")
|
||||
webui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [
|
||||
{"func_name": "bash", "needs_approval": True, "preview": "ls"},
|
||||
],
|
||||
}
|
||||
client = self._make_client(webui)
|
||||
resp = client.post(
|
||||
"/v1/api/approve",
|
||||
json={"approved": True, "always": False, "ws_id": "ws-1"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert webui.auto_approve_tools == set()
|
||||
|
||||
def test_deny_with_always_does_not_add_tools(self):
|
||||
webui = WebUI(ws_id="ws-1")
|
||||
webui._pending_approval = {
|
||||
"type": "approve_request",
|
||||
"items": [
|
||||
{"func_name": "bash", "needs_approval": True, "preview": "ls"},
|
||||
],
|
||||
}
|
||||
client = self._make_client(webui)
|
||||
resp = client.post(
|
||||
"/v1/api/approve",
|
||||
json={"approved": False, "always": True, "ws_id": "ws-1"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert webui.auto_approve_tools == set()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protocol round-trip — CreateWorkstreamMessage
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""turnstone - Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."""
|
||||
|
||||
__version__ = "0.6.0"
|
||||
__version__ = "0.7.0"
|
||||
|
||||
@@ -501,4 +501,152 @@ class ListChannelUsersResponse(BaseModel):
|
||||
class CreateChannelUserRequest(BaseModel):
|
||||
channel_type: str = Field(..., description="Channel type (e.g. discord, slack)")
|
||||
channel_user_id: str = Field(..., description="External channel user identifier")
|
||||
total: int
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Admin: Memories
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class AdminMemoryInfo(BaseModel):
|
||||
memory_id: str
|
||||
name: str
|
||||
description: str = ""
|
||||
type: str
|
||||
scope: str
|
||||
scope_id: str = ""
|
||||
content: str
|
||||
created: str
|
||||
updated: str
|
||||
last_accessed: str = ""
|
||||
access_count: int = 0
|
||||
|
||||
|
||||
class ListAdminMemoriesResponse(BaseModel):
|
||||
memories: list[AdminMemoryInfo]
|
||||
total: int = 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Admin: System Settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SettingInfo(BaseModel):
|
||||
key: str
|
||||
value: Any = None
|
||||
source: str = "default" # "storage" | "default"
|
||||
type: str = "str"
|
||||
description: str = ""
|
||||
section: str = ""
|
||||
is_secret: bool = False
|
||||
node_id: str = ""
|
||||
changed_by: str = ""
|
||||
updated: str = ""
|
||||
restart_required: bool = False
|
||||
|
||||
|
||||
class ListSettingsResponse(BaseModel):
|
||||
settings: list[SettingInfo]
|
||||
|
||||
|
||||
class SettingSchemaInfo(BaseModel):
|
||||
key: str
|
||||
type: str
|
||||
default: Any = None
|
||||
description: str = ""
|
||||
section: str = ""
|
||||
is_secret: bool = False
|
||||
min_value: float | None = None
|
||||
max_value: float | None = None
|
||||
choices: list[str] | None = None
|
||||
restart_required: bool = False
|
||||
|
||||
|
||||
class ListSettingSchemaResponse(BaseModel):
|
||||
settings_schema: list[SettingSchemaInfo] = Field(alias="schema")
|
||||
|
||||
|
||||
class UpdateSettingRequest(BaseModel):
|
||||
value: Any
|
||||
node_id: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Admin: MCP Servers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class McpServerInfo(BaseModel):
|
||||
server_id: str
|
||||
name: str
|
||||
transport: str
|
||||
command: str = ""
|
||||
args: str = "[]"
|
||||
url: str = ""
|
||||
headers: str = "{}"
|
||||
env: str = "{}"
|
||||
auto_approve: bool = False
|
||||
enabled: bool = True
|
||||
created_by: str = ""
|
||||
created: str
|
||||
updated: str
|
||||
|
||||
|
||||
class McpServerStatus(BaseModel):
|
||||
connected: bool = False
|
||||
tools: int = 0
|
||||
resources: int = 0
|
||||
prompts: int = 0
|
||||
error: str = ""
|
||||
transport: str = ""
|
||||
command: str = ""
|
||||
url: str = ""
|
||||
|
||||
|
||||
class McpServerDetail(McpServerInfo):
|
||||
status: dict[str, McpServerStatus] = Field(default_factory=dict)
|
||||
source: str = "" # "config" for config-file servers, empty for DB-managed
|
||||
|
||||
|
||||
class CreateMcpServerRequest(BaseModel):
|
||||
name: str
|
||||
transport: str # "stdio" | "streamable-http"
|
||||
command: str = ""
|
||||
args: list[str] = []
|
||||
url: str = ""
|
||||
headers: dict[str, str] = Field(default_factory=dict)
|
||||
env: dict[str, str] = Field(default_factory=dict)
|
||||
auto_approve: bool = False
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
class UpdateMcpServerRequest(BaseModel):
|
||||
name: str | None = None
|
||||
transport: str | None = None
|
||||
command: str | None = None
|
||||
args: list[str] | None = None
|
||||
url: str | None = None
|
||||
headers: dict[str, str] | None = None
|
||||
env: dict[str, str] | None = None
|
||||
auto_approve: bool | None = None
|
||||
enabled: bool | None = None
|
||||
|
||||
|
||||
class ListMcpServersResponse(BaseModel):
|
||||
servers: list[McpServerDetail]
|
||||
|
||||
|
||||
class ImportMcpConfigRequest(BaseModel):
|
||||
config: dict[str, Any] = Field(..., description="JSON config object with mcpServers key")
|
||||
|
||||
|
||||
class ImportMcpConfigResponse(BaseModel):
|
||||
imported: list[str] = []
|
||||
skipped: list[str] = []
|
||||
errors: list[str] = []
|
||||
|
||||
|
||||
class McpReloadResponse(BaseModel):
|
||||
status: str = "ok"
|
||||
results: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@@ -8,6 +8,7 @@ if TYPE_CHECKING:
|
||||
from pydantic import BaseModel
|
||||
|
||||
from turnstone.api.console_schemas import (
|
||||
AdminMemoryInfo,
|
||||
AssignRoleRequest,
|
||||
AuditEventInfo,
|
||||
ChannelUserInfo,
|
||||
@@ -19,29 +20,42 @@ from turnstone.api.console_schemas import (
|
||||
ConsoleCreateWsResponse,
|
||||
ConsoleHealthResponse,
|
||||
CreateChannelUserRequest,
|
||||
CreateMcpServerRequest,
|
||||
CreatePromptTemplateRequest,
|
||||
CreateRoleRequest,
|
||||
CreateToolPolicyRequest,
|
||||
CreateWsTemplateRequest,
|
||||
ImportMcpConfigRequest,
|
||||
ImportMcpConfigResponse,
|
||||
ListAdminMemoriesResponse,
|
||||
ListAuditEventsResponse,
|
||||
ListChannelUsersResponse,
|
||||
ListMcpServersResponse,
|
||||
ListOrgsResponse,
|
||||
ListPromptTemplatesResponse,
|
||||
ListRolesResponse,
|
||||
ListSettingSchemaResponse,
|
||||
ListSettingsResponse,
|
||||
ListToolPoliciesResponse,
|
||||
ListUserRolesResponse,
|
||||
ListVerdictsResponse,
|
||||
ListWsTemplatesResponse,
|
||||
ListWsTemplateSummaryResponse,
|
||||
ListWsTemplateVersionsResponse,
|
||||
McpReloadResponse,
|
||||
McpServerDetail,
|
||||
NodeDetailResponse,
|
||||
OrgInfo,
|
||||
PromptTemplateInfo,
|
||||
RoleInfo,
|
||||
SettingInfo,
|
||||
SettingSchemaInfo,
|
||||
ToolPolicyInfo,
|
||||
UpdateMcpServerRequest,
|
||||
UpdateOrgRequest,
|
||||
UpdatePromptTemplateRequest,
|
||||
UpdateRoleRequest,
|
||||
UpdateSettingRequest,
|
||||
UpdateToolPolicyRequest,
|
||||
UpdateWsTemplateRequest,
|
||||
UsageBreakdownItem,
|
||||
@@ -57,6 +71,7 @@ from turnstone.api.schemas import (
|
||||
AuthSetupRequest,
|
||||
AuthSetupResponse,
|
||||
AuthStatusResponse,
|
||||
AuthWhoamiResponse,
|
||||
CreateScheduleRequest,
|
||||
CreateTokenRequest,
|
||||
CreateTokenResponse,
|
||||
@@ -71,6 +86,7 @@ from turnstone.api.schemas import (
|
||||
UpdateScheduleRequest,
|
||||
UserInfo,
|
||||
)
|
||||
from turnstone.api.server_schemas import ListPromptTemplateSummaryResponse, PromptTemplateSummary
|
||||
|
||||
CONSOLE_ENDPOINTS: list[EndpointSpec] = [
|
||||
# --- Cluster ---
|
||||
@@ -183,6 +199,29 @@ CONSOLE_ENDPOINTS: list[EndpointSpec] = [
|
||||
response_model=StatusResponse,
|
||||
tags=["Auth"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/auth/oidc/authorize",
|
||||
"GET",
|
||||
"Redirect to OIDC provider for SSO login",
|
||||
response_code=302,
|
||||
error_codes=[404, 503],
|
||||
tags=["Auth"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/auth/oidc/callback",
|
||||
"GET",
|
||||
"OIDC callback — validates code, provisions user, sets JWT cookie, redirects to app",
|
||||
response_code=302,
|
||||
tags=["Auth"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/auth/whoami",
|
||||
"GET",
|
||||
"Return authenticated user info and permissions",
|
||||
response_model=AuthWhoamiResponse,
|
||||
error_codes=[401],
|
||||
tags=["Auth"],
|
||||
),
|
||||
# --- Admin ---
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/users",
|
||||
@@ -255,6 +294,20 @@ CONSOLE_ENDPOINTS: list[EndpointSpec] = [
|
||||
error_codes=[404],
|
||||
tags=["Admin"],
|
||||
),
|
||||
# --- OIDC Identities ---
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/users/{user_id}/oidc-identities",
|
||||
"GET",
|
||||
"List OIDC identities linked to a user",
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/oidc-identities",
|
||||
"DELETE",
|
||||
"Unlink an OIDC identity (issuer + subject as query params)",
|
||||
error_codes=[400, 404],
|
||||
tags=["Admin"],
|
||||
),
|
||||
# --- Schedules ---
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/schedules",
|
||||
@@ -518,6 +571,14 @@ CONSOLE_ENDPOINTS: list[EndpointSpec] = [
|
||||
response_model=ListWsTemplateSummaryResponse,
|
||||
tags=["Workstreams"],
|
||||
),
|
||||
# --- Prompt templates ---
|
||||
EndpointSpec(
|
||||
"/v1/api/templates",
|
||||
"GET",
|
||||
"List available prompt templates (summary)",
|
||||
response_model=ListPromptTemplateSummaryResponse,
|
||||
tags=["Templates"],
|
||||
),
|
||||
# --- Governance: Usage & Audit ---
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/usage",
|
||||
@@ -572,6 +633,149 @@ CONSOLE_ENDPOINTS: list[EndpointSpec] = [
|
||||
],
|
||||
tags=["Admin"],
|
||||
),
|
||||
# --- Admin: Memories ---
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/memories",
|
||||
"GET",
|
||||
"List structured memories",
|
||||
response_model=ListAdminMemoriesResponse,
|
||||
query_params=[
|
||||
QueryParam("type", "Filter by memory type"),
|
||||
QueryParam("scope", "Filter by scope"),
|
||||
QueryParam("scope_id", "Filter by scope identifier"),
|
||||
QueryParam("limit", "Page size (max 200)", schema_type="integer", default=100),
|
||||
],
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/memories/search",
|
||||
"GET",
|
||||
"Search memories by query",
|
||||
response_model=ListAdminMemoriesResponse,
|
||||
query_params=[
|
||||
QueryParam("q", "Search query", required=True),
|
||||
QueryParam("type", "Filter by memory type"),
|
||||
QueryParam("scope", "Filter by scope"),
|
||||
QueryParam("scope_id", "Filter by scope identifier"),
|
||||
QueryParam("limit", "Max results", schema_type="integer", default=20),
|
||||
],
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/memories/{memory_id}",
|
||||
"GET",
|
||||
"Get a single memory by ID",
|
||||
response_model=AdminMemoryInfo,
|
||||
error_codes=[404],
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/memories/{memory_id}",
|
||||
"DELETE",
|
||||
"Delete a memory by ID",
|
||||
response_model=StatusResponse,
|
||||
error_codes=[404],
|
||||
tags=["Admin"],
|
||||
),
|
||||
# --- Admin: System Settings ---
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/settings",
|
||||
"GET",
|
||||
"List all settings with effective values",
|
||||
response_model=ListSettingsResponse,
|
||||
query_params=[
|
||||
QueryParam("reveal", "Show secret values in plaintext", schema_type="boolean"),
|
||||
],
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/settings/schema",
|
||||
"GET",
|
||||
"Return the full settings registry schema",
|
||||
response_model=ListSettingSchemaResponse,
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/settings/{key}",
|
||||
"PUT",
|
||||
"Set a configuration setting value",
|
||||
request_model=UpdateSettingRequest,
|
||||
response_model=SettingInfo,
|
||||
error_codes=[400],
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/settings/{key}",
|
||||
"DELETE",
|
||||
"Reset a setting to its default value",
|
||||
response_model=StatusResponse,
|
||||
query_params=[
|
||||
QueryParam("node_id", "Node ID for node-scoped settings"),
|
||||
],
|
||||
error_codes=[400, 404],
|
||||
tags=["Admin"],
|
||||
),
|
||||
# --- Admin: MCP Servers ---
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/mcp-servers",
|
||||
"GET",
|
||||
"List MCP server definitions with live status",
|
||||
response_model=ListMcpServersResponse,
|
||||
query_params=[
|
||||
QueryParam("reveal", "Show secret env/header values", schema_type="boolean"),
|
||||
],
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/mcp-servers",
|
||||
"POST",
|
||||
"Create an MCP server definition",
|
||||
request_model=CreateMcpServerRequest,
|
||||
response_model=McpServerDetail,
|
||||
error_codes=[400, 409],
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/mcp-servers/{server_id}",
|
||||
"GET",
|
||||
"Get a single MCP server with status",
|
||||
response_model=McpServerDetail,
|
||||
error_codes=[404],
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/mcp-servers/{server_id}",
|
||||
"PUT",
|
||||
"Update an MCP server definition",
|
||||
request_model=UpdateMcpServerRequest,
|
||||
response_model=McpServerDetail,
|
||||
error_codes=[400, 404, 409],
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/mcp-servers/{server_id}",
|
||||
"DELETE",
|
||||
"Delete an MCP server definition",
|
||||
response_model=StatusResponse,
|
||||
error_codes=[404],
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/mcp-servers/reload",
|
||||
"POST",
|
||||
"Tell all nodes to re-read MCP server config from DB and reconcile",
|
||||
response_model=McpReloadResponse,
|
||||
tags=["Admin"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/admin/mcp-servers/import",
|
||||
"POST",
|
||||
"Import MCP servers from a JSON config file",
|
||||
request_model=ImportMcpConfigRequest,
|
||||
response_model=ImportMcpConfigResponse,
|
||||
error_codes=[400],
|
||||
tags=["Admin"],
|
||||
),
|
||||
# --- Observability ---
|
||||
EndpointSpec(
|
||||
"/health",
|
||||
@@ -636,6 +840,22 @@ _ALL_MODELS: list[type[BaseModel]] = [
|
||||
ListAuditEventsResponse,
|
||||
VerdictInfo,
|
||||
ListVerdictsResponse,
|
||||
AdminMemoryInfo,
|
||||
ListAdminMemoriesResponse,
|
||||
SettingInfo,
|
||||
ListSettingsResponse,
|
||||
SettingSchemaInfo,
|
||||
ListSettingSchemaResponse,
|
||||
UpdateSettingRequest,
|
||||
McpServerDetail,
|
||||
CreateMcpServerRequest,
|
||||
UpdateMcpServerRequest,
|
||||
ListMcpServersResponse,
|
||||
ImportMcpConfigRequest,
|
||||
ImportMcpConfigResponse,
|
||||
McpReloadResponse,
|
||||
PromptTemplateSummary,
|
||||
ListPromptTemplateSummaryResponse,
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -155,6 +155,16 @@ class AuthStatusResponse(BaseModel):
|
||||
auth_enabled: bool
|
||||
has_users: bool
|
||||
setup_required: bool
|
||||
oidc_enabled: bool = False
|
||||
oidc_provider_name: str = ""
|
||||
password_enabled: bool = True
|
||||
|
||||
|
||||
class AuthWhoamiResponse(BaseModel):
|
||||
"""GET /v1/api/auth/whoami response."""
|
||||
|
||||
user_id: str
|
||||
permissions: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Workstream management
|
||||
@@ -21,7 +23,9 @@ class SendResponse(BaseModel):
|
||||
class ApproveRequest(BaseModel):
|
||||
approved: bool = Field(description="True to approve, false to deny")
|
||||
feedback: str | None = Field(default=None, description="Optional denial reason")
|
||||
always: bool = Field(default=False, description="Enable auto-approve for this tool")
|
||||
always: bool = Field(
|
||||
default=False, description="Auto-approve the tools in this batch going forward"
|
||||
)
|
||||
ws_id: str = Field(description="Target workstream ID")
|
||||
|
||||
|
||||
@@ -163,3 +167,88 @@ class HealthResponse(BaseModel):
|
||||
workstreams: WorkstreamCounts = WorkstreamCounts()
|
||||
backend: BackendStatus | None = None
|
||||
mcp: McpStatus | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Memories
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MemoryType = Literal["user", "project", "feedback", "reference"]
|
||||
MemoryScope = Literal["global", "workstream", "user"]
|
||||
|
||||
|
||||
class SaveMemoryRequest(BaseModel):
|
||||
name: str = Field(description="Memory identifier (normalized to snake_case)")
|
||||
content: str = Field(description="Memory content", max_length=65536)
|
||||
description: str = Field(default="", description="Short description for relevance matching")
|
||||
type: MemoryType = Field(default="project", description="Memory type")
|
||||
scope: MemoryScope = Field(default="global", description="Memory scope")
|
||||
scope_id: str = Field(
|
||||
default="",
|
||||
description="Scope identifier (ws_id for workstream, user_id for user scope)",
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_scope_scope_id(self) -> SaveMemoryRequest:
|
||||
scope_id = self.scope_id.strip()
|
||||
if self.scope == "global" and scope_id:
|
||||
raise ValueError("scope_id is not allowed with global scope")
|
||||
if self.scope == "workstream" and not scope_id:
|
||||
raise ValueError("scope_id is required for workstream scope")
|
||||
return self
|
||||
|
||||
|
||||
class MemoryInfo(BaseModel):
|
||||
memory_id: str
|
||||
name: str
|
||||
description: str = ""
|
||||
type: MemoryType
|
||||
scope: MemoryScope
|
||||
scope_id: str = ""
|
||||
content: str
|
||||
created: str
|
||||
updated: str
|
||||
|
||||
|
||||
class ListMemoriesResponse(BaseModel):
|
||||
memories: list[MemoryInfo]
|
||||
total: int = 0
|
||||
|
||||
|
||||
MemoryTypeFilter = Literal["", "user", "project", "feedback", "reference"]
|
||||
MemoryScopeFilter = Literal["", "global", "workstream", "user"]
|
||||
|
||||
|
||||
class SearchMemoriesRequest(BaseModel):
|
||||
query: str = Field(description="Search query text")
|
||||
type: MemoryTypeFilter = Field(default="", description="Filter by memory type")
|
||||
scope: MemoryScopeFilter = Field(default="", description="Filter by scope")
|
||||
scope_id: str = Field(default="", description="Filter by scope_id")
|
||||
limit: int = Field(default=20, description="Max results (1-50)", ge=1, le=50)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_scope_scope_id(self) -> SearchMemoriesRequest:
|
||||
scope_id = self.scope_id.strip()
|
||||
if self.scope == "global" and scope_id:
|
||||
raise ValueError("scope_id is not allowed with global scope")
|
||||
if scope_id and not self.scope:
|
||||
raise ValueError("scope is required when scope_id is provided")
|
||||
return self
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prompt templates (read-only listing)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PromptTemplateSummary(BaseModel):
|
||||
name: str = Field(description="Template name")
|
||||
category: str = Field(default="", description="Template category")
|
||||
is_default: bool = Field(
|
||||
default=False, description="Whether this template is applied by default"
|
||||
)
|
||||
origin: str = Field(default="manual", description="Template origin: manual or mcp")
|
||||
|
||||
|
||||
class ListPromptTemplateSummaryResponse(BaseModel):
|
||||
templates: list[PromptTemplateSummary]
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from turnstone.api.console_schemas import ListWsTemplateSummaryResponse, WsTemplateSummary
|
||||
from turnstone.api.openapi import EndpointSpec, QueryParam, build_openapi
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -14,6 +15,7 @@ from turnstone.api.schemas import (
|
||||
AuthSetupRequest,
|
||||
AuthSetupResponse,
|
||||
AuthStatusResponse,
|
||||
AuthWhoamiResponse,
|
||||
ErrorResponse,
|
||||
StatusResponse,
|
||||
)
|
||||
@@ -26,9 +28,15 @@ from turnstone.api.server_schemas import (
|
||||
CreateWorkstreamResponse,
|
||||
DashboardResponse,
|
||||
HealthResponse,
|
||||
ListMemoriesResponse,
|
||||
ListPromptTemplateSummaryResponse,
|
||||
ListSavedWorkstreamsResponse,
|
||||
ListWorkstreamsResponse,
|
||||
MemoryInfo,
|
||||
PlanFeedbackRequest,
|
||||
PromptTemplateSummary,
|
||||
SaveMemoryRequest,
|
||||
SearchMemoriesRequest,
|
||||
SendRequest,
|
||||
SendResponse,
|
||||
)
|
||||
@@ -140,6 +148,22 @@ SERVER_ENDPOINTS: list[EndpointSpec] = [
|
||||
response_model=ListSavedWorkstreamsResponse,
|
||||
tags=["Workstreams"],
|
||||
),
|
||||
# --- Prompt templates ---
|
||||
EndpointSpec(
|
||||
"/v1/api/templates",
|
||||
"GET",
|
||||
"List available prompt templates (summary)",
|
||||
response_model=ListPromptTemplateSummaryResponse,
|
||||
tags=["Templates"],
|
||||
),
|
||||
# --- Workstream templates ---
|
||||
EndpointSpec(
|
||||
"/v1/api/ws-templates",
|
||||
"GET",
|
||||
"List enabled workstream templates (summary)",
|
||||
response_model=ListWsTemplateSummaryResponse,
|
||||
tags=["Templates"],
|
||||
),
|
||||
# --- Auth ---
|
||||
EndpointSpec(
|
||||
"/v1/api/auth/login",
|
||||
@@ -173,6 +197,74 @@ SERVER_ENDPOINTS: list[EndpointSpec] = [
|
||||
response_model=StatusResponse,
|
||||
tags=["Auth"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/auth/oidc/authorize",
|
||||
"GET",
|
||||
"Redirect to OIDC provider for SSO login",
|
||||
response_code=302,
|
||||
error_codes=[404, 503],
|
||||
tags=["Auth"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/auth/oidc/callback",
|
||||
"GET",
|
||||
"OIDC callback — validates code, provisions user, sets JWT cookie, redirects to app",
|
||||
response_code=302,
|
||||
tags=["Auth"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/auth/whoami",
|
||||
"GET",
|
||||
"Return authenticated user info and permissions",
|
||||
response_model=AuthWhoamiResponse,
|
||||
error_codes=[401],
|
||||
tags=["Auth"],
|
||||
),
|
||||
# --- Memories ---
|
||||
EndpointSpec(
|
||||
"/v1/api/memories",
|
||||
"GET",
|
||||
"List structured memories",
|
||||
response_model=ListMemoriesResponse,
|
||||
query_params=[
|
||||
QueryParam("type", "Filter by memory type"),
|
||||
QueryParam("scope", "Filter by scope"),
|
||||
QueryParam("scope_id", "Filter by scope identifier"),
|
||||
QueryParam(
|
||||
"limit", "Max results (default 100, max 200)", schema_type="integer", default=100
|
||||
),
|
||||
],
|
||||
tags=["Memories"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/memories",
|
||||
"POST",
|
||||
"Save (upsert) a structured memory",
|
||||
request_model=SaveMemoryRequest,
|
||||
response_model=MemoryInfo,
|
||||
error_codes=[400],
|
||||
tags=["Memories"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/memories/search",
|
||||
"POST",
|
||||
"Search structured memories by query",
|
||||
request_model=SearchMemoriesRequest,
|
||||
response_model=ListMemoriesResponse,
|
||||
tags=["Memories"],
|
||||
),
|
||||
EndpointSpec(
|
||||
"/v1/api/memories/{name}",
|
||||
"DELETE",
|
||||
"Delete a structured memory by name and scope",
|
||||
response_model=StatusResponse,
|
||||
query_params=[
|
||||
QueryParam("scope", "Scope (default: global)"),
|
||||
QueryParam("scope_id", "Scope identifier"),
|
||||
],
|
||||
error_codes=[404],
|
||||
tags=["Memories"],
|
||||
),
|
||||
# --- Observability ---
|
||||
EndpointSpec(
|
||||
"/health",
|
||||
@@ -204,6 +296,14 @@ _ALL_MODELS: list[type[BaseModel]] = [
|
||||
DashboardResponse,
|
||||
ListSavedWorkstreamsResponse,
|
||||
HealthResponse,
|
||||
SaveMemoryRequest,
|
||||
MemoryInfo,
|
||||
ListMemoriesResponse,
|
||||
SearchMemoriesRequest,
|
||||
PromptTemplateSummary,
|
||||
ListPromptTemplateSummaryResponse,
|
||||
WsTemplateSummary,
|
||||
ListWsTemplateSummaryResponse,
|
||||
]
|
||||
|
||||
|
||||
|
||||
+27
-1
@@ -93,6 +93,16 @@ For commercial providers (OpenAI, Anthropic-via-proxy), use the real key.
|
||||
- `TURNSTONE_JWT_SECRET` — JWT signing secret (required if auth enabled)
|
||||
- `TURNSTONE_AUTH_TOKEN` — Static bearer token for inter-service auth
|
||||
|
||||
### OIDC SSO (optional)
|
||||
- `TURNSTONE_OIDC_ISSUER` — OIDC issuer URL (e.g., https://accounts.google.com). Setting this + CLIENT_ID + CLIENT_SECRET enables SSO.
|
||||
- `TURNSTONE_OIDC_CLIENT_ID` — Client ID from the identity provider
|
||||
- `TURNSTONE_OIDC_CLIENT_SECRET` — Client secret (confidential client)
|
||||
- `TURNSTONE_OIDC_PROVIDER_NAME` — Display name for the SSO button (default: "SSO")
|
||||
- `TURNSTONE_OIDC_SCOPES` — OIDC scopes (default: "openid email profile")
|
||||
- `TURNSTONE_OIDC_ROLE_CLAIM` — Claim name for role mapping (e.g., "groups")
|
||||
- `TURNSTONE_OIDC_ROLE_MAP` — Comma-separated claim_value:role_id pairs (e.g., "admin:builtin-admin,eng:builtin-operator")
|
||||
- `TURNSTONE_OIDC_PASSWORD_ENABLED` — Set to "false" to hide password login and force SSO-only
|
||||
|
||||
### Ports
|
||||
- `SERVER_PORT` — Server port (default: 8080)
|
||||
- `CONSOLE_PORT` — Console port (default: 8090)
|
||||
@@ -122,6 +132,19 @@ This is a one-time endpoint that only works when zero users exist.
|
||||
Subsequent governance setup (roles, policies, templates) uses the console admin API \
|
||||
with the JWT returned from setup.
|
||||
|
||||
If OIDC is configured, users can also log in via the "Continue with [Provider]" button on the login page.
|
||||
The first admin user must still be created via the setup wizard (OIDC login requires at least one user to exist).
|
||||
OIDC users are auto-provisioned on first login with a default viewer role unless role mapping is configured.
|
||||
|
||||
## Runtime Settings (ConfigStore)
|
||||
After the stack is running, ~40 runtime settings (model, temperature, max_tokens, \
|
||||
reasoning_effort, tool timeout, rate limiting, health probes, judge config, memory \
|
||||
config, etc.) are configurable via the admin Settings tab in the console — no \
|
||||
config.toml edits or restarts needed for most changes. These settings are stored in \
|
||||
the database and apply cluster-wide. The `.env` file only needs bootstrap-critical \
|
||||
settings (database, Redis, auth, ports, API keys). Tell users they can fine-tune \
|
||||
model and behavioral settings after deployment through the admin panel.
|
||||
|
||||
## Built-in Roles
|
||||
- **Admin** (`builtin-admin`): Full access — read, write, approve, all admin.* permissions
|
||||
- **Operator** (`builtin-operator`): read, write, workstreams.create, workstreams.close
|
||||
@@ -148,7 +171,10 @@ Walk the user through setting up their deployment step by step:
|
||||
PostgreSQL is required for cluster mode.
|
||||
5. **Security**: Recommend enabling auth for any non-local deployment. \
|
||||
Use `generate_secret` for JWT secret, Redis password, auth token, and Postgres password. \
|
||||
Ask for initial admin username and password.
|
||||
Ask for initial admin username and password. \
|
||||
If the user's deployment will use an external identity provider (Okta, Azure AD, Google, etc.), \
|
||||
offer to configure OIDC SSO. Ask for the issuer URL, client ID, and client secret. \
|
||||
Optionally configure role mapping and OIDC-only mode.
|
||||
6. **Ports**: Check defaults with `check_port`, suggest alternatives if conflicts.
|
||||
7. **Optional features**: Discord integration, web search (Tavily key), \
|
||||
DuckDuckGo Search MCP (for cluster — uses `ddgCluster` profile with \
|
||||
|
||||
@@ -8,6 +8,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
import uuid
|
||||
from typing import TYPE_CHECKING, Any
|
||||
@@ -26,6 +27,9 @@ if TYPE_CHECKING:
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
# ws_id is a hex string (8–32 chars depending on entry point).
|
||||
_WS_ID_RE = re.compile(r"^[0-9a-f]{8,32}$")
|
||||
|
||||
|
||||
async def _handle_health(request: Request) -> JSONResponse:
|
||||
return JSONResponse({"status": "ok", "service": "channel"})
|
||||
@@ -81,6 +85,9 @@ async def _handle_notify(request: Request) -> JSONResponse:
|
||||
target = body.get("target")
|
||||
message = body.get("message", "").strip() if isinstance(body.get("message"), str) else ""
|
||||
title = body.get("title", "").strip() if isinstance(body.get("title"), str) else ""
|
||||
ws_id = body.get("ws_id", "").strip() if isinstance(body.get("ws_id"), str) else ""
|
||||
if ws_id and not _WS_ID_RE.match(ws_id):
|
||||
return JSONResponse({"error": "invalid ws_id format"}, status_code=400)
|
||||
|
||||
if not target or not message:
|
||||
return JSONResponse({"error": "target and message are required"}, status_code=400)
|
||||
@@ -132,7 +139,10 @@ async def _handle_notify(request: Request) -> JSONResponse:
|
||||
)
|
||||
continue
|
||||
try:
|
||||
msg_id = await adapter.send(channel_id, content)
|
||||
if ws_id:
|
||||
msg_id = await adapter.send_notification(channel_id, content, ws_id)
|
||||
else:
|
||||
msg_id = await adapter.send(channel_id, content)
|
||||
results.append(
|
||||
{
|
||||
"channel_type": channel_type,
|
||||
|
||||
@@ -41,6 +41,14 @@ class ChannelAdapter(Protocol):
|
||||
"""Send a message to a channel. Returns the platform message ID."""
|
||||
...
|
||||
|
||||
async def send_notification(self, channel_id: str, content: str, ws_id: str) -> str:
|
||||
"""Send a notification and track the reply mapping. Returns message ID.
|
||||
|
||||
Like :meth:`send` but associates the outgoing message with *ws_id*
|
||||
so that replies can be routed back to the originating workstream.
|
||||
"""
|
||||
...
|
||||
|
||||
async def edit_message(self, channel_id: str, message_id: str, content: str) -> None:
|
||||
"""Edit an existing message in a channel."""
|
||||
...
|
||||
|
||||
@@ -124,6 +124,7 @@ class TurnstoneBot:
|
||||
"""
|
||||
|
||||
channel_type: str = "discord"
|
||||
_MAX_NOTIFY_TRACKING: int = 100
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -151,6 +152,16 @@ class TurnstoneBot:
|
||||
# workstream so that IntentVerdictEvent can update it with LLM judge
|
||||
# results.
|
||||
self._pending_approval_msgs: dict[str, discord.Message] = {}
|
||||
# Notification reply tracking: maps Discord message ID →
|
||||
# (ws_id, target_discord_user_id) so that DM replies can be routed
|
||||
# back to the originating workstream. The target user ID is checked
|
||||
# on reply to prevent cross-user message injection.
|
||||
self._notify_ws_map: dict[int, tuple[str, str]] = {}
|
||||
# Temporary DM forwarding: maps ws_id → (DM channel, target_user_id)
|
||||
# for forwarding the workstream's next response back to the
|
||||
# notification reply DM. The target_user_id is carried so the
|
||||
# response message can be re-tracked for multi-turn DM conversations.
|
||||
self._notify_reply_channels: dict[str, tuple[discord.abc.Messageable, str]] = {}
|
||||
|
||||
intents = discord.Intents.default()
|
||||
intents.message_content = True
|
||||
@@ -256,6 +267,11 @@ class TurnstoneBot:
|
||||
self._subscribed_ws.discard(ws_id)
|
||||
self._streaming.pop(ws_id, None)
|
||||
self._pending_approval_msgs.pop(ws_id, None)
|
||||
self._notify_reply_channels.pop(ws_id, None)
|
||||
# Purge stale notification tracking entries for this workstream.
|
||||
stale = [mid for mid, entry in self._notify_ws_map.items() if entry[0] == ws_id]
|
||||
for mid in stale:
|
||||
del self._notify_ws_map[mid]
|
||||
log.info("discord.unsubscribed", ws_id=ws_id)
|
||||
|
||||
# -- event dispatch ------------------------------------------------------
|
||||
@@ -360,6 +376,26 @@ class TurnstoneBot:
|
||||
sm = self._streaming.pop(ws_id, None)
|
||||
if sm is not None:
|
||||
await sm.finalize()
|
||||
elif event.content:
|
||||
# Catch-up: content events were missed (race between global
|
||||
# SSE and per-ws SSE) — send the full response directly.
|
||||
for chunk in chunk_message(event.content, self.config.max_message_length):
|
||||
await thread.send(chunk)
|
||||
# Forward response to notification reply DM if active.
|
||||
dm_entry = self._notify_reply_channels.pop(ws_id, None)
|
||||
if dm_entry is not None and event.content:
|
||||
dm_channel, target_user_id = dm_entry
|
||||
last_msg: discord.Message | None = None
|
||||
for chunk in chunk_message(event.content, self.config.max_message_length):
|
||||
try:
|
||||
last_msg = await dm_channel.send(chunk)
|
||||
except Exception:
|
||||
log.debug("discord.notify_reply_dm_failed", ws_id=ws_id)
|
||||
break
|
||||
# Track the response message so the user can reply again
|
||||
# for multi-turn DM conversations.
|
||||
if last_msg is not None:
|
||||
self._track_notification(last_msg.id, ws_id, target_user_id)
|
||||
# Clean up pending approval message tracking.
|
||||
self._pending_approval_msgs.pop(ws_id, None)
|
||||
|
||||
@@ -390,6 +426,18 @@ class TurnstoneBot:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _track_notification(self, message_id: int, ws_id: str, target_user_id: str) -> None:
|
||||
"""Record a notification message for reply routing.
|
||||
|
||||
Evicts the oldest entry when the map exceeds
|
||||
``_MAX_NOTIFY_TRACKING``. Relies on dict insertion order
|
||||
(Python 3.7+).
|
||||
"""
|
||||
while len(self._notify_ws_map) >= self._MAX_NOTIFY_TRACKING:
|
||||
oldest = next(iter(self._notify_ws_map))
|
||||
del self._notify_ws_map[oldest]
|
||||
self._notify_ws_map[message_id] = (ws_id, target_user_id)
|
||||
|
||||
def _is_allowed_channel(self, channel_id: int) -> bool:
|
||||
"""Return True if *channel_id* is in the allowed list (or list is empty)."""
|
||||
if not self.config.allowed_channels:
|
||||
@@ -430,6 +478,26 @@ class TurnstoneBot:
|
||||
|
||||
return str(msg.id) if msg else ""
|
||||
|
||||
async def send_notification(self, channel_id: str, content: str, ws_id: str) -> str:
|
||||
"""Send a notification DM and track the message for reply routing.
|
||||
|
||||
Like :meth:`send` but records a mapping from the outgoing Discord
|
||||
message ID to ``(ws_id, channel_id)`` so that a user reply can be
|
||||
routed back to the originating workstream. The *channel_id* is the
|
||||
Discord user ID the notification was sent to — verified on reply to
|
||||
prevent cross-user message injection.
|
||||
"""
|
||||
msg_id_str = await self.send(channel_id, content)
|
||||
if msg_id_str and ws_id:
|
||||
self._track_notification(int(msg_id_str), ws_id, channel_id)
|
||||
log.debug(
|
||||
"discord.notification_tracked",
|
||||
message_id=msg_id_str,
|
||||
ws_id=ws_id,
|
||||
target_user=channel_id,
|
||||
)
|
||||
return msg_id_str
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Disconnect the bot and clean up subscriptions."""
|
||||
for ws_id in list(self._subscribed_ws):
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user