mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-13 15:32:24 -06:00
Compare commits
10 Commits
stable/1.6
...
v1.0.3
| Author | SHA1 | Date | |
|---|---|---|---|
| 79eec8194f | |||
| 425d1a7d8f | |||
| 06c41f0a59 | |||
| d107e6edf0 | |||
| 22c20a8dbd | |||
| 179143431d | |||
| d4a6866045 | |||
| c4abd62226 | |||
| b3926c372a | |||
| 5efe52d433 |
@@ -74,6 +74,53 @@ jobs:
|
||||
env:
|
||||
TURNSTONE_TEST_PG_URL: postgresql+psycopg://postgres:postgres@localhost:5432/turnstone_test
|
||||
|
||||
wheel-completeness:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- run: pip install build
|
||||
- run: python -m build --wheel
|
||||
- name: Check all data files are in wheel
|
||||
run: |
|
||||
SOURCE=$(find turnstone -type f \
|
||||
! -name '*.py' ! -name '*.pyc' ! -path '*__pycache__*' \
|
||||
| sort)
|
||||
WHEEL=$(python -m zipfile -l dist/*.whl \
|
||||
| awk '{print $1}' \
|
||||
| grep -v '\.py$' | grep -v '\.dist-info' | grep -v '\.pyc' | grep -v '^File$' \
|
||||
| sort)
|
||||
|
||||
# Files intentionally excluded from the wheel (one per line)
|
||||
ALLOW="
|
||||
turnstone/core/storage/migrations/script.py.mako
|
||||
"
|
||||
|
||||
MISSING=$(comm -23 <(echo "$SOURCE") <(echo "$WHEEL") \
|
||||
| grep -vFxf <(echo "$ALLOW" | sed '/^[[:space:]]*$/d; s/^[[:space:]]*//' ) || true)
|
||||
|
||||
if [ -n "$MISSING" ]; then
|
||||
echo "::error::Data files in source tree but missing from wheel:"
|
||||
echo "$MISSING"
|
||||
echo ""
|
||||
echo "Add them to [tool.hatch.build.targets.wheel] in pyproject.toml"
|
||||
echo "or to the ALLOW list in this job if intentionally excluded."
|
||||
exit 1
|
||||
fi
|
||||
echo "All source data files present in wheel"
|
||||
- name: Smoke-test entry points from installed wheel
|
||||
run: |
|
||||
python -m venv /tmp/smoke
|
||||
/tmp/smoke/bin/pip install dist/*.whl
|
||||
/tmp/smoke/bin/turnstone --help
|
||||
/tmp/smoke/bin/turnstone-server --help
|
||||
/tmp/smoke/bin/turnstone-console --help
|
||||
/tmp/smoke/bin/turnstone-admin --help
|
||||
/tmp/smoke/bin/turnstone-channel --help
|
||||
/tmp/smoke/bin/turnstone-bootstrap --help
|
||||
|
||||
lock-check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
@@ -5,6 +5,10 @@ on:
|
||||
workflows: ["CI"]
|
||||
types: [completed]
|
||||
|
||||
concurrency:
|
||||
group: docker-${{ github.event.workflow_run.head_sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
@@ -5,6 +5,10 @@ on:
|
||||
workflows: ["CI"]
|
||||
types: [completed]
|
||||
|
||||
concurrency:
|
||||
group: publish-${{ github.event.workflow_run.head_sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
|
||||
+11
-9
@@ -1,5 +1,10 @@
|
||||
# =============================================================================
|
||||
# Turnstone Docker Compose Stack
|
||||
# Turnstone Docker Compose Stack — Development
|
||||
#
|
||||
# This file is for local development from a git clone. It builds images
|
||||
# locally from the Dockerfile. If you installed via pip/pipx, run
|
||||
# `turnstone-bootstrap` instead — it writes a production compose.yaml
|
||||
# that pulls pre-built images from ghcr.io.
|
||||
#
|
||||
# Usage:
|
||||
# Infra only: docker compose up
|
||||
@@ -60,9 +65,7 @@ services:
|
||||
# turnstone-server — Web UI + chat workstreams + LLM interaction
|
||||
# -------------------------------------------------------------------
|
||||
server:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: turnstone:local
|
||||
profiles:
|
||||
- production
|
||||
command:
|
||||
@@ -115,6 +118,7 @@ services:
|
||||
# turnstone-console — Cluster dashboard
|
||||
# -------------------------------------------------------------------
|
||||
console:
|
||||
image: turnstone:local
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
@@ -145,9 +149,7 @@ services:
|
||||
# Requires TURNSTONE_DISCORD_TOKEN to enable Discord adapter
|
||||
# -------------------------------------------------------------------
|
||||
channel:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: turnstone:local
|
||||
profiles:
|
||||
- production
|
||||
- cluster
|
||||
@@ -164,7 +166,7 @@ services:
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-postgresql}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-postgresql://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:-turnstone}@postgres:5432/turnstone}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:-turnstone}@postgres:5432/turnstone}
|
||||
- TURNSTONE_CHANNEL_ADVERTISE_URL=http://channel:8091
|
||||
networks:
|
||||
- turnstone-net
|
||||
@@ -214,7 +216,7 @@ services:
|
||||
MODEL: ${MODEL:-}
|
||||
MCP_CONFIG: ${MCP_CONFIG:-}
|
||||
TURNSTONE_DB_BACKEND: ${DB_BACKEND:-postgresql}
|
||||
TURNSTONE_DB_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:?}@postgres:5432/turnstone}
|
||||
TURNSTONE_DB_URL: ${DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:?}@postgres:5432/turnstone}
|
||||
TURNSTONE_NODE_ID: node-1
|
||||
TURNSTONE_ADVERTISE_URL: http://server-1:8080
|
||||
extra_hosts: ["host.docker.internal:host-gateway"]
|
||||
|
||||
+3
-1
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "turnstone"
|
||||
version = "1.0.0"
|
||||
version = "1.0.3"
|
||||
description = "Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."
|
||||
readme = "README.md"
|
||||
license = "BUSL-1.1"
|
||||
@@ -67,6 +67,7 @@ turnstone-bootstrap = "turnstone.bootstrap:main"
|
||||
[tool.hatch.build.targets.wheel]
|
||||
include = [
|
||||
"turnstone/**/*.py",
|
||||
"turnstone/prompts/**/*.md",
|
||||
"turnstone/tools/*.json",
|
||||
"turnstone/ui/static/*.html",
|
||||
"turnstone/ui/static/*.css",
|
||||
@@ -80,6 +81,7 @@ include = [
|
||||
"turnstone/shared_static/hljs-11.11.1/**/*",
|
||||
"turnstone/shared_static/mermaid-11.14.0/**/*",
|
||||
"turnstone/sdk/py.typed",
|
||||
"turnstone/deploy/*.yaml",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
|
||||
+48
-1
@@ -19,6 +19,7 @@ from turnstone.bootstrap import (
|
||||
_tool_generate_secret,
|
||||
_tool_read_file,
|
||||
_tool_validate_api_key,
|
||||
_tool_write_compose,
|
||||
_tool_write_file,
|
||||
execute_tool,
|
||||
)
|
||||
@@ -103,6 +104,52 @@ class TestWriteFile:
|
||||
assert (tmp_path / "changed.txt").read_text() == "new\n"
|
||||
|
||||
|
||||
class TestWriteCompose:
|
||||
def test_writes_compose_file(self, tmp_path: Path) -> None:
|
||||
with patch("builtins.input", return_value="y"):
|
||||
result = _tool_write_compose(tmp_path, {})
|
||||
assert "written successfully" in result
|
||||
assert "ghcr.io" in result
|
||||
content = (tmp_path / "compose.yaml").read_text()
|
||||
assert "ghcr.io/turnstonelabs/turnstone" in content
|
||||
assert "TURNSTONE_IMAGE_TAG" in content
|
||||
|
||||
def test_user_declines(self, tmp_path: Path) -> None:
|
||||
with patch("builtins.input", return_value="n"):
|
||||
result = _tool_write_compose(tmp_path, {})
|
||||
assert "declined" in result
|
||||
assert not (tmp_path / "compose.yaml").exists()
|
||||
|
||||
def test_identical_content_skipped(self, tmp_path: Path) -> None:
|
||||
# Write it once
|
||||
with patch("builtins.input", return_value="y"):
|
||||
_tool_write_compose(tmp_path, {})
|
||||
# Second call should skip
|
||||
result = _tool_write_compose(tmp_path, {})
|
||||
assert "already exists" in result
|
||||
|
||||
def test_no_build_blocks(self, tmp_path: Path) -> None:
|
||||
with patch("builtins.input", return_value="y"):
|
||||
_tool_write_compose(tmp_path, {})
|
||||
content = (tmp_path / "compose.yaml").read_text()
|
||||
assert "build:" not in content
|
||||
assert "dockerfile:" not in content.lower()
|
||||
|
||||
def test_overwrites_different_content(self, tmp_path: Path) -> None:
|
||||
(tmp_path / "compose.yaml").write_text("old content\n")
|
||||
with patch("builtins.input", return_value="y"):
|
||||
result = _tool_write_compose(tmp_path, {})
|
||||
assert "written successfully" in result
|
||||
content = (tmp_path / "compose.yaml").read_text()
|
||||
assert "ghcr.io" in content
|
||||
|
||||
def test_no_local_image_references(self, tmp_path: Path) -> None:
|
||||
with patch("builtins.input", return_value="y"):
|
||||
_tool_write_compose(tmp_path, {})
|
||||
content = (tmp_path / "compose.yaml").read_text()
|
||||
assert "turnstone:local" not in content
|
||||
|
||||
|
||||
class TestGenerateSecret:
|
||||
def test_default_length(self) -> None:
|
||||
secret = _tool_generate_secret({})
|
||||
@@ -620,7 +667,7 @@ class TestConstants:
|
||||
assert func["parameters"]["type"] == "object"
|
||||
|
||||
def test_tool_count(self) -> None:
|
||||
assert len(TOOLS) == 7
|
||||
assert len(TOOLS) == 8
|
||||
|
||||
def test_all_tools_have_implementations(self) -> None:
|
||||
from turnstone.bootstrap import TOOL_FUNCTIONS
|
||||
|
||||
@@ -41,6 +41,21 @@ class TestHashRingBuckets:
|
||||
# Empty list returns 0
|
||||
assert storage.assign_buckets([], "node-x") == 0
|
||||
|
||||
def test_assign_large_list_exceeds_chunk_size(self, storage):
|
||||
"""Regression: lists larger than chunk_size must not hit param limits."""
|
||||
n = 1200 # exceeds SQLite chunk_size (500) and exercises multi-chunk path
|
||||
storage.seed_ring_buckets([(i, "node-a") for i in range(n)])
|
||||
count = storage.assign_buckets(list(range(n)), "node-b")
|
||||
assert count == n
|
||||
rows = storage.list_ring_buckets()
|
||||
assert all(r["node_id"] == "node-b" for r in rows)
|
||||
|
||||
def test_assign_deduplicates_input(self, storage):
|
||||
"""Duplicates in the input list should not inflate rowcount."""
|
||||
storage.seed_ring_buckets([(0, "node-a"), (1, "node-a")])
|
||||
count = storage.assign_buckets([0, 1, 0, 1, 0], "node-b")
|
||||
assert count == 2
|
||||
|
||||
|
||||
class TestBucketStats:
|
||||
def test_increment_creates_row(self, storage):
|
||||
|
||||
@@ -37,3 +37,55 @@ class TestStripHtml:
|
||||
def test_self_closing_tags(self):
|
||||
result = strip_html("hello<br/>world")
|
||||
assert result == "helloworld"
|
||||
|
||||
# -- invisible element stripping -----------------------------------------
|
||||
|
||||
def test_strips_script_content(self):
|
||||
html = "<p>before</p><script>var x = 1;</script><p>after</p>"
|
||||
result = strip_html(html)
|
||||
assert "var x" not in result
|
||||
assert "before" in result
|
||||
assert "after" in result
|
||||
|
||||
def test_strips_style_content(self):
|
||||
html = "<style>.foo { color: red; }</style><p>visible</p>"
|
||||
result = strip_html(html)
|
||||
assert "color" not in result
|
||||
assert "visible" in result
|
||||
|
||||
def test_strips_template_content(self):
|
||||
html = "<template><div>hidden</div></template><p>shown</p>"
|
||||
result = strip_html(html)
|
||||
assert "hidden" not in result
|
||||
assert "shown" in result
|
||||
|
||||
def test_strips_noscript_content(self):
|
||||
html = "<noscript>Enable JS</noscript><p>content</p>"
|
||||
result = strip_html(html)
|
||||
assert "Enable JS" not in result
|
||||
assert "content" in result
|
||||
|
||||
def test_strips_multiple_script_blocks(self):
|
||||
html = "<script>a()</script><p>middle</p><script>b()</script>"
|
||||
result = strip_html(html)
|
||||
assert "a()" not in result
|
||||
assert "b()" not in result
|
||||
assert "middle" in result
|
||||
|
||||
def test_strips_multiline_script(self):
|
||||
html = "<script>\nfunction foo() {\n return 1;\n}\n</script><p>ok</p>"
|
||||
result = strip_html(html)
|
||||
assert "function" not in result
|
||||
assert "ok" in result
|
||||
|
||||
def test_strips_script_case_insensitive(self):
|
||||
html = "<SCRIPT>code()</SCRIPT><p>text</p>"
|
||||
result = strip_html(html)
|
||||
assert "code()" not in result
|
||||
assert "text" in result
|
||||
|
||||
def test_strips_script_with_attributes(self):
|
||||
html = '<script type="text/javascript" src="app.js">init();</script><p>done</p>'
|
||||
result = strip_html(html)
|
||||
assert "init()" not in result
|
||||
assert "done" in result
|
||||
|
||||
@@ -759,6 +759,8 @@ class TestTitleRetry:
|
||||
"""_generate_title resets _title_generated on failure."""
|
||||
|
||||
def test_title_generated_reset_on_failure(self, tmp_db):
|
||||
from turnstone.core.providers._protocol import ModelCapabilities
|
||||
|
||||
session = _make_session()
|
||||
session._title_generated = True
|
||||
session.messages = [
|
||||
@@ -767,6 +769,7 @@ class TestTitleRetry:
|
||||
]
|
||||
# Mock provider to raise
|
||||
session._provider = MagicMock()
|
||||
session._provider.get_capabilities.return_value = ModelCapabilities()
|
||||
session._provider.create_completion.side_effect = RuntimeError("API error")
|
||||
|
||||
session._generate_title()
|
||||
@@ -774,6 +777,8 @@ class TestTitleRetry:
|
||||
assert session._title_generated is False
|
||||
|
||||
def test_title_generated_stays_true_on_success(self, tmp_db):
|
||||
from turnstone.core.providers._protocol import ModelCapabilities
|
||||
|
||||
session = _make_session()
|
||||
session._title_generated = True
|
||||
session.messages = [
|
||||
@@ -783,6 +788,7 @@ class TestTitleRetry:
|
||||
result = MagicMock()
|
||||
result.content = "Test Title"
|
||||
session._provider = MagicMock()
|
||||
session._provider.get_capabilities.return_value = ModelCapabilities()
|
||||
session._provider.create_completion.return_value = result
|
||||
|
||||
with patch("turnstone.core.session.update_workstream_title"):
|
||||
@@ -793,6 +799,8 @@ class TestTitleRetry:
|
||||
|
||||
def test_title_skipped_after_resume_changes_ws_id(self, tmp_db):
|
||||
"""If ws_id changes (via resume) during title generation, discard the result."""
|
||||
from turnstone.core.providers._protocol import ModelCapabilities
|
||||
|
||||
session = _make_session()
|
||||
session._title_generated = True
|
||||
session.messages = [
|
||||
@@ -803,6 +811,7 @@ class TestTitleRetry:
|
||||
result = MagicMock()
|
||||
result.content = "Test Title"
|
||||
session._provider = MagicMock()
|
||||
session._provider.get_capabilities.return_value = ModelCapabilities()
|
||||
session._provider.create_completion.return_value = result
|
||||
|
||||
# Simulate resume() changing ws_id while title generation is in flight
|
||||
|
||||
@@ -1,8 +1,17 @@
|
||||
"""Tests for the storage backend registry."""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import patch
|
||||
|
||||
from turnstone.core.storage import get_storage, init_storage, reset_storage
|
||||
import pytest
|
||||
import sqlalchemy as sa
|
||||
|
||||
from turnstone.core.storage import (
|
||||
StorageUnavailableError,
|
||||
get_storage,
|
||||
init_storage,
|
||||
reset_storage,
|
||||
)
|
||||
from turnstone.core.storage._postgresql import PostgreSQLBackend
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
|
||||
@@ -52,3 +61,69 @@ class TestResetStorage:
|
||||
init_storage("sqlite", path=str(tmp_path / "test2.db"), run_migrations=False)
|
||||
s2 = get_storage()
|
||||
assert s1 is not s2
|
||||
|
||||
|
||||
class TestConnUnavailableLogging:
|
||||
"""Test that _conn() deduplicates DB unavailable/restored logging."""
|
||||
|
||||
def _make_backend(self, tmp_path):
|
||||
"""Create a minimal SQLite backend for testing _conn()."""
|
||||
from turnstone.core.storage._sqlite import SQLiteBackend
|
||||
|
||||
return SQLiteBackend(str(tmp_path / "test.db"), create_tables=True)
|
||||
|
||||
def test_logs_unavailable_once(self, tmp_path, caplog: pytest.LogCaptureFixture) -> None:
|
||||
backend = self._make_backend(tmp_path)
|
||||
with patch.object(backend, "_engine") as mock_engine:
|
||||
mock_engine.connect.side_effect = sa.exc.OperationalError(
|
||||
"conn", {}, Exception("refused")
|
||||
)
|
||||
for _ in range(3):
|
||||
with pytest.raises(StorageUnavailableError), backend._conn():
|
||||
pass # pragma: no cover
|
||||
unavailable_msgs = [r for r in caplog.records if "database.unavailable" in r.message]
|
||||
assert len(unavailable_msgs) == 1
|
||||
|
||||
def test_logs_restored_on_recovery(self, tmp_path, caplog: pytest.LogCaptureFixture) -> None:
|
||||
import logging
|
||||
|
||||
caplog.set_level(logging.INFO)
|
||||
backend = self._make_backend(tmp_path)
|
||||
# Simulate outage
|
||||
with patch.object(backend, "_engine") as mock_engine:
|
||||
mock_engine.connect.side_effect = sa.exc.OperationalError(
|
||||
"conn", {}, Exception("refused")
|
||||
)
|
||||
with pytest.raises(StorageUnavailableError), backend._conn():
|
||||
pass # pragma: no cover
|
||||
assert backend._db_unavailable is True
|
||||
# Real connection — should log restored
|
||||
caplog.clear()
|
||||
with backend._conn():
|
||||
pass
|
||||
restored_msgs = [r for r in caplog.records if "database.connection_restored" in r.message]
|
||||
assert len(restored_msgs) == 1
|
||||
assert backend._db_unavailable is False
|
||||
|
||||
def test_postgresql_conn_raises_storage_unavailable(self) -> None:
|
||||
import threading
|
||||
|
||||
backend = PostgreSQLBackend.__new__(PostgreSQLBackend)
|
||||
backend._db_unavailable = False
|
||||
backend._db_unavailable_lock = threading.Lock()
|
||||
|
||||
def _raise_op_error():
|
||||
raise sa.exc.OperationalError("conn", {}, Exception("refused"))
|
||||
|
||||
mock_engine = type(
|
||||
"E",
|
||||
(),
|
||||
{
|
||||
"connect": staticmethod(_raise_op_error),
|
||||
"url": sa.engine.make_url("postgresql://user:pass@localhost/db"),
|
||||
},
|
||||
)()
|
||||
backend._engine = mock_engine
|
||||
with pytest.raises(StorageUnavailableError), backend._conn():
|
||||
pass # pragma: no cover
|
||||
assert backend._db_unavailable is True
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""turnstone - Multi-node AI orchestration platform with tool use, agent routing, and cluster simulation."""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
__version__ = "1.0.3"
|
||||
|
||||
+97
-38
@@ -2,14 +2,15 @@
|
||||
|
||||
Entry point: turnstone-bootstrap
|
||||
|
||||
Walks users through configuring a single-node or multi-node Turnstone
|
||||
deployment via a conversational AI assistant. Generates .env files,
|
||||
docker-compose overrides, and post-start setup scripts.
|
||||
Walks users through configuring a Turnstone deployment via a conversational
|
||||
AI assistant. Generates compose.yaml, .env files, and post-start setup
|
||||
scripts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import getpass
|
||||
import importlib.resources
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
@@ -60,8 +61,6 @@ Turnstone is a multi-node AI orchestration platform. A deployment consists of:
|
||||
## Deployment Profiles (compose.yaml)
|
||||
- **Default** (no flag): console only (infrastructure, good for running external servers)
|
||||
- **Production** (`--profile production`): 1 server + console + PostgreSQL + channel (single node)
|
||||
- **Cluster** (`--profile cluster`): 10-node server fleet + PostgreSQL + channel + console (multi-node)
|
||||
- **ddgCluster** (`--profile ddgCluster`): Cluster + DuckDuckGo Search MCP sidecar (web search via MCP, no API key needed)
|
||||
|
||||
## Environment Variables (.env)
|
||||
The compose.yaml reads these from a `.env` file:
|
||||
@@ -78,9 +77,9 @@ For commercial providers (OpenAI, Anthropic-via-proxy), use the real key.
|
||||
|
||||
### Database
|
||||
- `DB_BACKEND` — `sqlite` (default) or `postgresql`
|
||||
- `DATABASE_URL` — PostgreSQL connection string (production/cluster only)
|
||||
- `DATABASE_URL` — PostgreSQL connection string (production only)
|
||||
- `POSTGRES_USER` — PostgreSQL username (default: turnstone)
|
||||
- `POSTGRES_PASSWORD` — PostgreSQL password (required for production/cluster)
|
||||
- `POSTGRES_PASSWORD` — PostgreSQL password (required for production)
|
||||
|
||||
### Authentication (always enabled)
|
||||
- `TURNSTONE_JWT_SECRET` — JWT signing secret (required). All services must share the same secret. \
|
||||
@@ -104,16 +103,15 @@ Generate with: `python -c "import secrets; print(secrets.token_hex(32))"`
|
||||
- `TURNSTONE_DISCORD_TOKEN` — Discord bot token
|
||||
- `TURNSTONE_DISCORD_GUILD` — Restrict to single guild ID
|
||||
|
||||
### MCP Integration (optional)
|
||||
- `MCP_CONFIG` — Path to MCP server config inside the container \
|
||||
(e.g., `/etc/turnstone/mcp-ddg.json`). When set, servers connect to configured MCP servers on startup.
|
||||
- The `ddgCluster` profile runs a DuckDuckGo Search MCP sidecar (Python) that provides \
|
||||
`duckduckgo_web_search` and `duckduckgo_fetch_content` tools to every node. No API key required. \
|
||||
The sidecar uses MCP streamable-http transport with DNS rebinding protection disabled \
|
||||
(required for Docker internal networking) and binds to 0.0.0.0:3000 via FastMCP settings. \
|
||||
Safe search is disabled by default.
|
||||
### Docker Image
|
||||
- `TURNSTONE_IMAGE_TAG` — Docker image tag (default: `latest`). \
|
||||
Set this to pin the image version (e.g., `1.1.0a3`, `stable`, `experimental`).
|
||||
|
||||
### Cluster
|
||||
### MCP Integration (optional)
|
||||
- `MCP_CONFIG` — Path to MCP server config inside the container. \
|
||||
When set, servers connect to configured MCP servers on startup.
|
||||
|
||||
### Other
|
||||
- `APPROVAL_TIMEOUT` — Tool approval timeout in seconds (default: 3600)
|
||||
|
||||
## Auth Setup Flow
|
||||
@@ -154,28 +152,28 @@ Categories like "engineering", "analysis", etc.
|
||||
## Your Task
|
||||
Walk the user through setting up their deployment step by step:
|
||||
|
||||
1. **First**: Call `check_docker` and `read_file` on `.env` to detect existing state.
|
||||
2. **Deployment mode**: Ask if they want single-node (`--profile production`) or multi-node \
|
||||
(`--profile cluster`). Explain trade-offs.
|
||||
3. **LLM provider for the deployment**: Which LLM backend their Turnstone will use \
|
||||
1. **First**: Call `check_docker`, `read_file` on `.env`, and `read_file` on `compose.yaml` \
|
||||
to detect existing state. If `compose.yaml` does not exist, call `write_compose` to \
|
||||
extract the bundled production compose file. This is essential — without it, \
|
||||
`docker compose` will fail.
|
||||
2. **LLM provider for the deployment**: Which LLM backend their Turnstone will use \
|
||||
(may differ from this wizard's model). Ask for base URL, API key, model name.
|
||||
4. **Database**: SQLite (dev/simple) vs PostgreSQL (production/cluster). \
|
||||
PostgreSQL is required for cluster mode.
|
||||
5. **Security**: Auth is always enabled and requires `TURNSTONE_JWT_SECRET`. \
|
||||
3. **Database**: SQLite (dev/simple) vs PostgreSQL (production). \
|
||||
PostgreSQL is recommended for production use.
|
||||
4. **Security**: Auth is always enabled and requires `TURNSTONE_JWT_SECRET`. \
|
||||
Use `generate_secret` for JWT secret and Postgres password. \
|
||||
Always set `TURNSTONE_JWT_SECRET` in the .env. \
|
||||
Ask for initial admin username and password. \
|
||||
If the user's deployment will use an external identity provider (Okta, Azure AD, Google, etc.), \
|
||||
offer to configure OIDC SSO. Ask for the issuer URL, client ID, and client secret. \
|
||||
Optionally configure role mapping and OIDC-only mode.
|
||||
6. **Ports**: Check defaults with `check_port`, suggest alternatives if conflicts.
|
||||
7. **Optional features**: Discord integration, web search (Tavily key), \
|
||||
DuckDuckGo Search MCP (for cluster — uses `ddgCluster` profile with \
|
||||
`MCP_CONFIG=/etc/turnstone/mcp-ddg.json`, no API key needed).
|
||||
8. **Generate .env**: Call `write_file` with the complete `.env` content.
|
||||
9. **Generate setup.sh**: Call `write_file` with a post-start script that creates the admin \
|
||||
5. **Ports**: Check defaults with `check_port`, suggest alternatives if conflicts.
|
||||
6. **Optional features**: Discord integration, web search (Tavily key).
|
||||
7. **Generate .env**: Call `write_file` with the complete `.env` content. \
|
||||
Include `TURNSTONE_IMAGE_TAG` set to the version matching the installed package.
|
||||
8. **Generate setup.sh**: Call `write_file` with a post-start script that creates the admin \
|
||||
user and any roles/policies/skills the user wants.
|
||||
10. **Finish**: Call the `finish` tool with a summary of what was configured and the \
|
||||
9. **Finish**: Call the `finish` tool with a summary of what was configured and the \
|
||||
exact commands to run next (e.g., `docker compose --profile production up -d` then `./setup.sh`).
|
||||
|
||||
## Rules
|
||||
@@ -183,14 +181,9 @@ exact commands to run next (e.g., `docker compose --profile production up -d` th
|
||||
- NEVER echo API keys or passwords back to the user in your text responses.
|
||||
- ALWAYS use `generate_secret` for passwords and secrets — never invent them.
|
||||
- When writing files, use `write_file` — the user will see a preview and confirm.
|
||||
- If `compose.yaml` is missing, call `write_compose` before anything else. \
|
||||
The compose file uses pre-built images from ghcr.io — no local Docker build is needed.
|
||||
- If an existing .env is detected, summarize what's configured and ask what to change.
|
||||
- For cluster mode, the compose.yaml has a fixed 10-node fleet — no override needed.
|
||||
- For cluster + DuckDuckGo Search, use `--profile ddgCluster` instead of `--profile cluster`. \
|
||||
Set `MCP_CONFIG=/etc/turnstone/mcp-ddg.json` in `.env`. No API key needed. \
|
||||
The DuckDuckGo MCP sidecar starts automatically and all cluster nodes connect to it. \
|
||||
Note: the MCP SDK's DNS rebinding protection must be disabled for Docker-internal networking \
|
||||
(the compose.yaml handles this), and the server must bind to 0.0.0.0 (not 127.0.0.1) to be \
|
||||
reachable from other containers.
|
||||
- The `DATABASE_URL` for docker compose internal networking uses the hostname `postgres` \
|
||||
(e.g., `postgresql+psycopg://turnstone:<password>@postgres:5432/turnstone`).
|
||||
- For local LLM backends (vLLM, llama.cpp, Ollama, etc.), set `OPENAI_API_KEY=dummy` in the \
|
||||
@@ -342,6 +335,23 @@ TOOLS: list[dict[str, Any]] = [
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "write_compose",
|
||||
"description": (
|
||||
"Write the production Docker Compose file to the project directory. "
|
||||
"This extracts the compose.yaml bundled with Turnstone, which uses "
|
||||
"pre-built images from ghcr.io (no local Docker build required). "
|
||||
"The user will be shown a preview and asked to confirm."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"required": [],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
@@ -561,6 +571,54 @@ def _tool_check_docker(args: dict[str, Any]) -> str:
|
||||
return "\n".join(results)
|
||||
|
||||
|
||||
def _tool_write_compose(project_dir: Path, args: dict[str, Any]) -> str:
|
||||
"""Extract the bundled production compose.yaml to the project directory."""
|
||||
dest = project_dir / "compose.yaml"
|
||||
|
||||
# Read the bundled template
|
||||
try:
|
||||
ref = importlib.resources.files("turnstone.deploy").joinpath("compose.yaml")
|
||||
content = ref.read_text(encoding="utf-8")
|
||||
except Exception as exc:
|
||||
return f"Error: could not read bundled compose template: {exc}"
|
||||
|
||||
# Skip if identical
|
||||
if dest.exists():
|
||||
try:
|
||||
existing = dest.read_text(encoding="utf-8")
|
||||
if existing == content:
|
||||
return "compose.yaml already exists with identical content."
|
||||
except (OSError, UnicodeDecodeError):
|
||||
pass
|
||||
|
||||
line_count = content.count("\n") + (1 if content and not content.endswith("\n") else 0)
|
||||
|
||||
# Show preview
|
||||
print(f"\n{YELLOW} Writing compose.yaml ({line_count} lines){RESET}")
|
||||
print(f"{DIM}{'─' * 50}{RESET}")
|
||||
for line in content.split("\n")[:30]:
|
||||
print(f" {DIM}{line}{RESET}")
|
||||
if line_count > 30:
|
||||
print(f" {DIM}... ({line_count - 30} more lines){RESET}")
|
||||
print(f"{DIM}{'─' * 50}{RESET}")
|
||||
|
||||
try:
|
||||
choice = input(f"{BOLD}Write this file? [Y/n]{RESET} ").strip().lower()
|
||||
except (EOFError, KeyboardInterrupt):
|
||||
return "User cancelled the write."
|
||||
if choice in ("n", "no"):
|
||||
return "User declined to write compose.yaml."
|
||||
|
||||
dest.write_text(content, encoding="utf-8")
|
||||
|
||||
return (
|
||||
f"compose.yaml written successfully. "
|
||||
f"It uses ghcr.io/turnstonelabs/turnstone images. "
|
||||
f"Add TURNSTONE_IMAGE_TAG={__version__} to .env to pin the image "
|
||||
f"to the currently installed version, or omit it to use 'latest'."
|
||||
)
|
||||
|
||||
|
||||
class _FinishError(Exception):
|
||||
"""Raised by the finish tool to signal the wizard is done."""
|
||||
|
||||
@@ -581,11 +639,12 @@ TOOL_FUNCTIONS: dict[str, Any] = {
|
||||
"check_port": _tool_check_port,
|
||||
"validate_api_key": _tool_validate_api_key,
|
||||
"check_docker": _tool_check_docker,
|
||||
"write_compose": _tool_write_compose,
|
||||
"finish": _tool_finish,
|
||||
}
|
||||
|
||||
# Tools that need the project_dir argument
|
||||
_PROJECT_DIR_TOOLS = frozenset({"read_file", "write_file"})
|
||||
_PROJECT_DIR_TOOLS = frozenset({"read_file", "write_file", "write_compose"})
|
||||
|
||||
|
||||
def execute_tool(name: str, args: dict[str, Any], project_dir: Path) -> str:
|
||||
|
||||
@@ -282,10 +282,14 @@ def main() -> None:
|
||||
|
||||
async def _heartbeat_loop() -> None:
|
||||
"""Periodically update service heartbeat."""
|
||||
from turnstone.core.storage._registry import StorageUnavailableError
|
||||
|
||||
while True:
|
||||
await asyncio.sleep(30)
|
||||
try:
|
||||
await asyncio.to_thread(storage.heartbeat_service, "channel", service_id)
|
||||
except StorageUnavailableError:
|
||||
pass # already logged by storage layer
|
||||
except Exception:
|
||||
log.exception("channel.heartbeat_failed")
|
||||
|
||||
|
||||
@@ -289,9 +289,13 @@ class ClusterCollector:
|
||||
|
||||
def _discovery_loop(self) -> None:
|
||||
"""Periodically scan the service registry for active nodes."""
|
||||
from turnstone.core.storage._registry import StorageUnavailableError
|
||||
|
||||
while self._running:
|
||||
try:
|
||||
self._discover_nodes()
|
||||
except StorageUnavailableError:
|
||||
pass # already logged by storage layer
|
||||
except Exception:
|
||||
log.exception("Node discovery error")
|
||||
time.sleep(self._discovery_interval)
|
||||
|
||||
@@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Any
|
||||
import structlog
|
||||
|
||||
from turnstone.core.hash_ring import RING_SIZE, RingNode, bucket_of
|
||||
from turnstone.core.storage._registry import StorageUnavailableError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from turnstone.console.collector import ClusterCollector
|
||||
@@ -149,6 +150,8 @@ class Rebalancer:
|
||||
result = self.rebalance_once(trigger=trigger)
|
||||
self._last_result = result
|
||||
self._record_result_metrics(result)
|
||||
except StorageUnavailableError:
|
||||
pass # already logged by storage layer
|
||||
except Exception:
|
||||
log.exception("rebalancer.error")
|
||||
finally:
|
||||
|
||||
@@ -90,9 +90,13 @@ class TaskScheduler:
|
||||
|
||||
def _loop(self) -> None:
|
||||
"""Main scheduler loop — tick then sleep."""
|
||||
from turnstone.core.storage._registry import StorageUnavailableError
|
||||
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
self._tick()
|
||||
except StorageUnavailableError:
|
||||
pass # already logged by storage layer
|
||||
except Exception:
|
||||
log.exception("scheduler.tick_error")
|
||||
self._stop_event.wait(self._check_interval)
|
||||
|
||||
@@ -1167,10 +1167,14 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]:
|
||||
import asyncio
|
||||
|
||||
async def _console_heartbeat() -> None:
|
||||
from turnstone.core.storage._registry import StorageUnavailableError
|
||||
|
||||
while True:
|
||||
await asyncio.sleep(30)
|
||||
try:
|
||||
storage.heartbeat_service("console", "console")
|
||||
except StorageUnavailableError:
|
||||
pass # already logged by storage layer
|
||||
except Exception:
|
||||
log.warning("console.heartbeat_failed", exc_info=True)
|
||||
|
||||
|
||||
@@ -543,9 +543,13 @@ class AnthropicProvider:
|
||||
if extra_params and "thinking_budget_tokens" in extra_params:
|
||||
budget = extra_params["thinking_budget_tokens"]
|
||||
if budget > 0:
|
||||
# Budget must leave room for the response
|
||||
# Budget must be strictly less than max_tokens (API requirement).
|
||||
# If max_tokens is too small to fit even a minimal thinking
|
||||
# budget alongside the response, disable thinking entirely.
|
||||
if budget >= max_tokens:
|
||||
budget = max(1024, max_tokens - 1024)
|
||||
budget = max_tokens - 1024
|
||||
if budget < 1:
|
||||
return {}
|
||||
return {"thinking": {"type": "enabled", "budget_tokens": budget}}
|
||||
return {}
|
||||
|
||||
|
||||
+53
-31
@@ -924,10 +924,8 @@ class ChatSession:
|
||||
if asst_msg:
|
||||
snippet += f"\nAssistant: {asst_msg}"
|
||||
snippet += "\n\nTitle:"
|
||||
result = self._provider.create_completion(
|
||||
client=self.client,
|
||||
model=self.model,
|
||||
messages=[
|
||||
result = self._utility_completion(
|
||||
[
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
@@ -942,9 +940,6 @@ class ChatSession:
|
||||
{"role": "user", "content": snippet},
|
||||
],
|
||||
max_tokens=200,
|
||||
temperature=0.3,
|
||||
reasoning_effort="low",
|
||||
extra_params=self._provider_extra_params(reasoning_effort="low"),
|
||||
)
|
||||
raw = (result.content or "").strip()
|
||||
# Take first line, strip quotes
|
||||
@@ -1281,6 +1276,33 @@ class ChatSession:
|
||||
return {"chat_template_kwargs": kwargs}
|
||||
return None
|
||||
|
||||
def _utility_completion(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
*,
|
||||
max_tokens: int = 4096,
|
||||
temperature: float = 0.3,
|
||||
reasoning_effort: str = "low",
|
||||
) -> CompletionResult:
|
||||
"""Run a lightweight internal completion (title gen, compaction, extraction).
|
||||
|
||||
Threads ``reasoning_effort`` through both the direct keyword (for
|
||||
commercial providers) and ``extra_params`` (for local model servers)
|
||||
so callers don't need to duplicate it. ``max_tokens`` is clamped to
|
||||
the model's advertised output limit so small models don't error.
|
||||
"""
|
||||
caps = self._get_capabilities()
|
||||
clamped = min(max_tokens, caps.max_output_tokens) if caps.max_output_tokens else max_tokens
|
||||
return self._provider.create_completion(
|
||||
client=self.client,
|
||||
model=self.model,
|
||||
messages=messages,
|
||||
max_tokens=clamped,
|
||||
temperature=temperature,
|
||||
reasoning_effort=reasoning_effort,
|
||||
extra_params=self._provider_extra_params(reasoning_effort=reasoning_effort),
|
||||
)
|
||||
|
||||
# -- tool search helpers --------------------------------------------------
|
||||
|
||||
def _get_active_tools(self) -> list[dict[str, Any]] | None:
|
||||
@@ -2482,14 +2504,9 @@ class ChatSession:
|
||||
result: CompletionResult | None = None
|
||||
for attempt in range(self._MAX_RETRIES + 1):
|
||||
try:
|
||||
result = self._provider.create_completion(
|
||||
client=self.client,
|
||||
model=self.model,
|
||||
messages=summary_msgs,
|
||||
result = self._utility_completion(
|
||||
summary_msgs,
|
||||
max_tokens=summary_max_tokens,
|
||||
temperature=0.3,
|
||||
reasoning_effort="low",
|
||||
extra_params=self._provider_extra_params(reasoning_effort="low"),
|
||||
)
|
||||
break
|
||||
except Exception as e:
|
||||
@@ -6167,26 +6184,30 @@ class ChatSession:
|
||||
return call_id, msg
|
||||
|
||||
if not text.strip():
|
||||
return call_id, "(empty response from URL)"
|
||||
msg = "Error: fetch returned empty response"
|
||||
self._report_tool_result(call_id, "web_fetch", msg, is_error=True)
|
||||
return call_id, msg
|
||||
|
||||
original_len = len(text)
|
||||
self.ui.on_info(f"fetched {original_len} chars, extracting...")
|
||||
|
||||
# Phase 2: truncate for summarization context
|
||||
max_content = 50_000
|
||||
# Phase 2: truncate for summarization context.
|
||||
# Reserve ~25% of the context window for the extraction prompt
|
||||
# overhead (system message, URL, question) and response tokens.
|
||||
# Convert token budget to chars using the calibrated ratio.
|
||||
max_content = int(self.context_window * self._chars_per_token * 0.75)
|
||||
max_content = min(max(max_content, 50_000), 500_000) # 50k–500k
|
||||
if len(text) > max_content:
|
||||
text = (
|
||||
text[: max_content // 2]
|
||||
+ f"\n\n... [{len(text) - max_content} chars omitted] ...\n\n"
|
||||
+ text[-(max_content // 2) :]
|
||||
)
|
||||
# Prefer the beginning — page content is usually top-heavy.
|
||||
text = text[:max_content] + f"\n\n... [{len(text) - max_content} chars truncated] ...\n"
|
||||
|
||||
# Phase 3: summarization API call
|
||||
# Phase 3: summarization API call.
|
||||
# Use a generous max_tokens so thinking models don't starve the
|
||||
# visible answer, and pass reasoning_effort="low" to avoid wasting
|
||||
# budget on deep reasoning for a simple extraction task.
|
||||
try:
|
||||
result = self._provider.create_completion(
|
||||
client=self.client,
|
||||
model=self.model,
|
||||
messages=[
|
||||
result = self._utility_completion(
|
||||
[
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
@@ -6206,11 +6227,12 @@ class ChatSession:
|
||||
),
|
||||
},
|
||||
],
|
||||
max_tokens=2000,
|
||||
max_tokens=8192,
|
||||
temperature=0.2,
|
||||
extra_params=self._provider_extra_params(),
|
||||
)
|
||||
answer = result.content or "(no answer)"
|
||||
answer = result.content or ""
|
||||
if not answer:
|
||||
answer = "Error: extraction returned no answer"
|
||||
except Exception as e:
|
||||
answer = f"Extraction failed (page was fetched but summarization errored): {e}"
|
||||
|
||||
@@ -6218,7 +6240,7 @@ class ChatSession:
|
||||
call_id,
|
||||
"web_fetch",
|
||||
answer,
|
||||
is_error=answer.startswith("Extraction failed"),
|
||||
is_error=answer.startswith(("Error:", "Extraction failed")),
|
||||
)
|
||||
|
||||
return call_id, answer
|
||||
|
||||
@@ -4,10 +4,16 @@ Supports SQLite (default, zero-config) and PostgreSQL (multi-node, production).
|
||||
"""
|
||||
|
||||
from turnstone.core.storage._protocol import StorageBackend
|
||||
from turnstone.core.storage._registry import get_storage, init_storage, reset_storage
|
||||
from turnstone.core.storage._registry import (
|
||||
StorageUnavailableError,
|
||||
get_storage,
|
||||
init_storage,
|
||||
reset_storage,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"StorageBackend",
|
||||
"StorageUnavailableError",
|
||||
"get_storage",
|
||||
"init_storage",
|
||||
"reset_storage",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,6 +15,14 @@ log = get_logger(__name__)
|
||||
_storage: StorageBackend | None = None
|
||||
|
||||
|
||||
class StorageUnavailableError(Exception):
|
||||
"""Raised when the database is unreachable.
|
||||
|
||||
The storage layer has already logged a clean one-liner — callers
|
||||
should catch this to avoid duplicate tracebacks.
|
||||
"""
|
||||
|
||||
|
||||
def init_storage(
|
||||
backend: str = "sqlite",
|
||||
*,
|
||||
|
||||
+251
-207
File diff suppressed because it is too large
Load Diff
@@ -271,9 +271,13 @@ class WatchRunner:
|
||||
# -- Main loop -----------------------------------------------------------
|
||||
|
||||
def _run(self) -> None:
|
||||
from turnstone.core.storage._registry import StorageUnavailableError
|
||||
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
self._tick()
|
||||
except StorageUnavailableError:
|
||||
pass # already logged by storage layer
|
||||
except Exception:
|
||||
log.exception("watch_runner.tick_error")
|
||||
self._stop_event.wait(self._check_interval)
|
||||
|
||||
@@ -6,14 +6,20 @@ import socket
|
||||
from html import unescape as _html_unescape
|
||||
from urllib.parse import urlparse
|
||||
|
||||
_RE_INVISIBLE = re.compile(
|
||||
r"<(script|style|template|noscript)\b[^>]*>.*?</\1\s*>",
|
||||
re.DOTALL | re.IGNORECASE,
|
||||
)
|
||||
_RE_TAGS = re.compile(r"<[^>]+>")
|
||||
_RE_WS = re.compile(r"[ \t]+")
|
||||
_RE_BLANKLINES = re.compile(r"\n{3,}")
|
||||
|
||||
|
||||
def strip_html(html: str) -> str:
|
||||
"""Convert HTML to plain text: strip tags, decode entities, collapse whitespace."""
|
||||
text = _RE_TAGS.sub("", html)
|
||||
"""Convert HTML to plain text: strip invisible elements, tags, decode entities."""
|
||||
# Remove elements whose content should never appear as text
|
||||
text = _RE_INVISIBLE.sub("", html)
|
||||
text = _RE_TAGS.sub("", text)
|
||||
text = _html_unescape(text)
|
||||
text = _RE_WS.sub(" ", text)
|
||||
text = _RE_BLANKLINES.sub("\n\n", text)
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Bundled deployment templates (compose files, overlays).
|
||||
|
||||
These files are included in the wheel so that ``turnstone-bootstrap`` can
|
||||
extract them for users who install via pip/pipx and don't have a git clone.
|
||||
"""
|
||||
@@ -0,0 +1,173 @@
|
||||
# =============================================================================
|
||||
# Turnstone Docker Compose Stack — Production
|
||||
#
|
||||
# This file is bundled with the turnstone wheel and written by
|
||||
# turnstone-bootstrap for users who install via pip/pipx.
|
||||
# It pulls pre-built images from ghcr.io instead of building locally.
|
||||
#
|
||||
# Usage:
|
||||
# Infra only: docker compose up
|
||||
# Single node: docker compose --profile production up
|
||||
# Production (PG): docker compose --profile production up
|
||||
# (set DB_BACKEND, DATABASE_URL, POSTGRES_PASSWORD in .env)
|
||||
#
|
||||
# Set TURNSTONE_IMAGE_TAG in .env to pin the image version (default: latest).
|
||||
# =============================================================================
|
||||
|
||||
name: turnstone
|
||||
|
||||
networks:
|
||||
turnstone-net:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
turnstone-data:
|
||||
workspace:
|
||||
postgres-data:
|
||||
|
||||
services:
|
||||
# -------------------------------------------------------------------
|
||||
# PostgreSQL — production database (profile: production)
|
||||
# -------------------------------------------------------------------
|
||||
postgres:
|
||||
image: pgautoupgrade/pgautoupgrade:18-alpine
|
||||
profiles:
|
||||
- production
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- max_connections=${POSTGRES_MAX_CONNECTIONS:-300}
|
||||
- -c
|
||||
- shared_buffers=128MB
|
||||
environment:
|
||||
POSTGRES_DB: turnstone
|
||||
POSTGRES_USER: ${POSTGRES_USER:-turnstone}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required for production profile}
|
||||
PGDATA: /var/lib/postgresql/data
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
networks:
|
||||
- turnstone-net
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-turnstone}"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 1G
|
||||
cpus: '1.0'
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# turnstone-server — Web UI + chat workstreams + LLM interaction
|
||||
# -------------------------------------------------------------------
|
||||
server:
|
||||
image: ghcr.io/turnstonelabs/turnstone:${TURNSTONE_IMAGE_TAG:-latest}
|
||||
profiles:
|
||||
- production
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
turnstone-server
|
||||
--host 0.0.0.0
|
||||
--port 8080
|
||||
--base-url "$${LLM_BASE_URL}"
|
||||
--api-key "$${OPENAI_API_KEY}"
|
||||
$${MODEL:+--model $$MODEL}
|
||||
$${SKIP_PERMISSIONS:+--skip-permissions}
|
||||
$${MCP_CONFIG:+--mcp-config $$MCP_CONFIG}
|
||||
ports:
|
||||
- "${SERVER_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- turnstone-data:/data
|
||||
- ${WORKSPACE_MOUNT:-workspace}:/workspace
|
||||
environment:
|
||||
- LLM_BASE_URL=${LLM_BASE_URL:-http://host.docker.internal:8000/v1}
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY:-dummy}
|
||||
- TAVILY_API_KEY=${TAVILY_API_KEY:-}
|
||||
- SKIP_PERMISSIONS=${SKIP_PERMISSIONS:-}
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
|
||||
- MODEL=${MODEL:-}
|
||||
- MCP_CONFIG=${MCP_CONFIG:-}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-}
|
||||
- TURNSTONE_NODE_ID=${TURNSTONE_NODE_ID:-}
|
||||
- TURNSTONE_ADVERTISE_URL=${TURNSTONE_ADVERTISE_URL:-http://server:8080}
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
required: false
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8080/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# turnstone-console — Cluster dashboard
|
||||
# -------------------------------------------------------------------
|
||||
console:
|
||||
image: ghcr.io/turnstonelabs/turnstone:${TURNSTONE_IMAGE_TAG:-latest}
|
||||
command:
|
||||
- turnstone-console
|
||||
- --host=0.0.0.0
|
||||
- --port=8090
|
||||
ports:
|
||||
- "${CONSOLE_PORT:-8090}:8090"
|
||||
environment:
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-}
|
||||
- TURNSTONE_CONSOLE_URL=http://console:8090
|
||||
networks:
|
||||
- turnstone-net
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "/usr/local/bin/healthcheck.py", "http://127.0.0.1:8090/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# turnstone-channel — Channel gateway (Discord, Slack, etc.)
|
||||
# Requires TURNSTONE_DISCORD_TOKEN to enable Discord adapter
|
||||
# -------------------------------------------------------------------
|
||||
channel:
|
||||
image: ghcr.io/turnstonelabs/turnstone:${TURNSTONE_IMAGE_TAG:-latest}
|
||||
profiles:
|
||||
- production
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
turnstone-channel
|
||||
--http-host=0.0.0.0
|
||||
$${TURNSTONE_DISCORD_GUILD:+--discord-guild $$TURNSTONE_DISCORD_GUILD}
|
||||
environment:
|
||||
- TURNSTONE_DISCORD_TOKEN=${TURNSTONE_DISCORD_TOKEN:-}
|
||||
- TURNSTONE_DISCORD_GUILD=${TURNSTONE_DISCORD_GUILD:-0}
|
||||
# Generate with: python -c "import secrets; print(secrets.token_hex(32))"
|
||||
- TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:?Set TURNSTONE_JWT_SECRET in .env}
|
||||
- TURNSTONE_DB_BACKEND=${DB_BACKEND:-postgresql}
|
||||
- TURNSTONE_DB_URL=${DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:-turnstone}@postgres:5432/turnstone}
|
||||
- TURNSTONE_CHANNEL_ADVERTISE_URL=http://channel:8091
|
||||
networks:
|
||||
- turnstone-net
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
required: false
|
||||
restart: unless-stopped
|
||||
@@ -2389,10 +2389,14 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]:
|
||||
|
||||
async def _heartbeat_loop() -> None:
|
||||
"""Periodically update service heartbeat."""
|
||||
from turnstone.core.storage._registry import StorageUnavailableError
|
||||
|
||||
while True:
|
||||
await asyncio.sleep(30)
|
||||
try:
|
||||
await asyncio.to_thread(_svc_storage.heartbeat_service, "server", _svc_node_id)
|
||||
except StorageUnavailableError:
|
||||
pass # already logged by storage layer
|
||||
except Exception:
|
||||
log.exception("server.heartbeat_failed")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user