diff --git a/.env.example b/.env.example index 0af5a660..d3e7a5b8 100644 --- a/.env.example +++ b/.env.example @@ -1,29 +1,49 @@ # ============================================================================= # Turnstone Environment Variables -# Copy to .env and adjust values for your deployment +# Copy to .env and adjust values for your deployment. +# +# Usage: +# Single node: docker compose --profile production up +# 10-node cluster: docker compose --profile cluster up # ============================================================================= # -- LLM Backend -------------------------------------------------------------- LLM_BASE_URL=http://host.docker.internal:8000/v1 -OPENAI_API_KEY=sk-... -# ANTHROPIC_API_KEY=sk-ant-... # Set instead for Anthropic provider -# TAVILY_API_KEY=tvly-... # For web search fallback (local models only) +OPENAI_API_KEY=dummy +# ANTHROPIC_API_KEY=sk-ant-...# Set instead of OPENAI_API_KEY for Anthropic +# TAVILY_API_KEY=tvly-... # Web search fallback (local models only) +# MODEL=# Override default model alias -# -- Database (production profile) -------------------------------------------- +# -- Authentication (required) ------------------------------------------------ +# Generate with: python -c "import secrets; print(secrets.token_hex(32))" +TURNSTONE_JWT_SECRET=changeme-to-32-bytes-of-hex + +# -- Database ------------------------------------------------------------------ +# Single-node default is SQLite (zero config). Set these for PostgreSQL: # DB_BACKEND=postgresql # POSTGRES_USER=turnstone # POSTGRES_PASSWORD=changeme # DATABASE_URL=postgresql+psycopg://turnstone:changeme@postgres:5432/turnstone -# -- Redis --------------------------------------------------------------------- -# REDIS_PASSWORD= -# REDIS_PORT=6379 - -# -- Authentication ------------------------------------------------------------ -# TURNSTONE_AUTH_ENABLED=true -# TURNSTONE_AUTH_TOKEN=your-secret-token -# TURNSTONE_JWT_SECRET=python -c "import secrets; print(secrets.token_hex(32))" - # -- Ports --------------------------------------------------------------------- # SERVER_PORT=8080 # CONSOLE_PORT=8090 + +# -- Workspace ----------------------------------------------------------------- +# Bind-mount a host directory into the container at /workspace. +# The model can read/write files here. Default: empty Docker volume. +# WORKSPACE_MOUNT=/path/to/your/project + +# -- Agent behavior ------------------------------------------------------------ +# SKIP_PERMISSIONS=true # Auto-approve all tool calls (dev only) +# MCP_CONFIG=/workspace/mcp.json# MCP server configuration file + +# -- Discord channel gateway --------------------------------------------------- +# TURNSTONE_DISCORD_TOKEN= +# TURNSTONE_DISCORD_GUILD=0 + +# -- Cluster (profile: cluster) ----------------------------------------------- +# These are set per-node in compose.yaml; only override for custom topologies. +# TURNSTONE_NODE_ID=node-1 +# TURNSTONE_ADVERTISE_URL=http://server-1:8080 + diff --git a/.github/renovate.json b/.github/renovate.json index 8b4e3147..6cee1175 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -83,7 +83,7 @@ { "description": "Infrastructure dependencies", "groupName": "Infrastructure", - "matchPackageNames": ["structlog", "redis", "croniter", "discord.py"], + "matchPackageNames": ["structlog", "croniter", "discord.py"], "schedule": ["before 9am on the first day of the month"], "automerge": true, "matchUpdateTypes": ["patch"] @@ -101,7 +101,6 @@ "matchPackageNames": [ "ruff", "mypy", - "types-redis", "pytest", "pytest-cov", "pre-commit" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e998c45e..551d4e61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,10 @@ name: CI on: push: - branches: [main] + branches: [main, "stable/*"] + tags: ["v*"] pull_request: - branches: [main] + branches: [main, "stable/*"] jobs: lint: @@ -25,8 +26,8 @@ jobs: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: "3.14" - - run: pip install mypy types-redis - - run: pip install -e ".[mq]" + - run: pip install mypy + - run: pip install -e ".[all]" - run: mypy turnstone/ test: @@ -39,7 +40,7 @@ jobs: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: ${{ matrix.python-version }} - - run: pip install -e ".[test,mq]" + - run: pip install -e ".[test]" - run: pytest tests/ -m "not live" --cov=turnstone --cov-report=term-missing --cov-report=xml -q - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 if: always() @@ -68,7 +69,7 @@ jobs: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: "3.14" - - run: pip install -e ".[test,mq,postgres]" + - run: pip install -e ".[test,postgres]" - run: pytest tests/ -m "not live" --storage-backend=postgresql -q env: TURNSTONE_TEST_PG_URL: postgresql+psycopg://postgres:postgres@localhost:5432/turnstone_test diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 00000000..6f1eb5a8 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,75 @@ +name: Publish Docker Image + +on: + workflow_run: + workflows: ["CI"] + types: [completed] + +permissions: + contents: read + packages: write + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + docker: + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: ${{ github.event.workflow_run.head_sha }} + fetch-depth: 0 + + - name: Resolve release tag + id: tag + run: | + TAG=$(git tag --points-at HEAD | grep '^v' | head -1) + if [ -z "$TAG" ]; then + echo "No v* tag at HEAD — skipping publish" + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + + - name: Log in to GHCR + if: steps.tag.outputs.skip == 'false' + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute Docker tags + if: steps.tag.outputs.skip == 'false' + id: tags + env: + REF: ${{ steps.tag.outputs.tag }} + run: | + VERSION="${REF#v}" + FULL="${REGISTRY}/${IMAGE_NAME}" + FULL="${FULL,,}" + + if echo "$VERSION" | grep -qE '(a|b|rc)[0-9]+$'; then + TAGS="${FULL}:${VERSION},${FULL}:experimental" + else + MINOR="${VERSION%.*}" + TAGS="${FULL}:${VERSION},${FULL}:${MINOR},${FULL}:stable,${FULL}:latest" + fi + echo "tags=${TAGS}" >> "$GITHUB_OUTPUT" + + - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3 + if: steps.tag.outputs.skip == 'false' + + - name: Build and push + if: steps.tag.outputs.skip == 'false' + uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6 + with: + context: . + push: true + tags: ${{ steps.tags.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/docker-scan.yml b/.github/workflows/docker-scan.yml index 4a1db50a..552b552b 100644 --- a/.github/workflows/docker-scan.yml +++ b/.github/workflows/docker-scan.yml @@ -2,7 +2,7 @@ name: Docker Security Scan on: push: - branches: [main] + branches: [main, "stable/*"] schedule: - cron: "0 6 * * 1" # Weekly Monday 06:00 UTC diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ce54cf00..30467dcf 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,8 +1,9 @@ name: Publish to PyPI on: - push: - tags: ["v*"] + workflow_run: + workflows: ["CI"] + types: [completed] permissions: contents: write @@ -10,20 +11,43 @@ permissions: jobs: publish: + if: github.event.workflow_run.conclusion == 'success' runs-on: ubuntu-latest environment: pypi steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: ${{ github.event.workflow_run.head_sha }} + fetch-depth: 0 + + - name: Resolve release tag + id: tag + run: | + TAG=$(git tag --points-at HEAD | grep '^v' | head -1) + if [ -z "$TAG" ]; then + echo "No v* tag at HEAD — skipping publish" + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + if: steps.tag.outputs.skip == 'false' with: python-version: "3.14" - run: pip install build + if: steps.tag.outputs.skip == 'false' - run: python -m build + if: steps.tag.outputs.skip == 'false' - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1 + if: steps.tag.outputs.skip == 'false' - name: Create GitHub Release + if: steps.tag.outputs.skip == 'false' uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2 with: + tag_name: ${{ steps.tag.outputs.tag }} generate_release_notes: true draft: false - prerelease: ${{ contains(github.ref, '-') }} + prerelease: ${{ contains(steps.tag.outputs.tag, 'a') || contains(steps.tag.outputs.tag, 'b') || contains(steps.tag.outputs.tag, 'rc') }} diff --git a/Dockerfile b/Dockerfile index cc8944b5..40a8c33f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,6 +55,9 @@ COPY docker/entrypoint.sh /usr/local/bin/entrypoint.sh WORKDIR /data RUN chown turnstone:turnstone /data +# Workspace mount point — bind-mount a host directory here +RUN mkdir -p /workspace && chown turnstone:turnstone /workspace + USER turnstone ENTRYPOINT ["entrypoint.sh"] diff --git a/README.md b/README.md index 50e37e35..696d6434 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,21 @@ Multi-node AI orchestration platform. Deploy tool-using AI agents across a cluster of servers with direct HTTP routing, interactive interfaces, and enterprise governance. -> **Beta — Use at your own risk.** APIs, configuration formats, and database schemas may change between versions without migration paths. +

+ Turnstone console — multi-workstream AI orchestration with mermaid diagrams +

Named after the [Ruddy Turnstone](https://en.wikipedia.org/wiki/Ruddy_turnstone) (*Arenaria interpres*) — a shorebird that flips stones to discover what's hiding underneath. +### Release Tracks + +| Track | Install | Docker | Description | +|-------|---------|--------|-------------| +| **Stable** | `pip install turnstone` | `ghcr.io/turnstonelabs/turnstone:stable` | Production-grade. Bugfixes only. | +| **Experimental** | `pip install turnstone --pre` | `ghcr.io/turnstonelabs/turnstone:experimental` | New features. May have rough edges. | + +See [docs/releasing.md](docs/releasing.md) for the full release process. + ## What it does Turnstone gives LLMs tools — shell, files, search, web, planning — and orchestrates multi-turn conversations where the model investigates, acts, and reports. diff --git a/compose.yaml b/compose.yaml index 308ad593..9578caee 100644 --- a/compose.yaml +++ b/compose.yaml @@ -16,6 +16,7 @@ networks: volumes: turnstone-data: + workspace: postgres-data: services: @@ -80,6 +81,7 @@ services: - "${SERVER_PORT:-8080}:8080" volumes: - turnstone-data:/data + - ${WORKSPACE_MOUNT:-workspace}:/workspace environment: - LLM_BASE_URL=${LLM_BASE_URL:-http://host.docker.internal:8000/v1} - OPENAI_API_KEY=${OPENAI_API_KEY:-dummy} @@ -201,6 +203,7 @@ services: $${MCP_CONFIG:+--mcp-config $$MCP_CONFIG} volumes: - turnstone-data:/data + - ${WORKSPACE_MOUNT:-workspace}:/workspace environment: &cluster-server-env LLM_BASE_URL: ${LLM_BASE_URL:-http://host.docker.internal:8000/v1} OPENAI_API_KEY: ${OPENAI_API_KEY:-dummy} diff --git a/docs/assets/hero.png b/docs/assets/hero.png new file mode 100644 index 00000000..56198a37 --- /dev/null +++ b/docs/assets/hero.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75c1832b6079e8628f4bbf4ce98d37880c4de133636b7555e3869990b046ddc6 +size 567704 diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 00000000..7d3eee12 --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,80 @@ +# Release Process + +Turnstone uses two parallel release tracks published from a single PyPI package. + +## Release Tracks + +| Track | Versions | Branch | Docker tags | PyPI install | +|-------|----------|--------|-------------|--------------| +| **Stable** | `1.0.0`, `1.0.1` | `stable/1.0` | `:1.0.1`, `:1.0`, `:stable`, `:latest` | `pip install turnstone` | +| **Experimental** | `1.1.0a1`, `1.1.0a2` | `main` | `:1.1.0a1`, `:experimental` | `pip install turnstone --pre` | + +- **Stable** receives bugfixes only. Production-grade. +- **Experimental** receives new features. May be rough around the edges. +- When experimental matures, it is promoted to stable. The previous stable branch stops receiving patches. + +## Version Scheme + +[PEP 440](https://peps.python.org/pep-0440/) pre-release suffixes on a single package: + +- `1.0.0` — stable release +- `1.1.0a1` — alpha (experimental) +- `1.1.0b1` — beta (experimental, more stable) +- `1.1.0rc1` — release candidate (experimental, nearly stable) +- `1.1.0` — promoted to stable + +## Releasing an Experimental Version (from main) + +```bash +scripts/release.sh 1.1.0a2 --push +``` + +This bumps `pyproject.toml` + `turnstone/__init__.py`, regenerates `uv.lock`, commits, tags `v1.1.0a2`, and pushes. CI runs, then publish + Docker workflows fire automatically. + +## Releasing a Stable Patch (from stable/X.Y) + +```bash +git checkout stable/1.0 +git cherry-pick # bugfix from main +scripts/release.sh 1.0.2 --push +``` + +## Promoting Experimental to Stable + +When `main` is ready for a stable release: + +```bash +# 1. Tag the stable release on main +scripts/release.sh 1.1.0 --push + +# 2. Create the stable maintenance branch from that tag +git branch stable/1.1 v1.1.0 +git push origin stable/1.1 + +# 3. Start the next experimental cycle on main +scripts/release.sh 1.2.0a1 --push +``` + +The previous `stable/1.0` branch stops receiving patches at this point. + +## CI/CD Pipeline + +All releases are gated on CI success: + +1. `git push` with `v*` tag triggers **CI** (lint, typecheck, test, test-postgres, lock-check, security audit) +2. On CI success, **Publish to PyPI** fires via `workflow_run` +3. On CI success, **Publish Docker Image** fires via `workflow_run` + +Pre-release tags (`a`, `b`, `rc` suffixes) produce: +- PyPI: pre-release version (not installed by default) +- GitHub Release: marked as pre-release +- Docker: `:experimental` alias + exact version tag + +Stable tags produce: +- PyPI: stable version (default `pip install`) +- GitHub Release: full release +- Docker: `:stable`, `:latest`, `:X.Y`, `:X.Y.Z` tags + +## Dependency Updates + +Renovate targets `main` (experimental) only. Stable branches receive manual dependency updates via cherry-pick when security-relevant. diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 00000000..5c18d371 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Bump version, regenerate lockfile, commit, and tag. +# +# Usage: +# scripts/release.sh 1.0.0 # stable release +# scripts/release.sh 1.1.0a1 # experimental pre-release +# scripts/release.sh 1.0.1 --push # bump + push tag to origin +# +set -euo pipefail + +VERSION="${1:?Usage: scripts/release.sh VERSION [--push]}" +PUSH="${2:-}" + +# Validate PEP 440 version +if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(a[0-9]+|b[0-9]+|rc[0-9]+)?$'; then + echo "error: invalid PEP 440 version: $VERSION" >&2 + echo " examples: 1.0.0, 1.1.0a1, 1.0.1rc2" >&2 + exit 1 +fi + +TAG="v${VERSION}" + +# Check for clean working tree +if ! git diff --quiet || ! git diff --cached --quiet; then + echo "error: working tree is dirty — commit or stash first" >&2 + exit 1 +fi + +# Check tag doesn't already exist +if git rev-parse "$TAG" >/dev/null 2>&1; then + echo "error: tag $TAG already exists" >&2 + exit 1 +fi + +# Detect current version +CURRENT=$(grep -oP '(?<=^version = ")[^"]+' pyproject.toml) +echo "Bumping $CURRENT → $VERSION" + +# Update version in both files +sed -i "s/^version = \".*\"/version = \"$VERSION\"/" pyproject.toml +sed -i "s/^__version__ = \".*\"/__version__ = \"$VERSION\"/" turnstone/__init__.py + +# Regenerate lockfile +echo "Regenerating uv.lock..." +uv lock + +# Commit and tag +git add pyproject.toml turnstone/__init__.py uv.lock +git commit -m "chore: bump version to $VERSION" +git tag "$TAG" + +echo "" +echo "Created commit and tag $TAG" + +if [ "$PUSH" = "--push" ]; then + BRANCH=$(git rev-parse --abbrev-ref HEAD) + echo "Pushing $BRANCH + $TAG to origin..." + git push origin "$BRANCH" "$TAG" +else + echo "Run 'git push origin $TAG' to publish" +fi