feat: add dependency management with Renovate, uv.lock, and security … (#83)

* feat: add dependency management with Renovate, uv.lock, and security scanning

Adds automated dependency update detection and vulnerability scanning
across all dependency layers (Python, vendored JS, TypeScript SDK, Docker,
GitHub Actions).

- Renovate config with 10 package groups and custom regex managers for
  vendored JS (KaTeX, Highlight.js, Mermaid) tracking via npm registry
- uv.lock for reproducible builds (80 packages)
- Dockerfile switched to uv sync --frozen with layer caching
- CI: pip-audit (via lock file), npm audit, lock-check jobs
- CI: lint job uses pre-commit for ruff version consistency
- Docker security scan workflow (weekly Trivy, HIGH/CRITICAL)
- Helper script for vendored JS library updates

* fix: resolve CI failures and address review feedback

- Update pre-commit hooks: ruff v0.9.10 -> v0.15.6 (fixes deprecated
  UP038 rule), mypy v1.14.1 -> v1.19.1
- Add per-file-ignore for N802 on sandbox.py (ast visitor convention)
- Fix pip-audit: install into uv venv so uv run can find it
- Pin uv-version in CI to match lock file generator (0.9.18)
- Upgrade vitest ^2.0 -> ^4.1 to fix esbuild GHSA-67mh-4wv8-2f99
- Vendored JS script: use grep -rl for auto-discovery of version refs
  (catches docs/architecture.md), fix LICENSE comment, portable grep

.github/renovate.json

@@ -0,0 +1,136 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended",
+    "helpers:pinGitHubActionDigests",
+    ":separateMajorReleases"
+  ],
+  "labels": ["dependencies"],
+  "prConcurrentLimit": 5,
+  "prHourlyLimit": 2,
+  "schedule": ["before 9am on Monday"],
+  "timezone": "America/New_York",
+  "lockFileMaintenance": {
+    "enabled": true,
+    "schedule": ["before 9am on Monday"]
+  },
+  "customManagers": [
+    {
+      "customType": "regex",
+      "description": "Track vendored KaTeX version",
+      "fileMatch": ["pyproject\\.toml$"],
+      "matchStrings": ["katex-(?<currentValue>[\\d.]+)/"],
+      "depNameTemplate": "katex",
+      "datasourceTemplate": "npm"
+    },
+    {
+      "customType": "regex",
+      "description": "Track vendored Highlight.js version",
+      "fileMatch": ["pyproject\\.toml$"],
+      "matchStrings": ["hljs-(?<currentValue>[\\d.]+)/"],
+      "depNameTemplate": "highlight.js",
+      "datasourceTemplate": "npm"
+    },
+    {
+      "customType": "regex",
+      "description": "Track vendored Mermaid version",
+      "fileMatch": ["pyproject\\.toml$"],
+      "matchStrings": ["mermaid-(?<currentValue>[\\d.]+)/"],
+      "depNameTemplate": "mermaid",
+      "datasourceTemplate": "npm"
+    }
+  ],
+  "packageRules": [
+    {
+      "description": "LLM SDKs — always review manually",
+      "groupName": "LLM SDKs",
+      "matchPackageNames": ["openai", "anthropic", "mcp"],
+      "schedule": ["before 9am on Monday"],
+      "automerge": false
+    },
+    {
+      "description": "Web framework stack",
+      "groupName": "Web Framework",
+      "matchPackageNames": [
+        "starlette",
+        "uvicorn",
+        "sse-starlette",
+        "httpx",
+        "httpx-sse",
+        "pydantic"
+      ],
+      "schedule": ["before 9am on Wednesday"],
+      "automerge": true,
+      "matchUpdateTypes": ["patch"]
+    },
+    {
+      "description": "Database layer",
+      "groupName": "Database",
+      "matchPackageNames": ["sqlalchemy", "alembic", "psycopg"],
+      "schedule": ["before 9am on Wednesday"],
+      "automerge": true,
+      "matchUpdateTypes": ["patch"]
+    },
+    {
+      "description": "Security-critical — always review manually",
+      "groupName": "Security",
+      "matchPackageNames": ["PyJWT", "pyjwt", "bcrypt"],
+      "automerge": false
+    },
+    {
+      "description": "Infrastructure dependencies",
+      "groupName": "Infrastructure",
+      "matchPackageNames": ["structlog", "redis", "croniter", "discord.py"],
+      "schedule": ["before 9am on the first day of the month"],
+      "automerge": true,
+      "matchUpdateTypes": ["patch"]
+    },
+    {
+      "description": "Vendored JS — requires manual file download after merge",
+      "groupName": "Vendored JS",
+      "matchPackageNames": ["katex", "highlight.js", "mermaid"],
+      "schedule": ["before 9am on the first day of the month"],
+      "automerge": false,
+      "prBodyNotes": [
+        "This PR updates version references only.",
+        "After merging, run `scripts/update-vendored-js.sh <lib> <version>` to download the actual files."
+      ]
+    },
+    {
+      "description": "Dev/test tooling",
+      "groupName": "Tooling",
+      "matchPackageNames": [
+        "ruff",
+        "mypy",
+        "types-redis",
+        "pytest",
+        "pytest-cov",
+        "pre-commit"
+      ],
+      "schedule": ["before 9am on the first day of the month"],
+      "automerge": true,
+      "matchUpdateTypes": ["patch"]
+    },
+    {
+      "description": "Docker base images",
+      "groupName": "Docker Images",
+      "matchManagers": ["dockerfile", "docker-compose"],
+      "schedule": ["before 9am on the first day of the month"],
+      "automerge": false
+    },
+    {
+      "description": "TypeScript SDK dev dependencies",
+      "groupName": "TypeScript SDK",
+      "matchFileNames": ["sdk/typescript/**"],
+      "schedule": ["before 9am on the first day of the month"],
+      "automerge": true,
+      "matchUpdateTypes": ["patch"]
+    },
+    {
+      "description": "GitHub Actions — group all action updates",
+      "groupName": "GitHub Actions",
+      "matchManagers": ["github-actions"],
+      "automerge": false
+    }
+  ]
+}

.github/workflows/ci.yml

@@ -14,9 +14,9 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.13"
-      - run: pip install ruff
-      - run: ruff check turnstone/ tests/
-      - run: ruff format --check turnstone/ tests/
+      - run: pip install pre-commit
+      # mypy runs separately in typecheck job with full project deps
+      - run: SKIP=mypy pre-commit run --all-files
 
   typecheck:
     runs-on: ubuntu-latest
@@ -46,3 +46,39 @@ jobs:
         with:
           name: coverage-${{ matrix.python-version }}
           path: coverage.xml
+
+  lock-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          uv-version: "0.9.18"
+      - run: uv lock --check
+
+  security:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          uv-version: "0.9.18"
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - run: uv sync --frozen --all-extras
+      - run: uv pip install pip-audit
+      - run: uv run pip-audit --strict --desc
+
+  security-ts:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: sdk/typescript
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - run: npm ci
+      - run: npm audit --audit-level=moderate

.github/workflows/docker-scan.yml

@@ -0,0 +1,22 @@
+name: Docker Security Scan
+
+on:
+  push:
+    branches: [main]
+  schedule:
+    - cron: "0 6 * * 1"  # Weekly Monday 06:00 UTC
+
+permissions:
+  contents: read
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: docker build -t turnstone:scan .
+      - uses: aquasecurity/trivy-action@0.30.0
+        with:
+          image-ref: "turnstone:scan"
+          severity: "HIGH,CRITICAL"
+          exit-code: "1"

.pre-commit-config.yaml

@@ -1,13 +1,13 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.10
+    rev: v0.15.6
     hooks:
       - id: ruff
         args: [--fix]
       - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.14.1
+    rev: v1.19.1
     hooks:
       - id: mypy
         additional_dependencies: [types-redis>=4.6, redis>=7.2]

Dockerfile

@@ -1,30 +1,15 @@
 # =============================================================================
-# Turnstone — multi-stage Docker build
+# Turnstone — Docker build with uv for reproducible, locked installs
 # Single image for all services: server, bridge, console, sim, eval
 # =============================================================================
 
-# ----------------------------------------------------------------------------
-# Stage 1: Builder — build the wheel
-# ----------------------------------------------------------------------------
-FROM python:3.13-slim AS builder
-
-WORKDIR /build
-
-RUN pip install --no-cache-dir hatchling
-
-COPY pyproject.toml README.md LICENSE ./
-COPY turnstone/ turnstone/
-
-RUN pip wheel --no-deps --wheel-dir /build/wheels .
-
-# ----------------------------------------------------------------------------
-# Stage 2: Runtime — slim image with the installed package
-# ----------------------------------------------------------------------------
 FROM python:3.13-slim
 
 LABEL org.opencontainers.image.title="turnstone" \
       org.opencontainers.image.description="Multi-node AI orchestration platform"
 
+COPY --from=ghcr.io/astral-sh/uv:0.9.18 /uv /usr/local/bin/uv
+
 # System dependencies for psycopg (PostgreSQL client library)
 RUN apt-get update && apt-get install -y --no-install-recommends libpq5 \
     && rm -rf /var/lib/apt/lists/*
@@ -32,10 +17,23 @@ RUN apt-get update && apt-get install -y --no-install-recommends libpq5 \
 # Non-root user
 RUN useradd --create-home --shell /bin/bash turnstone
 
-# Install the wheel with all optional extras
-COPY --from=builder /build/wheels/*.whl /tmp/wheels/
-RUN pip install --no-cache-dir "$(ls /tmp/wheels/*.whl)[mq,console,sim,postgres,discord]" \
-    && rm -rf /tmp/wheels
+WORKDIR /app
+
+# Compile bytecode for faster startup
+ENV UV_COMPILE_BYTECODE=1
+
+# Install dependencies first (cached layer — only re-runs when deps change)
+COPY pyproject.toml uv.lock README.md LICENSE ./
+RUN uv sync --frozen --no-install-project --no-dev \
+    --extra mq --extra console --extra sim --extra postgres --extra discord --extra anthropic
+
+# Install the project itself
+COPY turnstone/ turnstone/
+RUN uv sync --frozen --no-dev \
+    --extra mq --extra console --extra sim --extra postgres --extra discord --extra anthropic
+
+# Add venv to PATH so entry points are found
+ENV PATH="/app/.venv/bin:$PATH"
 
 # Health check script (stdlib only, no pip deps needed)
 COPY docker/healthcheck.py /usr/local/bin/healthcheck.py

pyproject.toml

@@ -93,6 +93,7 @@ line-length = 100
 [tool.ruff.lint]
 select = ["E", "F", "W", "I", "N", "UP", "B", "A", "SIM", "TCH"]
 ignore = ["E501"]
+per-file-ignores = { "turnstone/core/sandbox.py" = ["N802"] }
 
 [tool.ruff.format]
 quote-style = "double"

scripts/update-vendored-js.sh

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+# Update a vendored JavaScript library in turnstone/shared_static/.
+#
+# Usage:
+#   scripts/update-vendored-js.sh katex 0.16.39
+#   scripts/update-vendored-js.sh hljs 11.12.0
+#   scripts/update-vendored-js.sh mermaid 11.14.0
+#
+# This script:
+#   1. Downloads the new version from CDN
+#   2. Creates the new versioned directory
+#   3. Updates all version references in source files
+#   4. Removes the old versioned directory
+
+set -euo pipefail
+
+STATIC_DIR="turnstone/shared_static"
+CDN="https://cdn.jsdelivr.net/npm"
+
+usage() {
+    echo "Usage: $0 <katex|hljs|mermaid> <version>"
+    echo "Example: $0 katex 0.16.39"
+    exit 1
+}
+
+[[ $# -eq 2 ]] || usage
+
+LIB="$1"
+VERSION="$2"
+
+# Detect current version from pyproject.toml
+detect_old_version() {
+    local pattern="$1"
+    grep -oE "${pattern}-[0-9.]+" pyproject.toml | head -1 | sed "s/${pattern}-//"
+}
+
+# Update version references across all source files
+update_refs() {
+    local old_pattern="$1"  # e.g. katex-0.16.38
+    local new_pattern="$2"  # e.g. katex-0.16.39
+
+    # Find all files with version references (excludes vendored JS and worktrees)
+    local files
+    files=$(grep -rl --include='*.toml' --include='*.html' --include='*.js' --include='*.md' \
+        -F "$old_pattern" . \
+        --exclude-dir='.claude' --exclude-dir='node_modules' --exclude-dir='shared_static' \
+        2>/dev/null || true)
+    for f in $files; do
+        sed -i "s|${old_pattern}|${new_pattern}|g" "$f"
+        echo "  Updated $f"
+    done
+}
+
+case "$LIB" in
+    katex)
+        OLD_VERSION=$(detect_old_version "katex")
+        OLD_DIR="${STATIC_DIR}/katex-${OLD_VERSION}"
+        NEW_DIR="${STATIC_DIR}/katex-${VERSION}"
+
+        echo "Updating KaTeX ${OLD_VERSION} -> ${VERSION}"
+        mkdir -p "${NEW_DIR}/fonts"
+
+        echo "  Downloading katex.min.js..."
+        curl -sSfL "${CDN}/katex@${VERSION}/dist/katex.min.js" -o "${NEW_DIR}/katex.min.js"
+        echo "  Downloading katex.min.css..."
+        curl -sSfL "${CDN}/katex@${VERSION}/dist/katex.min.css" -o "${NEW_DIR}/katex.min.css"
+
+        echo "  Downloading fonts..."
+        # Extract font filenames from the CSS
+        font_files=$(curl -sSfL "${CDN}/katex@${VERSION}/dist/katex.min.css" \
+            | grep -oE 'fonts/[^")]+' | sort -u)
+        for font in $font_files; do
+            if ! curl -sSfL "${CDN}/katex@${VERSION}/dist/${font}" -o "${NEW_DIR}/${font}" 2>/dev/null; then
+                echo "  WARNING: Failed to download font: ${font}"
+            fi
+        done
+
+        # Copy LICENSE from old dir if present
+        if [[ -f "${OLD_DIR}/LICENSE" ]]; then
+            cp "${OLD_DIR}/LICENSE" "${NEW_DIR}/LICENSE"
+        fi
+
+        update_refs "katex-${OLD_VERSION}" "katex-${VERSION}"
+        rm -rf "${OLD_DIR}"
+        echo "Done. Old directory removed: ${OLD_DIR}"
+        ;;
+
+    hljs)
+        OLD_VERSION=$(detect_old_version "hljs")
+        OLD_DIR="${STATIC_DIR}/hljs-${OLD_VERSION}"
+        NEW_DIR="${STATIC_DIR}/hljs-${VERSION}"
+
+        echo "Updating Highlight.js ${OLD_VERSION} -> ${VERSION}"
+        mkdir -p "${NEW_DIR}"
+
+        echo "  Downloading highlight.min.js..."
+        curl -sSfL "${CDN}/@highlightjs/cdn-assets@${VERSION}/highlight.min.js" -o "${NEW_DIR}/highlight.min.js"
+
+        if [[ -f "${OLD_DIR}/LICENSE" ]]; then
+            cp "${OLD_DIR}/LICENSE" "${NEW_DIR}/LICENSE"
+        fi
+
+        update_refs "hljs-${OLD_VERSION}" "hljs-${VERSION}"
+        rm -rf "${OLD_DIR}"
+        echo "Done. Old directory removed: ${OLD_DIR}"
+        ;;
+
+    mermaid)
+        OLD_VERSION=$(detect_old_version "mermaid")
+        OLD_DIR="${STATIC_DIR}/mermaid-${OLD_VERSION}"
+        NEW_DIR="${STATIC_DIR}/mermaid-${VERSION}"
+
+        echo "Updating Mermaid ${OLD_VERSION} -> ${VERSION}"
+        mkdir -p "${NEW_DIR}"
+
+        echo "  Downloading mermaid.min.js..."
+        curl -sSfL "${CDN}/mermaid@${VERSION}/dist/mermaid.min.js" -o "${NEW_DIR}/mermaid.min.js"
+
+        if [[ -f "${OLD_DIR}/LICENSE" ]]; then
+            cp "${OLD_DIR}/LICENSE" "${NEW_DIR}/LICENSE"
+        fi
+
+        update_refs "mermaid-${OLD_VERSION}" "mermaid-${VERSION}"
+        rm -rf "${OLD_DIR}"
+        echo "Done. Old directory removed: ${OLD_DIR}"
+        ;;
+
+    *)
+        echo "Unknown library: ${LIB}"
+        usage
+        ;;
+esac
+
+echo ""
+echo "Verify the update:"
+echo "  git diff --stat"
+echo "  python -m turnstone.server  # test locally"

sdk/typescript/package.json

@@ -33,6 +33,6 @@
   "license": "BUSL-1.1",
   "devDependencies": {
     "typescript": "^5.4",
-    "vitest": "^2.0"
+    "vitest": "^4.1"
   }
 }