mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-23 04:14:47 -06:00
33d16d19ce
* fix(renderer): handle LaTeX-style \(...\) and \[...\] math delimiters The browser renderer at turnstone/shared_static/renderer.js only recognized TeX-style $...$ / $$...$$ delimiters. Most modern LLMs (GPT-5 / o-series, Claude with reasoning effort) emit LaTeX-style \(...\) for inline math and \[...\] for display by default — those slipped through as raw text in the coord + interactive WebUIs, making KaTeX appear "broken when nested inside a markdown block" (actually broken everywhere, the surrounding markdown just made the failure noticeable). Added a second pass for each delimiter style alongside the existing $...$ / $$...$$ patterns. Both styles now feed the same mathBlocks / inlineMaths placeholder pipeline so all the existing nested-block handling (lists, blockquotes, tables, bold, headings, details, post-render KaTeX markup) Just Works. Edge cases verified by the new test_renderer_js.py harness: - \(...\) inside inline code stays literal - \(...\) inside fenced code blocks stays literal - Solo \[ with no closing \] doesn't trigger spurious math - Markdown links [text](url) untouched (regex uses \[ \], not [ ]) - Mixed TeX + LaTeX delimiters in one message both render The harness drives renderer.js through Node via vm.runInThisContext with stubbed document/katex globals — first JS-side regression guard for the renderer; previously it had no test coverage at all. * fix(renderer): apply Copilot feedback on PR #425 Three review items from Copilot: 1. Display-math sentinel could leak through inline-code spans. The original ordering ran $$...$$ / \[...\] extraction BEFORE inline code, so a backtick span around math (e.g. `$$x$$` or `\[x\]`) had its delimiters consumed by the math regex and replaced with \x00MB…\x00. Inline code then captured the sentinel; restore order put MB after IC, leaving the null-byte placeholder visible inside the rendered <code>. Reorder: inline code first, then display math, then inline math. Code spans now seal their content before any math regex sees it. The reverse edge case (math containing backticks, e.g. \verb|`x`|) is much rarer and KaTeX rejects \verb anyway. 2. Inline LaTeX-style \(...\) regex used [\s\S]+? which allowed newlines, so an unterminated \( on one line would eat the next paragraph until it found a closing \). Aligned with the existing $...$ behavior by switching to [^\n]+? — display math (\[...\] / $$...$$) stays multi-line by design. 3. tests/test_renderer_js.py was guarded with a node-availability skip, but CI's test + test-postgres jobs didn't explicitly install Node, so the suite would have silently no-op'd if the runner image dropped Node. Added actions/setup-node@v5 to both jobs. Four new regression tests cover the leak (both delimiter styles inside backticks must stay literal) and the cross-paragraph span (both \(...\) and $...$ must not eat newlines).
173 lines
6.0 KiB
YAML
173 lines
6.0 KiB
YAML
name: CI
|
|
|
|
on:
|
|
push:
|
|
branches: [main, "stable/*"]
|
|
tags: ["v*"]
|
|
pull_request:
|
|
branches: [main, "stable/*"]
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
lint:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
|
with:
|
|
python-version: "3.14"
|
|
- run: pip install pre-commit
|
|
# mypy runs separately in typecheck job with full project deps
|
|
- run: SKIP=mypy pre-commit run --all-files
|
|
|
|
typecheck:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
|
with:
|
|
python-version: "3.14"
|
|
- run: pip install mypy
|
|
- run: pip install -e ".[all]"
|
|
- run: mypy turnstone/
|
|
|
|
test:
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
python-version: ["3.11", "3.12", "3.13"]
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
# Node is required by tests/test_renderer_js.py — without
|
|
# explicit setup, that suite silently skips if the runner
|
|
# image happens not to ship Node, masking regressions in
|
|
# the browser-side renderer.
|
|
- uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
|
with:
|
|
node-version: "20"
|
|
- run: pip install -e ".[test]"
|
|
- run: pytest tests/ -m "not live" --cov=turnstone --cov-report=term-missing --cov-report=xml -q
|
|
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
if: always()
|
|
with:
|
|
name: coverage-${{ matrix.python-version }}
|
|
path: coverage.xml
|
|
|
|
test-postgres:
|
|
runs-on: ubuntu-latest
|
|
services:
|
|
postgres:
|
|
image: postgres:18
|
|
env:
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_DB: turnstone_test
|
|
ports:
|
|
- 5432:5432
|
|
options: >-
|
|
--health-cmd="pg_isready -U postgres"
|
|
--health-interval=10s
|
|
--health-timeout=5s
|
|
--health-retries=5
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
|
with:
|
|
python-version: "3.14"
|
|
- uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
|
|
with:
|
|
node-version: "20"
|
|
- run: pip install -e ".[test,postgres]"
|
|
- run: pytest tests/ -m "not live" --storage-backend=postgresql -q
|
|
env:
|
|
TURNSTONE_TEST_PG_URL: postgresql+psycopg://postgres:postgres@localhost:5432/turnstone_test
|
|
|
|
wheel-completeness:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
|
with:
|
|
python-version: "3.14"
|
|
- run: pip install build
|
|
- run: python -m build --wheel
|
|
- name: Check all data files are in wheel
|
|
run: |
|
|
SOURCE=$(find turnstone -type f \
|
|
! -name '*.py' ! -name '*.pyc' ! -path '*__pycache__*' \
|
|
| sort)
|
|
WHEEL=$(python -m zipfile -l dist/*.whl \
|
|
| awk '{print $1}' \
|
|
| grep -v '\.py$' | grep -v '\.dist-info' | grep -v '\.pyc' | grep -v '^File$' \
|
|
| sort)
|
|
|
|
# Files intentionally excluded from the wheel (one per line)
|
|
ALLOW="
|
|
turnstone/core/storage/migrations/script.py.mako
|
|
"
|
|
|
|
MISSING=$(comm -23 <(echo "$SOURCE") <(echo "$WHEEL") \
|
|
| grep -vFxf <(echo "$ALLOW" | sed '/^[[:space:]]*$/d; s/^[[:space:]]*//' ) || true)
|
|
|
|
if [ -n "$MISSING" ]; then
|
|
echo "::error::Data files in source tree but missing from wheel:"
|
|
echo "$MISSING"
|
|
echo ""
|
|
echo "Add them to [tool.hatch.build.targets.wheel] in pyproject.toml"
|
|
echo "or to the ALLOW list in this job if intentionally excluded."
|
|
exit 1
|
|
fi
|
|
echo "All source data files present in wheel"
|
|
- name: Smoke-test entry points from installed wheel
|
|
run: |
|
|
python -m venv /tmp/smoke
|
|
/tmp/smoke/bin/pip install dist/*.whl
|
|
/tmp/smoke/bin/turnstone --help
|
|
/tmp/smoke/bin/turnstone-server --help
|
|
/tmp/smoke/bin/turnstone-console --help
|
|
/tmp/smoke/bin/turnstone-admin --help
|
|
/tmp/smoke/bin/turnstone-channel --help
|
|
/tmp/smoke/bin/turnstone-bootstrap --help
|
|
|
|
lock-check:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
with:
|
|
uv-version: "0.9.18"
|
|
- run: uv lock --check
|
|
|
|
security:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
|
|
with:
|
|
uv-version: "0.9.18"
|
|
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
|
|
with:
|
|
python-version: "3.14"
|
|
- run: uv sync --frozen --all-extras
|
|
- run: uv pip install pip-audit
|
|
- name: Security audit (dependencies)
|
|
run: uv export --no-emit-project --frozen | uv run pip-audit --strict --desc -r /dev/stdin
|
|
|
|
security-ts:
|
|
runs-on: ubuntu-latest
|
|
defaults:
|
|
run:
|
|
working-directory: sdk/typescript
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
|
|
with:
|
|
node-version: "24"
|
|
- run: npm ci
|
|
- run: npm audit --audit-level=moderate
|