diff --git a/docs/tools.md b/docs/tools.md index d2149470..28f6fc13 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -1,6 +1,6 @@ # Tools Reference -turnstone exposes 16 built-in tools plus any number of external MCP tools to the +turnstone exposes 17 built-in tools plus any number of external MCP tools to the LLM via the OpenAI function-calling interface. Built-in tools are defined as JSON files under `turnstone/tools/` and loaded at startup by `turnstone/core/tools.py`. MCP tools are discovered from configured MCP servers at startup by @@ -44,10 +44,10 @@ schema plus turnstone-specific metadata keys: | Name | Description | |---------------------|-------------| -| `TOOLS` | All 28 loaded built-in tool definitions (interactive + coordinator union). Sessions send a kind-specific subset (`INTERACTIVE_TOOLS` or `COORDINATOR_TOOLS`). | +| `TOOLS` | All 29 loaded built-in tool definitions (interactive + coordinator union). Sessions send a kind-specific subset (`INTERACTIVE_TOOLS` or `COORDINATOR_TOOLS`). | | `TASK_AGENT_TOOLS` | Tools with `task_agent: true` -- available to task sub-agents. Includes write operations. | | `TASK_AUTO_TOOLS` | Set of all tool names with `auto_approve: true` -- used by task-agent sub-sessions to skip confirmation for matching available tools. | -| `BUILTIN_TOOL_NAMES`| Frozenset of all 28 built-in tool names (interactive + coordinator union). Used by tool search to distinguish always-on tools from deferrable MCP tools. | +| `BUILTIN_TOOL_NAMES`| Frozenset of all 29 built-in tool names (interactive + coordinator union). Used by tool search to distinguish always-on tools from deferrable MCP tools. | | `PRIMARY_KEY_MAP` | Dict mapping tool name to its `primary_key` parameter name. | --- @@ -65,7 +65,7 @@ Tool execution follows a three-phase pipeline inside `ChatSession._execute_tools - Parses the JSON arguments (with fallback for malformed JSON). - If JSON parsing fails entirely, uses `PRIMARY_KEY_MAP` to map a bare string to the correct parameter. -- Dispatches to the matching `_prepare_{func_name}()` handler. There are 16 +- Dispatches to the matching `_prepare_{func_name}()` handler. There are 17 built-in tools plus `tool_search` (synthetic, client-side BM25 fallback) and the generic `_prepare_mcp_tool()` handler for MCP tools. - Validates arguments and builds a preview dict containing: @@ -364,7 +364,10 @@ Show the user rich content in a preview pane beside the conversation. content, stores it content-addressed against the workstream, and opens the frontend preview pane beside the conversation: web pages render in a fully sandboxed iframe (no scripts, opaque origin), PDFs in the browser viewer, - images inline, CSV/TSV/JSON as a sortable table, text/markdown rendered. The + images inline, CSV/TSV/JSON as a sortable table, text/markdown rendered. A + previewed web page loads none of its remote images or styles by default, so + opening it never reveals the viewer to the page's site; a toggle in the pane + header turns remote content back on for that preview. The model receives only a one-line confirmation — to reason about content, use `web_fetch` / `read_file` instead. Preview content is size-capped per kind (pages 4 MB, PDFs 32 MB, images 4 MB, tables 2 MB, text 512 KB) and GC'd @@ -688,7 +691,7 @@ MCP-compatible service. 3. **Schema conversion**: Each MCP tool's `inputSchema` is converted to OpenAI function-calling format. The tool name is prefixed: `mcp__{server}__{tool}`. -4. **Merging**: MCP tools are appended after the 16 built-in tools via +4. **Merging**: MCP tools are appended after the 17 built-in tools via `merge_mcp_tools()`. Built-in tools appear first, giving them natural LLM priority. When dynamic tool search is active, MCP tools are deferred rather than directly visible -- the model discovers them via search as needed (see diff --git a/tests/test_open_preview_tool.py b/tests/test_open_preview_tool.py index e77a7711..f6f9577e 100644 --- a/tests/test_open_preview_tool.py +++ b/tests/test_open_preview_tool.py @@ -251,6 +251,33 @@ class TestExecOpenPreview: assert descriptor["kind"] == "markdown" assert descriptor["title"] == "d.md" + def test_legacy_charset_table_stored_as_utf8(self, monkeypatch): + # A latin-1 CSV attachment previews as a table, and the executor + # transcodes it to UTF-8 at store time so "café" round-trips instead of + # erroring "not previewable". + s = _make_session(ws_id="ws-1") + latin1_csv = "name,city\nRené,Montréal\n".encode("iso-8859-1") + monkeypatch.setattr( + "turnstone.core.memory.get_attachment", + lambda aid: { + "content": latin1_csv, + "mime_type": "text/csv; charset=iso-8859-1", + "filename": "people.csv", + }, + ) + monkeypatch.setattr( + "turnstone.core.memory.attachment_referenced_in_ws", + lambda aid, ws: True, + ) + item = s._prepare_open_preview("c1", {"target": "attachment:deadbeef"}) + _, msg = s._exec_open_preview(item) + assert not msg.startswith("Error:") + descriptor, att = s._tool_previews["c1"] + assert descriptor["kind"] == "table" + assert descriptor["content_type"].startswith("text/csv") + # Stored bytes are valid UTF-8 with the accented characters preserved. + assert att.content.decode("utf-8") == "name,city\nRené,Montréal\n" + def test_title_override_wins(self, tmp_path): s = _make_session() p = tmp_path / "x.csv" diff --git a/tests/test_preview.py b/tests/test_preview.py index bd3eae02..2301ecea 100644 --- a/tests/test_preview.py +++ b/tests/test_preview.py @@ -20,6 +20,7 @@ from turnstone.core.preview import ( page_title, preview_response_headers, resolve_preview_kind, + transcode_text, ) PNG_1x1 = ( @@ -127,14 +128,35 @@ class TestHtmlHelpers: assert page_title("") is None +_LOCKED_HTML_CSP = ( + "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:; font-src data:" +) + + class TestServingPolicy: - def test_html_gets_bare_sandbox_csp(self): + def test_html_default_locks_out_remote_assets(self): + # Default (no opt-in): sandboxed AND off the network — inline styling + + # data-URI images render, but the page can fetch nothing, so previewing + # never discloses the viewer to the origin site. h = preview_response_headers("text/html", "page.html") - assert h["Content-Security-Policy"] == "sandbox" + assert h["Content-Security-Policy"] == _LOCKED_HTML_CSP assert h["X-Content-Type-Options"] == "nosniff" assert h["Cache-Control"] == "private, no-store" assert h["Content-Disposition"].startswith("inline;") + def test_html_assets_opt_in_gets_bare_sandbox_csp(self): + # allow_remote_assets=True drops back to the bare sandbox so the page's + # own images / CSS load. + h = preview_response_headers("text/html", "page.html", allow_remote_assets=True) + assert h["Content-Security-Policy"] == "sandbox" + assert h["X-Content-Type-Options"] == "nosniff" + + def test_assets_flag_does_not_touch_non_html_kinds(self): + for mime in ("application/pdf", "image/png", "text/csv", "text/plain"): + assert preview_response_headers( + mime, "f", allow_remote_assets=True + ) == preview_response_headers(mime, "f") + def test_pdf_gets_no_csp(self): h = preview_response_headers("application/pdf", "doc.pdf") assert "Content-Security-Policy" not in h @@ -230,5 +252,54 @@ class TestReviewHardening: # Extension lane and explicit override agree. assert resolve_preview_kind("", "page.html", latin1_html)[0] == "web" assert resolve_preview_kind("", "page.bin", latin1_html, "web")[0] == "web" - # Non-web text kinds stay strict. - assert resolve_preview_kind("text/csv", "d.csv", latin1_html) is None + # Non-web text kinds now transcode too — a declared text/csv MIME on + # legacy-charset bytes is previewable (was strict-UTF-8-only before). + assert resolve_preview_kind("text/csv", "d.csv", latin1_html) == ( + "table", + "text/csv; charset=utf-8", + ) + # …but binary declared as text (a NUL byte) is still rejected. + assert resolve_preview_kind("text/csv", "d.csv", b"\x00\x01\x02" * 8) is None + + +class TestLegacyCharsetText: + """Text-family kinds transcode legacy charsets at store time; only the + undeclared fallback lane stays strict UTF-8 (2026-07-07 follow-up).""" + + def test_declared_latin1_csv_is_a_table(self): + latin1_csv = "name,city\nRené,Montréal\n".encode("iso-8859-1") + # MIME hint carrying the charset. + assert resolve_preview_kind("text/csv; charset=iso-8859-1", "d", latin1_csv) == ( + "table", + "text/csv; charset=utf-8", + ) + # Extension lane and explicit override agree — all "declared text". + assert resolve_preview_kind("", "data.csv", latin1_csv)[0] == "table" + assert resolve_preview_kind("", "data.bin", latin1_csv, "table")[0] == "table" + + def test_declared_text_nul_byte_still_binary(self): + # The ladder never fails, so the NUL check is the only binary gate left + # for declared text — it must hold in every declared lane. + nul = b"a,b\n1,\x00\n" + assert resolve_preview_kind("text/csv", "d.csv", nul) is None + assert resolve_preview_kind("", "d.csv", nul) is None + assert resolve_preview_kind("", "d", nul, "table") is None + + def test_undeclared_non_utf8_still_rejected(self): + # No MIME hint, no text-family extension, no override: the bare + # fallback lane stays strict UTF-8 — cp1252+replace would otherwise + # classify arbitrary binary as text. + assert resolve_preview_kind("", "mystery", b"caf\xe9 nonsense \xff\xfe") is None + + def test_transcode_ladder_rungs(self): + # (a) charset= parameter honored. + assert transcode_text("café".encode("iso-8859-1"), "text/csv; charset=iso-8859-1") == "café" + # (b) UTF-8 when the charset is absent / unknown. + assert transcode_text("héllo".encode(), "text/plain") == "héllo" + assert transcode_text("héllo".encode(), "text/plain; charset=made-up") == "héllo" + # (c) cp1252 fallback rung: smart quotes are invalid UTF-8 (the shape a + # legacy .txt with no charset takes — empty mime hint), decoded via the + # last rung rather than erroring. + smart = b"he said \x93hi\x94" + out = transcode_text(smart, "") + assert "“" in out and "”" in out diff --git a/tests/test_preview_js.py b/tests/test_preview_js.py index 8e911570..a36598bc 100644 --- a/tests/test_preview_js.py +++ b/tests/test_preview_js.py @@ -46,17 +46,45 @@ class TestPreviewPaneModule: pdf_fn = body.split("const renderPdf")[1].split("const renderImage")[0] assert "sandbox" not in pdf_fn or "No sandbox attribute" in pdf_fn - def test_content_loads_through_authfetch_preflight(self) -> None: - """src-loaded kinds preflight with authFetch HEAD (surfaces the - persist race + auth failures as a typed error card, and rides the - 401-refresh retry that a bare iframe/img src can't).""" + def test_content_loads_through_authfetch_probe(self) -> None: + """src-loaded kinds preflight with a probe request (authFetch of + ?probe=1), NOT a HEAD. The console reverse proxy forwards a HEAD as a + full GET, so a real HEAD would drag the whole blob across the hop just + to discard it; the probe still surfaces the persist race + auth + failures as a typed error card and rides the 401-refresh retry a bare + iframe/img src can't.""" body = _read(_PREVIEW_JS) - assert 'authFetch(url, { method: "HEAD" })' in body + assert "authFetch(probeUrl)" in body + assert "probe=1" in body + # The old full-GET HEAD preflight is gone. + assert 'method: "HEAD"' not in body def test_markdown_uses_the_sanctioned_html_lane(self) -> None: body = _read(_PREVIEW_JS) assert "setSafeHtml(doc, renderMarkdown(text))" in body + def test_markdown_runs_vendor_post_pass(self) -> None: + """The pane runs renderer.js's post-render pass (hljs token coloring + + mermaid) like the conversation pane — dropping it silently regresses + code highlighting and diagram rendering in previews.""" + body = _read(_PREVIEW_JS) + assert "postRenderMarkdown(" in body + + def test_remote_assets_toggle_is_default_off(self) -> None: + """The remote-assets opt-in defaults OFF: a previewed page must not + contact its origin site until the user asks. Pins the label / tooltip + copy and the sticky-boolean initializer.""" + body = _read(_PREVIEW_JS) + assert "Load remote images & styles" in body + assert "Off keeps this preview from contacting the site" in body + assert "pane._assetsOn = false" in body + + def test_assets_flag_only_rides_behind_toggle(self) -> None: + """assets=1 reaches the URL only when the per-pane toggle is on.""" + body = _read(_PREVIEW_JS) + assert "assets=1" in body + assert "pane._assetsOn" in body + def test_history_is_bounded(self) -> None: assert "HISTORY_CAP" in _read(_PREVIEW_JS) diff --git a/tests/test_server_attachments_endpoints.py b/tests/test_server_attachments_endpoints.py index 17e93676..688ae10f 100644 --- a/tests/test_server_attachments_endpoints.py +++ b/tests/test_server_attachments_endpoints.py @@ -1055,7 +1055,7 @@ def _seed_committed(ws_id: str, kind: str, mime: str, body: bytes, filename: str class TestGetPreview: - def test_html_served_renderable_with_bare_sandbox_csp(self, app_client): + def test_html_default_serves_locked_down_csp(self, app_client): client, _ = app_client body = b'x' aid = _seed_committed("ws-A", "preview", "text/html; charset=utf-8", body, "preview-web") @@ -1066,13 +1066,29 @@ class TestGetPreview: assert resp.status_code == 200 assert resp.headers["content-type"].startswith("text/html") assert resp.content == body - # Renderable but locked down: bare sandbox (no default-src 'none' — - # the page's own subresources must load), nosniff, inline, no-store. - assert resp.headers.get("content-security-policy") == "sandbox" + # Default (no ?assets): renderable but off the network — sandboxed, + # inline styling + data-URI images only, so previewing discloses + # nothing to the origin site. + assert resp.headers.get("content-security-policy") == ( + "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:; font-src data:" + ) assert resp.headers.get("x-content-type-options") == "nosniff" assert resp.headers.get("content-disposition", "").startswith("inline;") assert resp.headers.get("cache-control") == "private, no-store" + def test_html_assets_flag_serves_bare_sandbox(self, app_client): + # ?assets=1 is the per-pane opt-in: drop back to the bare sandbox so + # the page's own images / CSS load. + client, _ = app_client + body = b"x" + aid = _seed_committed("ws-A", "preview", "text/html; charset=utf-8", body, "preview-web") + resp = client.get( + f"/v1/api/workstreams/ws-A/attachments/{aid}/preview?assets=1", + headers=_auth("userA"), + ) + assert resp.status_code == 200 + assert resp.headers.get("content-security-policy") == "sandbox" + def test_pdf_served_without_csp(self, app_client): client, _ = app_client aid = _seed_committed("ws-A", "preview", "application/pdf", b"%PDF-1.4 x", "d.pdf") @@ -1125,13 +1141,47 @@ class TestGetPreview: ) assert resp.status_code == 404 - def test_head_preflight_supported(self, app_client): - # The pane preflights src-loaded kinds with HEAD (the persist race); - # Starlette derives HEAD from the GET route. + def test_probe_returns_204_with_hardening_headers(self, app_client): + # The pane preflights src-loaded kinds with ?probe=1 instead of HEAD: + # the console reverse proxy forwards a HEAD as a full GET, so a real + # HEAD would drag the whole blob across the hop just to discard it. The + # probe runs the ownership + renderable-type gates and returns the real + # response's hardening headers with an empty body. client, _ = app_client - aid = _seed_committed("ws-A", "preview", "text/html", b"

x

", "p") - resp = client.head( - f"/v1/api/workstreams/ws-A/attachments/{aid}/preview", + body = b"x" + aid = _seed_committed("ws-A", "preview", "text/html; charset=utf-8", body, "preview-web") + resp = client.get( + f"/v1/api/workstreams/ws-A/attachments/{aid}/preview?probe=1", headers=_auth("userA"), ) - assert resp.status_code == 200 + assert resp.status_code == 204 + assert resp.content == b"" + # Same hardening headers the real GET would carry (the probe answers + # "will the load paint?"): the html CSP is present. + assert resp.headers.get("content-security-policy") == ( + "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:; font-src data:" + ) + assert resp.headers.get("x-content-type-options") == "nosniff" + + def test_probe_composes_with_assets_flag(self, app_client): + # ?probe=1&assets=1 → 204 whose headers reflect the assets opt-in. + client, _ = app_client + body = b"x" + aid = _seed_committed("ws-A", "preview", "text/html; charset=utf-8", body, "preview-web") + resp = client.get( + f"/v1/api/workstreams/ws-A/attachments/{aid}/preview?probe=1&assets=1", + headers=_auth("userA"), + ) + assert resp.status_code == 204 + assert resp.headers.get("content-security-policy") == "sandbox" + + def test_probe_non_renderable_mime_still_415(self, app_client): + # A probe must answer "will the real load succeed?" — a non-renderable + # blob 415s exactly as the real GET would, before any 204. + client, _ = app_client + aid = _seed_committed("ws-A", "audio", "audio/wav", WAV_12, "a.wav") + resp = client.get( + f"/v1/api/workstreams/ws-A/attachments/{aid}/preview?probe=1", + headers=_auth("userA"), + ) + assert resp.status_code == 415 diff --git a/turnstone/core/preview.py b/turnstone/core/preview.py index 1bbc0596..cb3811a4 100644 --- a/turnstone/core/preview.py +++ b/turnstone/core/preview.py @@ -109,6 +109,52 @@ def _is_utf8_text(data: bytes) -> bool: return True +def _is_decodable_text(data: bytes) -> bool: + """True when *data* carries no NUL byte — the gate for DECLARED text. + + A text-family MIME hint / extension / ``kind`` override says "this is + text"; the store-time transcode ladder (:func:`transcode_text`) then + decodes it whatever the charset, so the only hard reject left is the NUL + byte that marks genuinely-binary content. The *undeclared* fallback lane + keeps the stricter :func:`_is_utf8_text`: cp1252-with-replacement never + fails, so unknown bytes must prove UTF-8 rather than be waved through as + text. + """ + return b"\x00" not in data + + +def _charset_param(mime: str) -> str | None: + """The ``charset=`` value from a MIME string, lowercased, or ``None``.""" + for part in mime.split(";")[1:]: + key, sep, value = part.partition("=") + if sep and key.strip().lower() == "charset": + return value.strip().strip('"').lower() or None + return None + + +def transcode_text(body: bytes, mime_hint: str) -> str: + """Decode text-family *body* to ``str`` via a charset ladder. + + Rungs: (a) the ``charset=`` parameter from *mime_hint* when it names a + codec Python knows, (b) UTF-8, (c) cp1252 with ``errors="replace"``. The + last rung never fails, so the return is always a usable string — this is + the store-time transcode that lets a legacy-charset page / CSV / log render + as UTF-8. Binary rejection stays upstream in :func:`resolve_preview_kind` + (the NUL check); by the time bytes reach here they are already classified + text. + """ + charset = _charset_param(mime_hint) + if charset: + try: + return body.decode(charset) + except (LookupError, UnicodeDecodeError): + pass + try: + return body.decode("utf-8") + except UnicodeDecodeError: + return body.decode("cp1252", errors="replace") + + def _kind_from_mime(mime: str) -> tuple[str, str] | None: """Map a transport MIME hint to ``(kind, stored_mime)``, or ``None``.""" bare = mime.split(";", 1)[0].strip().lower() @@ -154,10 +200,11 @@ def resolve_preview_kind( return ("image", sniffed_image) if sniffed_image else None if kind_override == "pdf": return ("pdf", "application/pdf") if sniff_pdf_mime(body) else None - # Text-family overrides (table / text / markdown) require text; web - # is transcoded at store time (see the mime lane below) so a legacy - # charset page can still be forced to render. - if kind_override != "web" and not _is_utf8_text(body): + # Text-family overrides (table / text / markdown) reject only genuine + # binary here — the NUL check. The executor transcodes the bytes to + # UTF-8 at store time, so a legacy-charset body forced to a text kind + # still renders (web always took this path; the others now join it). + if kind_override != "web" and not _is_decodable_text(body): return None if kind_override == "table": # Preserve a JSON payload's real type so the client parser branches. @@ -177,17 +224,19 @@ def resolve_preview_kind( return "pdf", "application/pdf" from_mime = _kind_from_mime(mime_hint) if from_mime: - # Text-declared bytes that aren't text are misdeclared — reject rather - # than serve binary under a text MIME. ``web`` is exempt: legacy - # charsets (windows-1252 / Shift-JIS pages) are not UTF-8 on the raw - # bytes, and the executor transcodes web content to UTF-8 at store - # time (charset-aware for fetches, replacement-decoded otherwise). - if from_mime[0] in ("table", "text", "markdown") and not _is_utf8_text(body): + # A text-family MIME hint declares text: reject only genuine binary + # (the NUL check). Legacy charsets (windows-1252 / Shift-JIS pages, + # iso-8859-1 CSVs / logs) are not UTF-8 on the raw bytes, and the + # executor transcodes every text-family kind to UTF-8 at store time + # (charset-aware for fetches, ladder-decoded otherwise). + if from_mime[0] in ("table", "text", "markdown") and not _is_decodable_text(body): return None return from_mime ext_match = _EXT_KINDS.get(_name_ext(name_hint)) if ext_match: - if ext_match[0] != "web" and not _is_utf8_text(body): + # A text-family extension declares text too — same NUL-only gate; the + # store-time ladder handles whatever charset the bytes are in. + if ext_match[0] != "web" and not _is_decodable_text(body): return None return ext_match if _is_utf8_text(body): @@ -272,17 +321,23 @@ def build_preview_descriptor( } -def preview_response_headers(bare_mime: str, filename: str) -> dict[str, str]: +def preview_response_headers( + bare_mime: str, filename: str, *, allow_remote_assets: bool = False +) -> dict[str, str]: """Response headers for the preview serving route, per rendered MIME. - ``text/html`` gets ``Content-Security-Policy: sandbox`` — the document - renders (its subresources load) but scripts never run and its origin is - opaque, so it can't touch the app origin's cookies or DOM; the embedding - iframe carries the ``sandbox`` attribute too. ``application/pdf`` gets no - CSP: Chromium's PDF viewer refuses to paint inside a sandboxed context, - and the response is inert media rendered by browser chrome, not an active - document. Everything else keeps the attachment endpoints' full - ``default-src 'none'; sandbox`` posture. + ``text/html`` is served sandboxed either way — scripts never run and its + origin is opaque, so it can't touch the app origin's cookies or DOM, and + the embedding iframe carries the ``sandbox`` attribute too. The default + (``allow_remote_assets=False``) additionally locks the document out of the + network: it renders with its inline styling and data-URI images but cannot + fetch anything, so previewing a page never discloses the viewer's IP or + traffic to the origin site. ``allow_remote_assets=True`` (a per-pane + opt-in) drops back to the bare ``sandbox`` so the page's own images / CSS + load. ``application/pdf`` gets no CSP: Chromium's PDF viewer refuses to + paint inside a sandboxed context, and the response is inert media rendered + by browser chrome, not an active document. Everything else keeps the + attachment endpoints' full ``default-src 'none'; sandbox`` posture. """ # Header values must be latin-1 encodable (Starlette raises on anything # else), and page-title-derived filenames routinely carry em dashes / CJK @@ -295,7 +350,13 @@ def preview_response_headers(bare_mime: str, filename: str) -> dict[str, str]: "Cache-Control": "private, no-store", } if bare_mime == "text/html": - headers["Content-Security-Policy"] = "sandbox" + if allow_remote_assets: + headers["Content-Security-Policy"] = "sandbox" + else: + headers["Content-Security-Policy"] = ( + "sandbox; default-src 'none'; style-src 'unsafe-inline'; " + "img-src data:; font-src data:" + ) elif bare_mime != "application/pdf": headers["Content-Security-Policy"] = "default-src 'none'; sandbox" return headers diff --git a/turnstone/core/session.py b/turnstone/core/session.py index f919fac7..76d302f6 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -132,6 +132,7 @@ from turnstone.core.preview import ( inject_base_href, page_title, resolve_preview_kind, + transcode_text, ) from turnstone.core.providers import create_provider from turnstone.core.ratelimit import TokenBucket @@ -16169,21 +16170,22 @@ class ChatSession: ) title = title_override - if kind == "web": - # Store what the fetch saw, made renderable: decode on the - # transport charset, give relative assets a base to resolve - # against, and re-encode UTF-8 (matching the stored mime). - if target_kind == "url": - text = resp.text - text = inject_base_href(text, final_url) - else: - text = body.decode("utf-8", errors="replace") - if not title: - title = page_title(text) + if kind in ("web", "table", "text", "markdown"): + # Store text-family content as UTF-8 so legacy charsets render + # instead of erroring "not previewable": a fetch honors the + # response charset (httpx ``resp.text``); local / attachment bytes + # go through the transcode ladder. Web additionally gains a + # ```` (url targets) and a title fallback. + text = resp.text if target_kind == "url" else transcode_text(body, mime_hint) + if kind == "web": + if target_kind == "url": + text = inject_base_href(text, final_url) + if not title: + title = page_title(text) body = text.encode("utf-8") if len(body) > cap: return _fail( - f"Error: web content too large to preview ({len(body):,} bytes; cap {cap:,})" + f"Error: {kind} content too large to preview ({len(body):,} bytes; cap {cap:,})" ) if not title: tail = name_hint.rsplit("/", 1)[-1].split("?", 1)[0] diff --git a/turnstone/core/session_routes.py b/turnstone/core/session_routes.py index 6b118d1f..da2f0498 100644 --- a/turnstone/core/session_routes.py +++ b/turnstone/core/session_routes.py @@ -4390,11 +4390,23 @@ def make_attachment_handlers(cfg: SessionEndpointConfig) -> AttachmentHandlers: bare_mime = stored_mime.split(";", 1)[0].strip().lower() if bare_mime not in PREVIEW_SERVE_MIMES: return JSONResponse({"error": "attachment is not previewable"}, status_code=415) - return _Response( - body, - media_type=stored_mime, - headers=preview_response_headers(bare_mime, filename), + # ``?assets=1`` opts a previewed page back into loading its remote + # images / styles; default-off keeps the sandboxed document off the + # network (see ``preview_response_headers``). + allow_remote_assets = bool(request.query_params.get("assets")) + headers = preview_response_headers( + bare_mime, filename, allow_remote_assets=allow_remote_assets ) + # ``?probe=1`` preflight: the pane asks "will the real load paint?" + # before pointing an iframe / img at this URL. Answer with the exact + # hardening headers the real response would carry but no body — the + # console reverse proxy forwards a HEAD as a full GET, so a HEAD + # preflight would drag the whole blob across the node→console hop just + # to discard it. The ownership gate and the renderable-type check + # above have already run, so a 204 here means the GET will succeed. + if request.query_params.get("probe"): + return _Response(status_code=204, headers=headers) + return _Response(body, media_type=stored_mime, headers=headers) async def get_thumbnail(request: Request) -> Response: import asyncio diff --git a/turnstone/shared_static/preview.css b/turnstone/shared_static/preview.css index 11e23189..8b557ada 100644 --- a/turnstone/shared_static/preview.css +++ b/turnstone/shared_static/preview.css @@ -81,6 +81,26 @@ color: var(--ink-1); } +/* remote-assets opt-in (web previews) — compact single-row header control */ +.preview-assets { + flex: none; + display: inline-flex; + align-items: center; + gap: 4px; + font-size: 11px; + color: var(--ink-2); + white-space: nowrap; + cursor: pointer; +} +.preview-assets:hover { + color: var(--ink-1); +} +.preview-assets-box { + flex: none; + margin: 0; + cursor: pointer; +} + /* ----- content host ----- */ .preview-content { flex: 1 1 auto; @@ -125,6 +145,37 @@ font-size: 13px; color: var(--ink-1); } +/* Code-block chrome + katex display: the per-ui style.css rules are scoped to + .msg.assistant and don't reach the pane, so restate them on DS tokens. The + hljs TOKEN colors are global and already apply. */ +.preview-markdown code { + padding: 1px 4px; + font-family: var(--font-mono); + font-size: 0.92em; + background: color-mix(in srgb, var(--ink-4) 18%, transparent); + border-radius: 3px; +} +.preview-markdown pre { + padding: 8px; + overflow-x: auto; + font-family: var(--font-mono); + font-size: 11px; + background: var(--panel); + border: 1px solid var(--hair); + border-radius: 3px; +} +.preview-markdown pre code { + padding: 0; + font-size: inherit; + background: none; +} +.preview-markdown .katex-display { + margin: 8px 0; + overflow-x: auto; +} +.preview-markdown img { + max-width: 100%; +} /* ----- table kind ----- */ .preview-tablewrap { diff --git a/turnstone/shared_static/preview.js b/turnstone/shared_static/preview.js index 97f07dc6..d590f09a 100644 --- a/turnstone/shared_static/preview.js +++ b/turnstone/shared_static/preview.js @@ -23,7 +23,7 @@ import { ShellPane } from "./pane.js"; import { authFetch } from "./auth.js"; import { redactCredentials } from "./redact_credentials.js"; -import { renderMarkdown } from "./renderer.js"; +import { renderMarkdown, postRenderMarkdown } from "./renderer.js"; import { setSafeHtml } from "./utils.js"; // How many viewed descriptors the ←/→ history keeps. Session-scoped and @@ -63,6 +63,18 @@ function previewContentUrl(ctx, descriptor) { ); } +// Append preview query flags to a content URL (which never carries a query of +// its own). ``probe`` asks the route for a bodyless 204 "will the real load +// paint?" preflight — the console reverse proxy forwards a HEAD as a full GET, +// so a real HEAD would drag the whole blob across the hop just to discard it. +// ``assets`` opts a sandboxed page back into loading its remote images/styles. +function withPreviewFlags(url, opts) { + const q = []; + if (opts && opts.probe) q.push("probe=1"); + if (opts && opts.assets) q.push("assets=1"); + return q.length ? url + "?" + q.join("&") : url; +} + // --------------------------------------------------------------------------- // Delimited-text parsing (table kind). Minimal RFC-4180 state machine: // quoted fields, "" escapes, \r\n and \n rows. Returns rows of strings. @@ -198,6 +210,10 @@ export function createPreviewPane(extra, hostApi) { pane._stack = []; pane._idx = -1; pane._loadToken = 0; + // Remote-assets opt-in: per-pane, sticky across previews, NOT persisted in + // pane meta. Default OFF — a previewed page must not contact its origin + // site (an IP/traffic disclosure) until the user asks. + pane._assetsOn = false; const shortTitle = (d) => { const t = redactCredentials(d.title || d.source || "preview"); @@ -277,6 +293,13 @@ export function createPreviewPane(extra, hostApi) { // The one sanctioned HTML lane: renderer.js output through setSafeHtml. setSafeHtml(doc, renderMarkdown(text)); pane._contentEl.replaceChildren(doc); + // Vendor post-pass — hljs token coloring + mermaid diagrams, matching the + // conversation pane. Runs AFTER the attach: the renderer tolerates + // detached elements (the async mermaid apply gates on isConnected and + // attachment here is synchronous), but attach-first matches the + // conversation pane's ordering and leaves no room for doubt. + // renderer.js typeof-guards absent vendors, so no try/catch is needed. + postRenderMarkdown(doc); }; const renderTable = (text, d) => { @@ -405,14 +428,23 @@ export function createPreviewPane(extra, hostApi) { const isWeb = d.kind === "web" && /^https?:\/\//.test(d.source || ""); pane._extLink.hidden = !isWeb; if (isWeb) pane._extLink.href = d.source; + // The remote-assets toggle rides web previews only; its checked state + // mirrors the pane's sticky opt-in on every render. + pane._assetsLabel.hidden = d.kind !== "web"; + pane._assetsBox.checked = !!pane._assetsOn; api.setTitle && api.setTitle("Preview · " + shortTitle(d)); pane._contentEl.replaceChildren(make("div", "preview-loading", "Loading…")); + // Remote assets are a web-only concern; the flag only reaches web URLs. + const assetsOn = d.kind === "web" && !!pane._assetsOn; + const probeUrl = withPreviewFlags(url, { probe: true, assets: assetsOn }); + const srcUrl = withPreviewFlags(url, { assets: assetsOn }); + if (d.kind === "web" || d.kind === "pdf" || d.kind === "image") { - // src-loaded kinds: preflight with authFetch so the persist race and - // auth failures surface as a typed error card, not a broken frame. - authFetch(url, { method: "HEAD" }) + // src-loaded kinds: preflight with a probe request so the persist race + // and auth failures surface as a typed error card, not a broken frame. + authFetch(probeUrl) .then((r) => { if (token !== pane._loadToken) return; if (!r.ok) { @@ -423,9 +455,9 @@ export function createPreviewPane(extra, hostApi) { ); return; } - if (d.kind === "web") renderWeb(url); - else if (d.kind === "pdf") renderPdf(url, d); - else renderImage(url, d); + if (d.kind === "web") renderWeb(srcUrl); + else if (d.kind === "pdf") renderPdf(srcUrl, d); + else renderImage(srcUrl, d); }) .catch(() => failed("Could not load the preview.")); return; @@ -506,7 +538,26 @@ export function createPreviewPane(extra, hostApi) { ext.target = "_blank"; ext.rel = "noopener noreferrer"; ext.hidden = true; - bar.append(back, fwd, kind, title, ext); + // Remote-assets opt-in — web previews only. Plain operator language; the + // tooltip states the default posture without naming the mechanism. + const assets = make("label", "preview-assets"); + assets.title = "Off keeps this preview from contacting the site"; + const assetsBox = make("input", "preview-assets-box"); + assetsBox.type = "checkbox"; + assets.append( + assetsBox, + make("span", "preview-assets-text", "Load remote images & styles"), + ); + assets.hidden = true; + assetsBox.addEventListener("change", () => { + pane._assetsOn = assetsBox.checked; + // Reload the current web preview so its iframe re-fetches in the new + // mode. A toggle is a deliberate act — no silent-backoff run. + const cur = pane._stack[pane._idx]; + if (cur && cur.descriptor.kind === "web") + renderEntry(cur, MAX_AUTO_RETRIES); + }); + bar.append(back, fwd, kind, title, ext, assets); const content = make("div", "preview-content"); @@ -515,6 +566,8 @@ export function createPreviewPane(extra, hostApi) { pane._kindEl = kind; pane._titleEl = title; pane._extLink = ext; + pane._assetsLabel = assets; + pane._assetsBox = assetsBox; pane._contentEl = content; root.append(bar, content);