mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
6cbd3eb2c1
* feat: workstream attachments at creation time + SDK + UI parity Closes the two big deferred items from PR #356: attaching files as part of the initial workstream-creation request, and full SDK coverage of the attachment surface. Server: POST /v1/api/workstreams/new now accepts multipart/form-data (meta JSON + 0..N file parts). Files are validated and saved as pending under the new ws; when initial_message is also set the create handler reserves them onto that turn before the dispatch worker fires, mirroring the /v1/api/send pattern. Validation failure rolls back the workstream via delete_workstream so we don't leak orphan rows or emit a phantom ws_created/ws_closed pair on SSE. JSON path is unchanged. Console routing: route_create accepts multipart with ?ws_id=<hex> as a query parameter (the console hashes the id before the body lands). Added /v1/api/route/workstreams/{ws_id}/attachments POST/GET/DELETE + .../{attachment_id}/content GET proxies that forward raw bytes and preserve upstream headers (Content-Disposition, X-Content-Type-Options, CSP sandbox). Python + TypeScript SDKs: AttachmentUpload type, upload_attachment, list_attachments, get_attachment_content, delete_attachment, and send(attachment_ids=...). create_workstream(attachments=...) sends multipart and pre-generates a ws_id client-side so cluster routing works. SDKs reject attachments+target_node combinations since the multipart route doesn't honor target_node. Web UI: dashboard composer refactored to a single unified create flow. Replaced the inconsistent split (Enter created+sent raw, "New Chat" opened a modal) with one rich composer carrying a textarea, paperclip + chip strip, drag-drop, paste-image, and a collapsible Options panel for model/judge_model/skill. Submit button dynamically labels Create vs Send. New-workstream modal also gained the same paperclip + chip strip + first-message field for the tab-bar + entry point. Tests: 30 new tests across server multipart create, console route multipart + attachment proxies, Python + TS SDK attachment surfaces, plus regressions for the three review-flagged bugs (Content-Type boundary preservation, attachments+target_node rejection, no phantom ws_created on validation failure). * fix: address Copilot review feedback on PR #362 - web_helpers: docstring now matches behaviour — read_multipart_create_or_400 does enforce the optional max_per_file_bytes cap as defense-in-depth. - app.js: drop the duplicated _formatAttachSize definition (one already exists earlier for pane chips); add a shared _isAttachmentAllowed helper that mirrors the server's classifier (png/jpeg/gif/webp images, text/* MIMEs, allowlisted application/* MIMEs, known text extensions) and call it from both _newWsAddFiles and _addDashboardFiles so unsupported files fail fast client-side instead of after a server roundtrip. - app.js: dashboardSubmit catch now suppresses the redundant error toast on authFetch's "auth" Error and falls back to a generic message when err.message is undefined, instead of rendering "Connection error: undefined". - SendResponse (Pydantic + TS): document and expose attached_ids, dropped_attachment_ids, priority, and msg_id so attachment-aware SDK callers can detect partial reservations and dequeue queued messages. - test_server_attachments_on_create: drop the dual `import turnstone.server` + `from turnstone.server import` style — use monkeypatch.setattr by dotted path for module-level mutation and `from … import …` for the helpers, keeping a single import style.
354 lines
13 KiB
Python
354 lines
13 KiB
Python
"""Tests for console routing of attachment endpoints + multipart route_create.
|
|
|
|
Covers the cluster-routing surface added alongside the workstream
|
|
attachment-on-create feature: the multipart variant of route_create and
|
|
the four ws-id-keyed attachment proxies under /v1/api/route/.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
from unittest.mock import MagicMock
|
|
|
|
import httpx
|
|
from starlette.testclient import TestClient
|
|
|
|
from turnstone.console.collector import ClusterCollector
|
|
from turnstone.console.router import ConsoleRouter, NodeRef
|
|
|
|
_TEST_JWT_SECRET = "test-jwt-secret-minimum-32-chars!"
|
|
|
|
|
|
def _test_jwt() -> str:
|
|
from turnstone.core.auth import JWT_AUD_CONSOLE, create_jwt
|
|
|
|
return create_jwt(
|
|
user_id="test-routing",
|
|
scopes=frozenset({"read", "write", "approve", "service"}),
|
|
source="test",
|
|
secret=_TEST_JWT_SECRET,
|
|
audience=JWT_AUD_CONSOLE,
|
|
)
|
|
|
|
|
|
_AUTH: dict[str, str] = {"Authorization": f"Bearer {_test_jwt()}"}
|
|
|
|
|
|
def _make_app(router: Any) -> Any:
|
|
from turnstone.console.server import _load_static, create_app
|
|
|
|
_load_static()
|
|
collector = MagicMock(spec=ClusterCollector)
|
|
return create_app(
|
|
collector=collector,
|
|
jwt_secret=_TEST_JWT_SECRET,
|
|
router=router,
|
|
)
|
|
|
|
|
|
def _make_router() -> MagicMock:
|
|
router = MagicMock(spec=ConsoleRouter)
|
|
router.is_ready.return_value = True
|
|
router.route.return_value = NodeRef("node-a", "http://a:8080")
|
|
return router
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# route_create multipart
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestRouteCreateMultipart:
|
|
def test_multipart_requires_ws_id_query(self):
|
|
router = _make_router()
|
|
app = _make_app(router=router)
|
|
app.state.proxy_client = MagicMock(spec=httpx.AsyncClient)
|
|
client = TestClient(app, raise_server_exceptions=False)
|
|
try:
|
|
resp = client.post(
|
|
"/v1/api/route/workstreams/new",
|
|
files=[("file", ("a.txt", b"hello", "text/plain"))],
|
|
data={"meta": "{}"},
|
|
headers=_AUTH,
|
|
)
|
|
assert resp.status_code == 400
|
|
assert "ws_id" in resp.json()["error"]
|
|
finally:
|
|
client.close()
|
|
|
|
def test_multipart_forwards_raw_body_to_routed_node(self):
|
|
router = _make_router()
|
|
app = _make_app(router=router)
|
|
|
|
captured: dict[str, Any] = {}
|
|
|
|
async def _mock_post(*args: Any, **kwargs: Any) -> httpx.Response:
|
|
captured["url"] = args[0] if args else ""
|
|
captured["headers"] = kwargs.get("headers") or {}
|
|
captured["content"] = kwargs.get("content")
|
|
return httpx.Response(
|
|
200,
|
|
json={"ws_id": "00ff" + "0" * 28, "name": "demo"},
|
|
request=httpx.Request("POST", args[0] if args else "http://test"),
|
|
)
|
|
|
|
mock_proxy = MagicMock(spec=httpx.AsyncClient)
|
|
mock_proxy.post = MagicMock(side_effect=_mock_post)
|
|
app.state.proxy_client = mock_proxy
|
|
client = TestClient(app, raise_server_exceptions=False)
|
|
try:
|
|
ws_id = "00ff" + "0" * 28
|
|
resp = client.post(
|
|
f"/v1/api/route/workstreams/new?ws_id={ws_id}",
|
|
files=[("file", ("a.txt", b"hello", "text/plain"))],
|
|
data={"meta": '{"name":"demo"}'},
|
|
headers=_AUTH,
|
|
)
|
|
assert resp.status_code == 200, resp.text
|
|
data = resp.json()
|
|
assert data["node_id"] == "node-a"
|
|
# Forwarded multipart Content-Type
|
|
assert captured["headers"].get("Content-Type", "").startswith("multipart/form-data")
|
|
# Body bytes were forwarded raw
|
|
assert isinstance(captured["content"], (bytes, bytearray))
|
|
assert b"hello" in bytes(captured["content"])
|
|
router.route.assert_called_with(ws_id)
|
|
finally:
|
|
client.close()
|
|
|
|
def test_multipart_preserves_mixed_case_boundary(self):
|
|
"""The boundary= param is case-sensitive — must match body bytes verbatim.
|
|
|
|
Regression for an earlier bug where route_create lowercased the
|
|
whole Content-Type header before forwarding, mangling boundaries
|
|
like ``WebKitFormBoundary7MA4YWxkTrZu0gW``.
|
|
"""
|
|
router = _make_router()
|
|
app = _make_app(router=router)
|
|
|
|
captured: dict[str, Any] = {}
|
|
|
|
async def _mock_post(*args: Any, **kwargs: Any) -> httpx.Response:
|
|
captured["headers"] = kwargs.get("headers") or {}
|
|
captured["content"] = kwargs.get("content")
|
|
return httpx.Response(
|
|
200,
|
|
json={"ws_id": "00ff" + "0" * 28, "name": "ok"},
|
|
request=httpx.Request("POST", args[0] if args else "http://test"),
|
|
)
|
|
|
|
mock_proxy = MagicMock(spec=httpx.AsyncClient)
|
|
mock_proxy.post = MagicMock(side_effect=_mock_post)
|
|
app.state.proxy_client = mock_proxy
|
|
client = TestClient(app, raise_server_exceptions=False)
|
|
try:
|
|
ws_id = "00ff" + "0" * 28
|
|
boundary = "WebKitFormBoundary7MA4YWxkTrZu0gW" # mixed-case
|
|
body = (
|
|
f"--{boundary}\r\n"
|
|
f'Content-Disposition: form-data; name="meta"\r\n\r\n'
|
|
f'{{"name":"demo"}}\r\n'
|
|
f"--{boundary}\r\n"
|
|
f'Content-Disposition: form-data; name="file"; filename="a.txt"\r\n'
|
|
f"Content-Type: text/plain\r\n\r\n"
|
|
f"hello\r\n"
|
|
f"--{boundary}--\r\n"
|
|
).encode()
|
|
resp = client.post(
|
|
f"/v1/api/route/workstreams/new?ws_id={ws_id}",
|
|
content=body,
|
|
headers={
|
|
**_AUTH,
|
|
"Content-Type": f"multipart/form-data; boundary={boundary}",
|
|
},
|
|
)
|
|
assert resp.status_code == 200, resp.text
|
|
forwarded = captured["headers"].get("Content-Type", "")
|
|
assert boundary in forwarded, (
|
|
f"boundary mangled in upstream Content-Type: {forwarded!r}"
|
|
)
|
|
# Body bytes still contain the mixed-case boundary
|
|
assert boundary.encode() in bytes(captured["content"])
|
|
finally:
|
|
client.close()
|
|
|
|
def test_json_path_unchanged(self):
|
|
"""Existing JSON callers should continue to work as before."""
|
|
router = _make_router()
|
|
app = _make_app(router=router)
|
|
|
|
async def _mock_post(*args: Any, **kwargs: Any) -> httpx.Response:
|
|
return httpx.Response(
|
|
200,
|
|
json={"ws_id": "abc123", "name": "json"},
|
|
request=httpx.Request("POST", args[0] if args else "http://test"),
|
|
)
|
|
|
|
mock_proxy = MagicMock(spec=httpx.AsyncClient)
|
|
mock_proxy.post = MagicMock(side_effect=_mock_post)
|
|
app.state.proxy_client = mock_proxy
|
|
client = TestClient(app, raise_server_exceptions=False)
|
|
try:
|
|
resp = client.post(
|
|
"/v1/api/route/workstreams/new",
|
|
json={"name": "json"},
|
|
headers=_AUTH,
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.json()["ws_id"] == "abc123"
|
|
# JSON path uses json= kwarg, not content=
|
|
call_kwargs = mock_proxy.post.call_args.kwargs
|
|
assert "json" in call_kwargs
|
|
assert "content" not in call_kwargs
|
|
finally:
|
|
client.close()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# route_attachment_proxy
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestRouteAttachmentProxy:
|
|
def _wire(self, mock_request_fn) -> tuple[Any, MagicMock]:
|
|
router = _make_router()
|
|
app = _make_app(router=router)
|
|
mock_proxy = MagicMock(spec=httpx.AsyncClient)
|
|
mock_proxy.request = MagicMock(side_effect=mock_request_fn)
|
|
mock_proxy.get = MagicMock(side_effect=mock_request_fn)
|
|
mock_proxy.post = MagicMock(side_effect=mock_request_fn)
|
|
app.state.proxy_client = mock_proxy
|
|
return app, mock_proxy
|
|
|
|
def test_upload_proxies_multipart(self):
|
|
captured: dict[str, Any] = {}
|
|
|
|
async def _mock(*args: Any, **kwargs: Any) -> httpx.Response:
|
|
captured["method"] = args[0] if args else kwargs.get("method")
|
|
captured["url"] = args[1] if len(args) > 1 else kwargs.get("url", "")
|
|
captured["headers"] = kwargs.get("headers") or {}
|
|
captured["content"] = kwargs.get("content")
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"attachment_id": "att-1",
|
|
"filename": "a.txt",
|
|
"mime_type": "text/plain",
|
|
"size_bytes": 5,
|
|
"kind": "text",
|
|
},
|
|
request=httpx.Request("POST", "http://a:8080/x"),
|
|
)
|
|
|
|
app, _ = self._wire(_mock)
|
|
client = TestClient(app, raise_server_exceptions=False)
|
|
try:
|
|
resp = client.post(
|
|
"/v1/api/route/workstreams/ws-X/attachments",
|
|
files=[("file", ("a.txt", b"hello", "text/plain"))],
|
|
headers=_AUTH,
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.json()["attachment_id"] == "att-1"
|
|
assert "/v1/api/workstreams/ws-X/attachments" in captured["url"]
|
|
assert "/route/" not in captured["url"]
|
|
assert captured["headers"].get("Content-Type", "").startswith("multipart/form-data")
|
|
finally:
|
|
client.close()
|
|
|
|
def test_list_proxies_get(self):
|
|
async def _mock(*args: Any, **kwargs: Any) -> httpx.Response:
|
|
return httpx.Response(
|
|
200,
|
|
json={"attachments": []},
|
|
request=httpx.Request("GET", "http://a:8080/x"),
|
|
)
|
|
|
|
app, mock_proxy = self._wire(_mock)
|
|
client = TestClient(app, raise_server_exceptions=False)
|
|
try:
|
|
resp = client.get(
|
|
"/v1/api/route/workstreams/ws-X/attachments",
|
|
headers=_AUTH,
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"attachments": []}
|
|
mock_proxy.get.assert_called()
|
|
finally:
|
|
client.close()
|
|
|
|
def test_get_content_preserves_upstream_headers(self):
|
|
async def _mock(*args: Any, **kwargs: Any) -> httpx.Response:
|
|
return httpx.Response(
|
|
200,
|
|
content=b"hello world",
|
|
headers={
|
|
"Content-Type": "text/plain; charset=utf-8",
|
|
"Content-Disposition": 'inline; filename="notes.md"',
|
|
"X-Content-Type-Options": "nosniff",
|
|
},
|
|
request=httpx.Request("GET", "http://a:8080/x"),
|
|
)
|
|
|
|
app, _ = self._wire(_mock)
|
|
client = TestClient(app, raise_server_exceptions=False)
|
|
try:
|
|
resp = client.get(
|
|
"/v1/api/route/workstreams/ws-X/attachments/att-1/content",
|
|
headers=_AUTH,
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.content == b"hello world"
|
|
assert resp.headers.get("X-Content-Type-Options") == "nosniff"
|
|
assert "filename" in resp.headers.get("Content-Disposition", "")
|
|
finally:
|
|
client.close()
|
|
|
|
def test_delete_proxies_method(self):
|
|
captured: dict[str, Any] = {}
|
|
|
|
async def _mock(*args: Any, **kwargs: Any) -> httpx.Response:
|
|
captured["method"] = args[0] if args else ""
|
|
captured["url"] = args[1] if len(args) > 1 else ""
|
|
return httpx.Response(
|
|
200,
|
|
json={"status": "deleted"},
|
|
request=httpx.Request("DELETE", "http://a:8080/x"),
|
|
)
|
|
|
|
app, _ = self._wire(_mock)
|
|
client = TestClient(app, raise_server_exceptions=False)
|
|
try:
|
|
resp = client.delete(
|
|
"/v1/api/route/workstreams/ws-X/attachments/att-1",
|
|
headers=_AUTH,
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"status": "deleted"}
|
|
assert captured["method"] == "DELETE"
|
|
finally:
|
|
client.close()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Routing-failure paths
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestRoutingFailures:
|
|
def test_router_not_ready_returns_503(self):
|
|
router = MagicMock(spec=ConsoleRouter)
|
|
router.is_ready.return_value = False
|
|
router.refresh_cache.return_value = None
|
|
app = _make_app(router=router)
|
|
app.state.proxy_client = MagicMock(spec=httpx.AsyncClient)
|
|
client = TestClient(app, raise_server_exceptions=False)
|
|
try:
|
|
resp = client.get(
|
|
"/v1/api/route/workstreams/ws-X/attachments",
|
|
headers=_AUTH,
|
|
)
|
|
assert resp.status_code == 503
|
|
finally:
|
|
client.close()
|