From 15b3aad8152a8b13bb1deecf79545acdac7abf37 Mon Sep 17 00:00:00 2001 From: Patrick Buckley Date: Sun, 31 May 2026 18:58:30 -0700 Subject: [PATCH] feat(web-search): let the model pick a SearxNG category MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename the web_search tool's `topic` parameter to `category` and expand the enum to general/news/it/science, mapped to SearxNG `categories=`. The model can now target the right corpus per query (e.g. `it` for code, `science` for papers) — useful when generic engines rate-limit. The Tavily-era `finance` topic (no SearxNG equivalent) is dropped. Threaded consistently through _prepare_web_search / _exec_web_search / both search clients. BREAKING: the web_search `topic` argument is now `category`. --- CHANGELOG.md | 4 ++++ docs/tools.md | 2 +- tests/test_web_search.py | 24 ++++++++++++++++++------ turnstone/core/session.py | 14 +++++++------- turnstone/core/web_search.py | 16 ++++++++-------- turnstone/tools/web_search.json | 5 +++-- 6 files changed, 41 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac5c9f97..ad7f9856 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,10 @@ Three release tracks are maintained: values are gone; a config still set to either disables web search and logs a warning. Auto-detect resolves to SearxNG when `searxng_url` is set, otherwise no client (the `web_search` tool is dropped for models without native search). +- **`web_search` tool: `topic` → `category`** *(BREAKING)* — the LLM-facing + parameter is renamed and its values are now `general` (default), `news`, `it` + (code/tech), or `science`, mapped to SearxNG categories so the model can target + the right corpus. The Tavily-era `finance` topic (no SearxNG equivalent) is gone. ### Removed diff --git a/docs/tools.md b/docs/tools.md index d66adb1f..9497a005 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -344,7 +344,7 @@ Search the web using a text query. |---------------|---------|----------|-------------| | `query` | string | yes | The search query. | | `max_results` | integer | no | Max results to return (default 5, max 20). | -| `topic` | string | no | Search topic: `general` or `news` (default `general`). | +| `category` | string | no | Search category: `general` (default), `news`, `it` (code/tech), or `science`. Maps to SearxNG categories; the model picks per query. | - **What it does**: Searches the web and returns ranked results with titles, URLs, and content snippets. Uses provider-native search when available: - **Anthropic**: Replaced at the API boundary with Anthropic's `web_search_20250305` server-side tool. Claude decides when to search; the API executes it and returns results with citations inline. No backend needed. diff --git a/tests/test_web_search.py b/tests/test_web_search.py index 7048cf92..202f21f0 100644 --- a/tests/test_web_search.py +++ b/tests/test_web_search.py @@ -131,16 +131,16 @@ class TestSearXNGClient: client = SearXNGClient( "http://searxng:8080", engines="duckduckgo,wikipedia", timeout=10 ) - out = client.search("python", max_results=2, topic="news") + out = client.search("python", max_results=2, category="news") assert captured["url"].startswith("http://searxng:8080/search") assert captured["params"]["q"] == "python" assert captured["params"]["format"] == "json" - assert captured["params"]["categories"] == "news" # topic -> categories + assert captured["params"]["categories"] == "news" # category -> categories assert captured["params"]["engines"] == "duckduckgo,wikipedia" assert "[Python.org](https://python.org)" in out - def test_general_topic_omits_categories(self): + def test_category_it_maps_to_categories(self): captured: dict = {} def handler(request: httpx.Request) -> httpx.Response: @@ -148,7 +148,19 @@ class TestSearXNGClient: return httpx.Response(200, json={"results": []}) with patch("turnstone.core.web_search.httpx.get", _mock_httpx_get(handler)): - SearXNGClient("http://searxng:8080").search("q", topic="general") + SearXNGClient("http://searxng:8080").search("q", category="it") + + assert captured["params"]["categories"] == "it" + + def test_general_category_omits_categories(self): + captured: dict = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["params"] = dict(request.url.params) + return httpx.Response(200, json={"results": []}) + + with patch("turnstone.core.web_search.httpx.get", _mock_httpx_get(handler)): + SearXNGClient("http://searxng:8080").search("q", category="general") assert "categories" not in captured["params"] @@ -199,10 +211,10 @@ class TestMCPSearchClient: mcp = MagicMock() mcp.call_tool_sync.return_value = "MCP search results" client = MCPSearchClient(mcp, "mcp__ddg__search", timeout=30) - result = client.search("test", max_results=3, topic="news") + result = client.search("test", max_results=3, category="news") mcp.call_tool_sync.assert_called_once_with( "mcp__ddg__search", - {"query": "test", "max_results": 3, "topic": "news"}, + {"query": "test", "max_results": 3, "category": "news"}, timeout=30, ) assert result == "MCP search results" diff --git a/turnstone/core/session.py b/turnstone/core/session.py index 79bf7fcc..2318862e 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -5098,7 +5098,7 @@ class ChatSession: elif name == "web_fetch": it["func_args"] = {"url": it.get("url", ""), "question": it.get("question", "")} elif name == "web_search": - it["func_args"] = {"query": it.get("query", ""), "topic": it.get("topic", "")} + it["func_args"] = {"query": it.get("query", ""), "category": it.get("category", "")} elif name == "skills": # Projection for judge / audit on the model-facing skills # tool. Mutating actions surface name + action + a snippet @@ -6977,9 +6977,9 @@ class ChatSession: max_results = min(max(int(args.get("max_results") or 5), 1), 20) except (ValueError, TypeError): max_results = 5 - topic = args.get("topic", "general") or "general" - if topic not in ("general", "news"): - topic = "general" + category = args.get("category", "general") or "general" + if category not in ("general", "news", "it", "science"): + category = "general" q_preview = query[:200] + ("..." if len(query) > 200 else "") preview = f" {q_preview}" return { @@ -6992,7 +6992,7 @@ class ChatSession: "execute": self._exec_web_search, "query": query, "max_results": max_results, - "topic": topic, + "category": category, } def _prepare_tool_search(self, call_id: str, args: dict[str, Any]) -> dict[str, Any]: @@ -12600,7 +12600,7 @@ class ChatSession: call_id = item["call_id"] query = item["query"] max_results = item.get("max_results", 5) - topic = item.get("topic", "general") + category = item.get("category", "general") client = self._resolve_search_client() if not client: @@ -12609,7 +12609,7 @@ class ChatSession: return call_id, msg try: - output = client.search(query, max_results=max_results, topic=topic) + output = client.search(query, max_results=max_results, category=category) except Exception as e: msg = f"Error: web search failed: {e}" self._report_tool_result(call_id, "web_search", msg, is_error=True) diff --git a/turnstone/core/web_search.py b/turnstone/core/web_search.py index 00f06865..adfa07a3 100644 --- a/turnstone/core/web_search.py +++ b/turnstone/core/web_search.py @@ -44,10 +44,10 @@ class SearXNGClient: config does this; a stock instance returns 403/HTML otherwise. """ - # The web_search tool's ``topic`` arg → SearxNG ``categories``. ``"general"`` - # is intentionally absent so the param is omitted and SearxNG uses its - # default category mix. - _TOPIC_CATEGORIES = {"news": "news"} + # The web_search tool's ``category`` arg → SearxNG ``categories``. + # ``"general"`` is intentionally absent so the param is omitted and SearxNG + # uses its default category mix. + _CATEGORIES = {"news": "news", "it": "it", "science": "science"} def __init__(self, base_url: str, engines: str = "", timeout: float = 120) -> None: self._base_url = base_url.rstrip("/") @@ -56,7 +56,7 @@ class SearXNGClient: def search(self, query: str, max_results: int = 5, **kwargs: Any) -> str: params: dict[str, str] = {"q": query, "format": "json"} - category = self._TOPIC_CATEGORIES.get(str(kwargs.get("topic", "general"))) + category = self._CATEGORIES.get(str(kwargs.get("category", "general"))) if category: params["categories"] = category if self._engines: @@ -84,9 +84,9 @@ class MCPSearchClient: args: dict[str, Any] = {"query": query} if max_results != 5: args["max_results"] = max_results - topic = kwargs.get("topic") - if topic: - args["topic"] = topic + category = kwargs.get("category") + if category: + args["category"] = category return self._mcp.call_tool_sync(self._tool, args, timeout=max(1, math.ceil(self._timeout))) diff --git a/turnstone/tools/web_search.json b/turnstone/tools/web_search.json index e3098a59..8bb82534 100644 --- a/turnstone/tools/web_search.json +++ b/turnstone/tools/web_search.json @@ -12,9 +12,10 @@ "type": "integer", "description": "Max results to return (default 5, max 20)." }, - "topic": { + "category": { "type": "string", - "description": "Search topic: general or news (default general)." + "enum": ["general", "news", "it", "science"], + "description": "Search category: general (default), news (current events), it (code/programming/tech), science (papers/research)." } }, "required": ["query"]