mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Patrick Buckley
0923add7db
* fix: improve web_fetch reliability — strip scripts, dynamic truncation, more tokens - strip_html() now removes <script>, <style>, <template>, <noscript> element content instead of just their tags - Truncation budget scales with context window (75% in chars, 50k floor) and takes from the beginning only instead of head+tail splice - max_tokens bumped from 2000 to 8192 so thinking models don't starve the visible extraction answer - reasoning_effort="low" on summarization call to avoid wasting tokens - Empty responses and empty extractions now report as tool errors * refactor: extract _utility_completion to fix reasoning_effort duplication Callers previously had to pass reasoning_effort both as a direct keyword (for commercial providers) and via _provider_extra_params (for local model servers). This duplication was easy to get wrong — web_fetch was already missing the direct keyword. _utility_completion threads it through both paths from a single call, used by title generation, compaction, and web_fetch extraction. * fix: disable thinking when max_tokens too small, cap extraction at 500k _reasoning_params now returns empty dict when max_tokens can't fit a thinking budget (e.g. title gen with max_tokens=200). Previously produced budget_tokens >= max_tokens which is an API error on manual-thinking Anthropic models. Also caps web_fetch content truncation at 500k chars — the dynamic context-window calc was producing 3M chars on 1M-context models. * fix: clamp utility max_tokens to model output limit, add strip_html tests _utility_completion now clamps max_tokens to the model's advertised max_output_tokens so small/local models don't reject 8192-token requests. Adds 8 tests for invisible element stripping (script, style, template, noscript) including multiline, case-insensitive, and attribute cases. * fix: mock get_capabilities in title retry tests for _utility_completion _utility_completion calls _get_capabilities to clamp max_tokens. The existing title tests mocked _provider as a bare MagicMock, so caps.max_output_tokens was a truthy MagicMock instead of an int. Set get_capabilities to return a real ModelCapabilities instance.
92 lines
3.1 KiB
Python
92 lines
3.1 KiB
Python
"""Tests for turnstone.core.web — strip_html."""
|
|
|
|
from turnstone.core.web import strip_html
|
|
|
|
|
|
class TestStripHtml:
|
|
def test_removes_tags(self):
|
|
assert strip_html("<b>hello</b>") == "hello"
|
|
|
|
def test_removes_nested_tags(self):
|
|
assert strip_html("<div><p>text</p></div>") == "text"
|
|
|
|
def test_decodes_entities(self):
|
|
assert strip_html("& < >") == "& < >"
|
|
|
|
def test_collapses_whitespace(self):
|
|
result = strip_html("hello world")
|
|
assert result == "hello world"
|
|
|
|
def test_collapses_blank_lines(self):
|
|
result = strip_html("a\n\n\n\n\nb")
|
|
assert result == "a\n\nb"
|
|
|
|
def test_empty_string(self):
|
|
assert strip_html("") == ""
|
|
|
|
def test_strips_leading_trailing_whitespace(self):
|
|
assert strip_html(" hello ") == "hello"
|
|
|
|
def test_complex_html(self):
|
|
html = "<html><body><h1>Title</h1><p>Some & text</p></body></html>"
|
|
result = strip_html(html)
|
|
assert "Title" in result
|
|
assert "Some & text" in result
|
|
assert "<" not in result
|
|
|
|
def test_self_closing_tags(self):
|
|
result = strip_html("hello<br/>world")
|
|
assert result == "helloworld"
|
|
|
|
# -- invisible element stripping -----------------------------------------
|
|
|
|
def test_strips_script_content(self):
|
|
html = "<p>before</p><script>var x = 1;</script><p>after</p>"
|
|
result = strip_html(html)
|
|
assert "var x" not in result
|
|
assert "before" in result
|
|
assert "after" in result
|
|
|
|
def test_strips_style_content(self):
|
|
html = "<style>.foo { color: red; }</style><p>visible</p>"
|
|
result = strip_html(html)
|
|
assert "color" not in result
|
|
assert "visible" in result
|
|
|
|
def test_strips_template_content(self):
|
|
html = "<template><div>hidden</div></template><p>shown</p>"
|
|
result = strip_html(html)
|
|
assert "hidden" not in result
|
|
assert "shown" in result
|
|
|
|
def test_strips_noscript_content(self):
|
|
html = "<noscript>Enable JS</noscript><p>content</p>"
|
|
result = strip_html(html)
|
|
assert "Enable JS" not in result
|
|
assert "content" in result
|
|
|
|
def test_strips_multiple_script_blocks(self):
|
|
html = "<script>a()</script><p>middle</p><script>b()</script>"
|
|
result = strip_html(html)
|
|
assert "a()" not in result
|
|
assert "b()" not in result
|
|
assert "middle" in result
|
|
|
|
def test_strips_multiline_script(self):
|
|
html = "<script>\nfunction foo() {\n return 1;\n}\n</script><p>ok</p>"
|
|
result = strip_html(html)
|
|
assert "function" not in result
|
|
assert "ok" in result
|
|
|
|
def test_strips_script_case_insensitive(self):
|
|
html = "<SCRIPT>code()</SCRIPT><p>text</p>"
|
|
result = strip_html(html)
|
|
assert "code()" not in result
|
|
assert "text" in result
|
|
|
|
def test_strips_script_with_attributes(self):
|
|
html = '<script type="text/javascript" src="app.js">init();</script><p>done</p>'
|
|
result = strip_html(html)
|
|
assert "init()" not in result
|
|
assert "done" in result
|