mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Patrick Buckley
723cad24bb
* feat: structured memory system — typed/scoped memories with BM25 relevance and metacognitive prompting Replace flat key-value memories table with structured_memories (migration 014). Four memory types (user/project/feedback/reference), three scopes (global/workstream/user). Consolidate remember/recall/forget into two tools: memory (action-based: save/search/delete/list) and recall (conversation history only). BM25 relevance scoring (extracted to turnstone/core/bm25.py) selects top-5 memories for system message injection based on conversation context. Metacognitive prompting injects ephemeral nudges after corrections, tool denials, workstream resume, and completion signals. Scope isolation enforced: system message injection and nudge counts filtered to visible memories only (global + current workstream + authenticated user). User scope requires authentication. Content capped at 32KB. ILIKE/LIKE metacharacters escaped in both backends. 113 new tests (2053 total). * fix: CI failure + copilot review feedback - Fix time.monotonic() cooldown: use None sentinel instead of 0.0 default (monotonic clock starts at boot, not epoch — fresh CI runners have uptime < 300s so cooldown check always triggered) - Catch sa.exc.IntegrityError specifically in upsert instead of broad Exception (copilot review) - Preserve existing description/type on upsert when caller doesn't explicitly set them (copilot review) - Add last_accessed + access_count columns to schema/migration for future LRU/LFU eviction support
72 lines
2.3 KiB
Python
72 lines
2.3 KiB
Python
"""Tests for turnstone.core.bm25 — tokenizer and BM25 index."""
|
|
|
|
from turnstone.core.bm25 import BM25Index, _tokenize
|
|
|
|
|
|
class TestTokenize:
|
|
def test_simple_words(self):
|
|
assert _tokenize("hello world") == ["hello", "world"]
|
|
|
|
def test_underscores(self):
|
|
assert _tokenize("read_file") == ["read", "file"]
|
|
|
|
def test_hyphens(self):
|
|
assert _tokenize("web-search") == ["web", "search"]
|
|
|
|
def test_dots(self):
|
|
assert _tokenize("foo.bar.baz") == ["foo", "bar", "baz"]
|
|
|
|
def test_mixed_separators(self):
|
|
assert _tokenize("mcp__server__read_file") == ["mcp", "server", "read", "file"]
|
|
|
|
def test_empty_string(self):
|
|
assert _tokenize("") == []
|
|
|
|
def test_case_folding(self):
|
|
assert _tokenize("Hello World") == ["hello", "world"]
|
|
|
|
|
|
class TestBM25Index:
|
|
def test_search_returns_relevant(self):
|
|
docs = ["read a file from disk", "search for file in directory", "execute a bash command"]
|
|
index = BM25Index(docs)
|
|
results = index.search("file", k=2)
|
|
assert 0 in results
|
|
assert 1 in results
|
|
|
|
def test_search_empty_query(self):
|
|
docs = ["hello world"]
|
|
index = BM25Index(docs)
|
|
assert index.search("") == []
|
|
|
|
def test_search_no_match(self):
|
|
docs = ["hello world", "foo bar"]
|
|
index = BM25Index(docs)
|
|
assert index.search("zzzznotfound") == []
|
|
|
|
def test_search_respects_k(self):
|
|
docs = [f"document {i} with common word" for i in range(20)]
|
|
index = BM25Index(docs)
|
|
results = index.search("common", k=3)
|
|
assert len(results) <= 3
|
|
|
|
def test_empty_corpus(self):
|
|
index = BM25Index([])
|
|
assert index.search("anything") == []
|
|
|
|
def test_single_document(self):
|
|
index = BM25Index(["the only document about turnstone"])
|
|
results = index.search("turnstone")
|
|
assert results == [0]
|
|
|
|
def test_ordering_by_relevance(self):
|
|
docs = [
|
|
"unrelated content about cooking recipes",
|
|
"python programming with file operations",
|
|
"read file write file file operations disk io",
|
|
]
|
|
index = BM25Index(docs)
|
|
results = index.search("file operations", k=3)
|
|
# Doc 2 has more file/operations mentions, should rank higher
|
|
assert results[0] == 2
|