diff --git a/README.md b/README.md index 46ac996e..d679d727 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ Bridges BLPOP from their per-node queue (priority) then the shared queue. Direct ## Tools -16 built-in tools, 2 agent tools, plus external tools via MCP: +15 built-in tools, 2 agent tools, plus external tools via MCP: | Tool | Description | Auto-approved | |------|-------------|:---:| @@ -206,9 +206,8 @@ Bridges BLPOP from their per-node queue (priority) then the shared queue. Direct | `man` | Read man pages | yes | | `web_fetch` | Fetch URL content | | | `web_search` | Web search (provider-native or Tavily) | | -| `remember` | Save persistent facts | yes | -| `recall` | Search memories and history | yes | -| `forget` | Remove a memory | yes | +| `memory` | Structured persistent memory (save/search/delete/list) | yes | +| `recall` | Search conversation history | yes | | `notify` | Send notifications to linked channels | yes | | `watch` | Periodic command polling with conditions | | | `task` | Spawn autonomous sub-agent | | diff --git a/docs/architecture.md b/docs/architecture.md index a36c0c70..f7f437a8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -3,7 +3,7 @@ Turnstone is an AI orchestration platform with tool use, parallel workstreams, and persistent memory. It connects to any OpenAI-compatible API (local vLLM, OpenAI, etc.) or Anthropic's native Messages API via pluggable provider adapters, and gives the -model 18 built-in tools plus external tools via MCP (Model Context Protocol) for +model 17 built-in tools plus external tools via MCP (Model Context Protocol) for reading, writing, searching, planning, and executing code. The core design principle is a **UI-agnostic engine with pluggable frontends**. @@ -436,13 +436,13 @@ from each schema and builds: - `PRIMARY_KEY_MAP` -- `{name: primary_key}` for JSON fallback recovery - `merge_mcp_tools(builtin, mcp_tools)` -- merges built-in + MCP tools at session init -### 14 Tools by Category +### 13 Tools by Category **Read-only (auto-approve)**: - `read_file` -- read file contents with optional offset/limit - `search` -- ripgrep-based codebase search - `man` -- read man pages -- `recall` -- retrieve stored memories +- `recall` -- search conversation history **Write (requires approval)**: - `bash` -- execute shell commands (with safety checks via `turnstone.core.safety`) @@ -456,9 +456,8 @@ from each schema and builds: - `task` -- delegate to a sub-agent with full tool access (`TASK_AGENT_TOOLS`) - `plan` -- explore codebase and write a structured plan (`AGENT_TOOLS`) -**Memory (persistent key-value store)**: -- `remember` -- save a fact -- `forget` -- delete a fact +**Memory (structured persistent store)**: +- `memory` -- save, search, delete, or list memories (typed and scoped) ### Prepare / Execute Pattern diff --git a/docs/diagrams/04-conversation-turn.puml b/docs/diagrams/04-conversation-turn.puml index 3a073800..1a1b0305 100644 --- a/docs/diagrams/04-conversation-turn.puml +++ b/docs/diagrams/04-conversation-turn.puml @@ -127,7 +127,7 @@ group loop [while tool_calls present] math → sandboxed subprocess web_fetch → httpx + LLM summarize web_search → provider-native or Tavily fallback - remember/recall/forget → SQLite + memory/recall → SQLite end note note right of TP diff --git a/docs/diagrams/05-tool-pipeline.puml b/docs/diagrams/05-tool-pipeline.puml index 1d8aba85..f4d65cd3 100644 --- a/docs/diagrams/05-tool-pipeline.puml +++ b/docs/diagrams/05-tool-pipeline.puml @@ -24,7 +24,7 @@ partition "Phase 1: Prepare" #E8F5E9 { :Dispatch to _prepare_{func_name}(); note right - **Dispatch table (18 tools):** + **Dispatch table (17 tools):** ┌───────────────┬──────────────────┐ │ Tool │ Needs Approval? │ ├───────────────┼──────────────────┤ @@ -40,9 +40,8 @@ partition "Phase 1: Prepare" #E8F5E9 { │ tool_search │ ✗ Auto-approve │ │ task │ ✓ Yes │ │ plan │ ✓ Yes │ - │ remember │ ✗ Auto-approve │ + │ memory │ ✗ Auto-approve │ │ recall │ ✗ Auto-approve │ - │ forget │ ✗ Auto-approve │ │ notify │ ✗ Auto-approve │ │ read_resource │ ✓ Yes │ │ use_prompt │ ✓ Yes │ @@ -116,9 +115,8 @@ partition "Phase 3: Execute" #E3F2FD { ├─ _exec_task: _run_agent(TASK_AGENT_TOOLS) ├─ _exec_plan: _run_agent(AGENT_TOOLS, read-only) ├─ _exec_notify: HTTP POST to channel gateway - ├─ _exec_remember: SQLite INSERT OR REPLACE - ├─ _exec_recall: SQLite FTS5/LIKE search - ├─ _exec_forget: SQLite DELETE + ├─ _exec_memory: structured memory save/search/delete/list + ├─ _exec_recall: conversation history FTS5 search ├─ _exec_read_resource: MCPClientManager.read_resource_sync() ├─ _exec_use_prompt: MCPClientManager.get_prompt_sync() └─ _exec_mcp_tool: MCPClientManager.call_tool_sync() diff --git a/docs/diagrams/07-message-routing.puml b/docs/diagrams/07-message-routing.puml index 521a59a8..a17d44bb 100644 --- a/docs/diagrams/07-message-routing.puml +++ b/docs/diagrams/07-message-routing.puml @@ -79,7 +79,7 @@ note right of BridgeA 1. _ws_auto_approve[ws_id]? → auto 2. All tools in safe set? → auto (read_file, search, man, - remember, recall, forget) + memory, recall) 3. Otherwise → manual approval end note diff --git a/docs/tools.md b/docs/tools.md index 836bdef4..ad0aaea0 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -1,6 +1,6 @@ # Tools Reference -turnstone exposes 18 built-in tools plus any number of external MCP tools to the +turnstone exposes 17 built-in tools plus any number of external MCP tools to the LLM via the OpenAI function-calling interface. Built-in tools are defined as JSON files under `turnstone/tools/` and loaded at startup by `turnstone/core/tools.py`. MCP tools are discovered from configured MCP servers at startup by @@ -46,12 +46,12 @@ schema plus turnstone-specific metadata keys: | Name | Description | |---------------------|-------------| -| `TOOLS` | All 18 tool definitions (sent to the model). | +| `TOOLS` | All 17 tool definitions (sent to the model). | | `AGENT_TOOLS` | Tools with `agent: true` -- available to plan sub-agents. Read-only tools. | | `TASK_AGENT_TOOLS` | Tools with `task_agent: true` -- available to task sub-agents. Includes write operations. | | `AGENT_AUTO_TOOLS` | Set of tool names with `auto_approve: true` -- no user confirmation needed. | | `TASK_AUTO_TOOLS` | Same as `AGENT_AUTO_TOOLS` (identical filter). | -| `BUILTIN_TOOL_NAMES`| Frozenset of all 18 built-in tool names. Used by tool search to distinguish always-on tools from deferrable MCP tools. | +| `BUILTIN_TOOL_NAMES`| Frozenset of all 17 built-in tool names. Used by tool search to distinguish always-on tools from deferrable MCP tools. | | `PRIMARY_KEY_MAP` | Dict mapping tool name to its `primary_key` parameter name. | --- @@ -69,7 +69,7 @@ Tool execution follows a three-phase pipeline inside `ChatSession._execute_tools - Parses the JSON arguments (with fallback for malformed JSON). - If JSON parsing fails entirely, uses `PRIMARY_KEY_MAP` to map a bare string to the correct parameter. -- Dispatches to the matching `_prepare_{func_name}()` handler. There are 18 +- Dispatches to the matching `_prepare_{func_name}()` handler. There are 17 built-in tools plus `tool_search` (synthetic, client-side BM25 fallback) and the generic `_prepare_mcp_tool()` handler for MCP tools. - Validates arguments and builds a preview dict containing: @@ -114,9 +114,8 @@ Each item's `execute` callable is invoked: - `read_file` -- reads files, no side effects - `search` -- grep-style search, no side effects - `man` -- reads man pages, no side effects -- `remember` -- writes to persistent memory database (lightweight, always auto-approved) -- `recall` -- reads from persistent memory database -- `forget` -- deletes from persistent memory database (lightweight, always auto-approved) +- `memory` -- structured persistent memory (save/search/delete/list) +- `recall` -- searches conversation history - `notify` -- sends notifications to linked channels (time-sensitive, auto-approved for urgency) **Requires user confirmation** (write operations, network access, side effects): @@ -164,9 +163,8 @@ Every tool defines a `primary_key`. The mapping is: | `web_search` | `query` | | `task` | `prompt` | | `plan` | `prompt` | -| `remember` | `key` | +| `memory` | `name` | | `recall` | `query` | -| `forget` | `key` | | `notify` | `message` | | `read_resource` | `uri` | | `use_prompt` | `name` | @@ -353,16 +351,22 @@ Plan before implementing -- an autonomous agent explores the codebase and writes ## Memory -### remember +### memory -Save a persistent memory that persists across sessions. +Structured persistent memory across sessions with typed, scoped entries. -| Parameter | Type | Required | Description | -|-----------|--------|----------|-------------| -| `key` | string | yes | Short identifier (e.g. `user_name`). | -| `value` | string | yes | Content to remember. | +| Parameter | Type | Required | Description | +|---------------|---------|----------|-------------| +| `action` | string | yes | `save`, `search`, `delete`, or `list`. | +| `name` | string | save/delete | Short snake_case identifier for the memory. | +| `content` | string | save | Memory content to store. | +| `description` | string | no | Short description for relevance matching (recommended for `save`). | +| `type` | string | no | Memory type: `user`, `project`, `feedback`, or `reference`. Default: `project`. | +| `scope` | string | no | Memory scope: `global`, `workstream`, or `user`. Default: `global`. | +| `query` | string | search | Search query for finding memories. | +| `limit` | integer | no | Max results for `search` or `list`. Default: 20. | -- **What it does**: Stores a key-value pair in the SQLite memory database. Memories persist across sessions and are included in the system prompt on startup. +- **What it does**: Manages structured persistent memories in the database. Memories persist across sessions, have a type classification (user preferences, project knowledge, feedback, reference material) and a scope (global across all workstreams, private to a workstream, or following a user). Relevant memories are included in the system prompt on startup. - **Auto-approve**: Yes. - **Agent availability**: Not available to sub-agents (top-level only). @@ -370,28 +374,14 @@ Save a persistent memory that persists across sessions. ### recall -Search memories and past conversations. +Search conversation history for past messages and tool results. | Parameter | Type | Required | Description | |-----------|---------|----------|-------------| -| `query` | string | no | Search term or phrase. Omit to list all memories. | -| `limit` | integer | no | Max conversation results to return (default 20). | +| `query` | string | yes | Search term or phrase to find in conversation history. | +| `limit` | integer | no | Max results to return (default 20). | -- **What it does**: With no query, lists all saved memories. With a query, searches both the memory store and conversation history using FTS5 full-text search. -- **Auto-approve**: Yes. -- **Agent availability**: Not available to sub-agents (top-level only). - ---- - -### forget - -Remove a persistent memory by key. - -| Parameter | Type | Required | Description | -|-----------|--------|----------|-------------| -| `key` | string | yes | The memory key to remove (e.g. `user_name`). | - -- **What it does**: Deletes the memory entry with the given key from the SQLite database. +- **What it does**: Searches conversation history across sessions using FTS5 full-text search. Returns matching messages, tool calls, and tool results with timestamps and workstream context. - **Auto-approve**: Yes. - **Agent availability**: Not available to sub-agents (top-level only). @@ -514,9 +504,8 @@ data.get("mergedAt") is not None | `web_search` | Info | No | Yes | Yes | `query` | | `task` | Agent | No | No | No | `prompt` | | `plan` | Agent | No | No | No | `prompt` | -| `remember` | Memory | Yes | No | No | `key` | +| `memory` | Memory | Yes | No | No | `name` | | `recall` | Memory | Yes | No | No | `query` | -| `forget` | Memory | Yes | No | No | `key` | | `notify` | Notify | Yes | Yes | Yes | `message` | | `watch` | Monitor | No (create) | No | No | `command` | | `read_resource`| MCP | No | Yes | Yes | `uri` | @@ -573,7 +562,7 @@ CLI flags override the config file: search stays off and all tools are sent to the model directly. 2. **Partitioning**: When active, tools are split into two sets: - - **Always-on** -- the 18 built-in tools (members of `BUILTIN_TOOL_NAMES`). + - **Always-on** -- the 17 built-in tools (members of `BUILTIN_TOOL_NAMES`). These are always visible to the model. - **Deferred** -- all MCP tools. These are not sent in the tool list unless the model searches for them. @@ -616,7 +605,7 @@ MCP-compatible service. 3. **Schema conversion**: Each MCP tool's `inputSchema` is converted to OpenAI function-calling format. The tool name is prefixed: `mcp__{server}__{tool}`. -4. **Merging**: MCP tools are appended after the 18 built-in tools via +4. **Merging**: MCP tools are appended after the 17 built-in tools via `merge_mcp_tools()`. Built-in tools appear first, giving them natural LLM priority. When dynamic tool search is active, MCP tools are deferred rather than directly visible -- the model discovers them via search as needed (see diff --git a/tests/test_bm25.py b/tests/test_bm25.py new file mode 100644 index 00000000..2ee97595 --- /dev/null +++ b/tests/test_bm25.py @@ -0,0 +1,71 @@ +"""Tests for turnstone.core.bm25 — tokenizer and BM25 index.""" + +from turnstone.core.bm25 import BM25Index, _tokenize + + +class TestTokenize: + def test_simple_words(self): + assert _tokenize("hello world") == ["hello", "world"] + + def test_underscores(self): + assert _tokenize("read_file") == ["read", "file"] + + def test_hyphens(self): + assert _tokenize("web-search") == ["web", "search"] + + def test_dots(self): + assert _tokenize("foo.bar.baz") == ["foo", "bar", "baz"] + + def test_mixed_separators(self): + assert _tokenize("mcp__server__read_file") == ["mcp", "server", "read", "file"] + + def test_empty_string(self): + assert _tokenize("") == [] + + def test_case_folding(self): + assert _tokenize("Hello World") == ["hello", "world"] + + +class TestBM25Index: + def test_search_returns_relevant(self): + docs = ["read a file from disk", "search for file in directory", "execute a bash command"] + index = BM25Index(docs) + results = index.search("file", k=2) + assert 0 in results + assert 1 in results + + def test_search_empty_query(self): + docs = ["hello world"] + index = BM25Index(docs) + assert index.search("") == [] + + def test_search_no_match(self): + docs = ["hello world", "foo bar"] + index = BM25Index(docs) + assert index.search("zzzznotfound") == [] + + def test_search_respects_k(self): + docs = [f"document {i} with common word" for i in range(20)] + index = BM25Index(docs) + results = index.search("common", k=3) + assert len(results) <= 3 + + def test_empty_corpus(self): + index = BM25Index([]) + assert index.search("anything") == [] + + def test_single_document(self): + index = BM25Index(["the only document about turnstone"]) + results = index.search("turnstone") + assert results == [0] + + def test_ordering_by_relevance(self): + docs = [ + "unrelated content about cooking recipes", + "python programming with file operations", + "read file write file file operations disk io", + ] + index = BM25Index(docs) + results = index.search("file operations", k=3) + # Doc 2 has more file/operations mentions, should rank higher + assert results[0] == 2 diff --git a/tests/test_db.py b/tests/test_db.py index 5c3de26c..e332c00c 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -16,7 +16,10 @@ class TestSchemaCreation: engine = get_storage()._engine # noqa: SLF001 with engine.connect() as conn: rows = conn.execute( - sa.text("SELECT name FROM sqlite_master WHERE type='table' AND name='memories'") + sa.text( + "SELECT name FROM sqlite_master " + "WHERE type='table' AND name='structured_memories'" + ) ).fetchall() assert len(rows) == 1 rows = conn.execute( diff --git a/tests/test_memory_relevance.py b/tests/test_memory_relevance.py new file mode 100644 index 00000000..9ca1d4ba --- /dev/null +++ b/tests/test_memory_relevance.py @@ -0,0 +1,194 @@ +"""Tests for turnstone.core.memory_relevance — scoring, formatting, context extraction.""" + +from turnstone.core.memory_relevance import ( + build_memory_context, + extract_recent_context, + score_memories, +) + +# --------------------------------------------------------------------------- +# score_memories +# --------------------------------------------------------------------------- + + +class TestScoreMemories: + def test_empty_memories(self): + assert score_memories([], "query") == [] + + def test_empty_query_returns_recent(self): + mems = [ + {"name": "a", "description": "", "content": "alpha"}, + {"name": "b", "description": "", "content": "beta"}, + {"name": "c", "description": "", "content": "gamma"}, + ] + result = score_memories(mems, "", k=2) + assert len(result) == 2 + assert result[0]["name"] == "a" + + def test_whitespace_query_returns_recent(self): + mems = [{"name": "a", "description": "", "content": "alpha"}] + assert score_memories(mems, " ", k=5) == mems + + def test_relevance_ranking(self): + mems = [ + {"name": "cooking", "description": "recipes", "content": "pasta sauce tomato"}, + {"name": "python", "description": "programming", "content": "python file io disk"}, + { + "name": "disk_io", + "description": "file operations", + "content": "read write file disk", + }, + ] + result = score_memories(mems, "file disk", k=2) + names = [m["name"] for m in result] + assert "disk_io" in names + assert "python" in names + + def test_k_limits_results(self): + mems = [{"name": f"m{i}", "description": "", "content": f"word{i}"} for i in range(10)] + result = score_memories(mems, "word0 word1 word2", k=2) + assert len(result) <= 2 + + def test_no_match_returns_empty(self): + mems = [{"name": "a", "description": "", "content": "hello world"}] + result = score_memories(mems, "zzzznotfound") + assert result == [] + + def test_uses_name_for_scoring(self): + mems = [ + {"name": "database_config", "description": "", "content": "host=localhost"}, + {"name": "unrelated", "description": "", "content": "nothing here"}, + ] + result = score_memories(mems, "database", k=1) + assert len(result) == 1 + assert result[0]["name"] == "database_config" + + def test_uses_description_for_scoring(self): + mems = [ + {"name": "x", "description": "postgresql connection settings", "content": "host=db"}, + {"name": "y", "description": "unrelated", "content": "nothing"}, + ] + result = score_memories(mems, "postgresql", k=1) + assert result[0]["name"] == "x" + + +# --------------------------------------------------------------------------- +# build_memory_context +# --------------------------------------------------------------------------- + + +class TestBuildMemoryContext: + def test_empty_memories(self): + assert build_memory_context([]) == "" + + def test_single_memory(self): + mems = [{"name": "test", "type": "project", "scope": "global", "content": "hello"}] + ctx = build_memory_context(mems) + assert "" in ctx + assert "" in ctx + assert 'name="test"' in ctx + assert "hello" in ctx + + def test_html_escaping(self): + mems = [ + { + "name": "a= 1 + assert any(r["name"] == "db_host" for r in results) + + +class TestCountStructuredMemories: + def test_count_zero(self, tmp_db): + assert count_structured_memories() == 0 + + def test_count_after_save(self, tmp_db): + save_structured_memory("a", "1") + save_structured_memory("b", "2") + assert count_structured_memories() == 2 + + +class TestNormalizeKey: + def test_basic(self): + assert normalize_key("My-Key Name") == "my_key_name" diff --git a/tests/test_structured_memory_storage.py b/tests/test_structured_memory_storage.py new file mode 100644 index 00000000..e0539ef6 --- /dev/null +++ b/tests/test_structured_memory_storage.py @@ -0,0 +1,137 @@ +"""Tests for structured memory storage backend operations.""" + +import pytest + +from turnstone.core.storage._sqlite import SQLiteBackend + + +@pytest.fixture +def backend(tmp_path): + return SQLiteBackend(str(tmp_path / "test.db")) + + +class TestCreateAndGet: + def test_create_and_get_by_id(self, backend): + backend.create_structured_memory("m1", "test_key", "desc", "project", "global", "", "data") + mem = backend.get_structured_memory("m1") + assert mem is not None + assert mem["name"] == "test_key" + assert mem["content"] == "data" + assert mem["type"] == "project" + + def test_get_nonexistent(self, backend): + assert backend.get_structured_memory("nope") is None + + def test_get_by_name(self, backend): + backend.create_structured_memory("m1", "mykey", "d", "project", "global", "", "val") + mem = backend.get_structured_memory_by_name("mykey", "global", "") + assert mem is not None + assert mem["memory_id"] == "m1" + + def test_get_by_name_scoped(self, backend): + backend.create_structured_memory("m1", "key", "d", "project", "global", "", "g") + backend.create_structured_memory("m2", "key", "d", "project", "workstream", "ws1", "w") + g = backend.get_structured_memory_by_name("key", "global", "") + w = backend.get_structured_memory_by_name("key", "workstream", "ws1") + assert g["content"] == "g" + assert w["content"] == "w" + + +class TestUpdate: + def test_update_content(self, backend): + backend.create_structured_memory("m1", "k", "d", "project", "global", "", "old") + assert backend.update_structured_memory("m1", content="new") + mem = backend.get_structured_memory("m1") + assert mem["content"] == "new" + + def test_update_nonexistent(self, backend): + assert not backend.update_structured_memory("nope", content="x") + + def test_update_no_fields(self, backend): + backend.create_structured_memory("m1", "k", "d", "project", "global", "", "data") + assert not backend.update_structured_memory("m1", bogus="val") + + def test_update_bumps_timestamp(self, backend): + backend.create_structured_memory("m1", "k", "d", "project", "global", "", "data") + old = backend.get_structured_memory("m1")["updated"] + import time + + time.sleep(0.01) + backend.update_structured_memory("m1", content="new") + new = backend.get_structured_memory("m1")["updated"] + assert new >= old + + +class TestDelete: + def test_delete_existing(self, backend): + backend.create_structured_memory("m1", "k", "d", "project", "global", "", "data") + assert backend.delete_structured_memory("k", "global", "") + assert backend.get_structured_memory("m1") is None + + def test_delete_nonexistent(self, backend): + assert not backend.delete_structured_memory("nope", "global", "") + + def test_delete_scoped(self, backend): + backend.create_structured_memory("m1", "k", "d", "project", "workstream", "ws1", "data") + assert not backend.delete_structured_memory("k", "global", "") + assert backend.delete_structured_memory("k", "workstream", "ws1") + + +class TestList: + def test_list_all(self, backend): + backend.create_structured_memory("m1", "a", "", "project", "global", "", "1") + backend.create_structured_memory("m2", "b", "", "user", "global", "", "2") + mems = backend.list_structured_memories() + assert len(mems) == 2 + + def test_list_by_type(self, backend): + backend.create_structured_memory("m1", "a", "", "project", "global", "", "1") + backend.create_structured_memory("m2", "b", "", "user", "global", "", "2") + mems = backend.list_structured_memories(mem_type="user") + assert len(mems) == 1 + assert mems[0]["name"] == "b" + + def test_list_by_scope(self, backend): + backend.create_structured_memory("m1", "a", "", "project", "global", "", "1") + backend.create_structured_memory("m2", "b", "", "project", "workstream", "ws1", "2") + mems = backend.list_structured_memories(scope="workstream") + assert len(mems) == 1 + + def test_list_respects_limit(self, backend): + for i in range(10): + backend.create_structured_memory(f"m{i}", f"k{i}", "", "project", "global", "", f"{i}") + mems = backend.list_structured_memories(limit=3) + assert len(mems) == 3 + + +class TestSearch: + def test_search_by_name(self, backend): + backend.create_structured_memory("m1", "database_config", "", "project", "global", "", "pg") + backend.create_structured_memory("m2", "api_key", "", "project", "global", "", "secret") + results = backend.search_structured_memories("database") + assert len(results) == 1 + assert results[0]["name"] == "database_config" + + def test_search_by_content(self, backend): + backend.create_structured_memory("m1", "a", "", "project", "global", "", "postgresql host") + results = backend.search_structured_memories("postgresql") + assert len(results) == 1 + + def test_search_empty_lists_all(self, backend): + backend.create_structured_memory("m1", "a", "", "project", "global", "", "1") + backend.create_structured_memory("m2", "b", "", "project", "global", "", "2") + results = backend.search_structured_memories("") + assert len(results) == 2 + + +class TestCount: + def test_count_all(self, backend): + backend.create_structured_memory("m1", "a", "", "project", "global", "", "1") + backend.create_structured_memory("m2", "b", "", "project", "global", "", "2") + assert backend.count_structured_memories() == 2 + + def test_count_by_scope(self, backend): + backend.create_structured_memory("m1", "a", "", "project", "global", "", "1") + backend.create_structured_memory("m2", "b", "", "project", "workstream", "ws1", "2") + assert backend.count_structured_memories(scope="global") == 1 + assert backend.count_structured_memories(scope="workstream") == 1 diff --git a/tests/test_tools_schema.py b/tests/test_tools_schema.py index 03a03087..a1569dac 100644 --- a/tests/test_tools_schema.py +++ b/tests/test_tools_schema.py @@ -72,7 +72,7 @@ class TestToolsMetadata: """Validate the metadata extracted from JSON files.""" def test_tool_count(self): - assert len(TOOLS) == 18 + assert len(TOOLS) == 17 def test_agent_tools_count(self): assert len(AGENT_TOOLS) == 9 @@ -106,9 +106,8 @@ class TestToolsMetadata: "web_search": "query", "task": "prompt", "create_plan": "goal", - "remember": "key", + "memory": "name", "recall": "query", - "forget": "key", "notify": "message", "watch": "command", "read_resource": "uri", diff --git a/turnstone/chat.py b/turnstone/chat.py index 9907bf60..4e5aef10 100755 --- a/turnstone/chat.py +++ b/turnstone/chat.py @@ -7,7 +7,7 @@ All functionality has been moved to submodules: - turnstone.core.sandbox: validate_math_code, execute_math_sandboxed - turnstone.core.safety: is_command_blocked, sanitize_command - turnstone.core.web: strip_html, check_ssrf - - turnstone.core.memory: open_db, load_memories, save_message, etc. + - turnstone.core.memory: save_message, structured memory facade, etc. - turnstone.ui.colors: ANSI constants and helpers - turnstone.ui.markdown: MarkdownRenderer - turnstone.ui.spinner: Spinner diff --git a/turnstone/core/bm25.py b/turnstone/core/bm25.py new file mode 100644 index 00000000..74b2711a --- /dev/null +++ b/turnstone/core/bm25.py @@ -0,0 +1,62 @@ +"""BM25 index — lightweight, pure-Python, zero external deps. + +Extracted from tool_search.py for reuse by memory relevance scoring. +""" + +from __future__ import annotations + +import math +import re +from collections import Counter + +_SPLIT_RE = re.compile(r"[_\-./\s]+") + + +def _tokenize(text: str) -> list[str]: + """Split text on whitespace, underscores, hyphens, dots.""" + return [t.lower() for t in _SPLIT_RE.split(text) if t] + + +class BM25Index: + """Okapi BM25 ranking index over short text documents.""" + + def __init__(self, documents: list[str], *, k1: float = 1.5, b: float = 0.75) -> None: + self.k1 = k1 + self.b = b + self._docs = documents + self._doc_tokens: list[list[str]] = [_tokenize(d) for d in documents] + self._doc_lens = [len(t) for t in self._doc_tokens] + self._avgdl = sum(self._doc_lens) / max(len(self._doc_lens), 1) + self._n = len(documents) + # Document frequency per term + self._df: Counter[str] = Counter() + for tokens in self._doc_tokens: + for term in set(tokens): + self._df[term] += 1 + + def search(self, query: str, k: int = 5) -> list[int]: + """Return indices of top-k documents sorted by descending BM25 score.""" + q_tokens = _tokenize(query) + if not q_tokens: + return [] + scores: list[tuple[float, int]] = [] + for idx, doc_tokens in enumerate(self._doc_tokens): + score = self._score(q_tokens, doc_tokens, self._doc_lens[idx]) + if score > 0: + scores.append((score, idx)) + scores.sort(key=lambda x: (-x[0], x[1])) + return [idx for _, idx in scores[:k]] + + def _score(self, q_tokens: list[str], doc_tokens: list[str], dl: int) -> float: + tf_map: Counter[str] = Counter(doc_tokens) + score = 0.0 + for term in q_tokens: + if term not in tf_map: + continue + tf = tf_map[term] + df = self._df.get(term, 0) + idf = math.log((self._n - df + 0.5) / (df + 0.5) + 1.0) + numerator = tf * (self.k1 + 1) + denominator = tf + self.k1 * (1 - self.b + self.b * dl / self._avgdl) + score += idf * numerator / denominator + return score diff --git a/turnstone/core/memory.py b/turnstone/core/memory.py index a8576e80..227b72bd 100644 --- a/turnstone/core/memory.py +++ b/turnstone/core/memory.py @@ -10,6 +10,8 @@ from __future__ import annotations import contextlib from typing import TYPE_CHECKING, Any +import sqlalchemy as sa + from turnstone.core.storage import get_storage if TYPE_CHECKING: @@ -220,41 +222,6 @@ def update_workstream_title(ws_id: str, title: str) -> None: get_storage().update_workstream_title(ws_id, title) -# -- Key-value store (memories) ------------------------------------------------ - - -def save_memory(key: str, value: str) -> str | None: - """Save a memory. Returns the previous value if it existed.""" - try: - return get_storage().kv_set(key, value) - except Exception: - return None - - -def delete_memory(key: str) -> bool: - """Delete a memory by key. Returns True if the key existed.""" - try: - return get_storage().kv_delete(key) - except Exception: - return False - - -def load_memories() -> list[tuple[str, str]]: - """Return all (key, value) memory pairs sorted by key.""" - try: - return get_storage().kv_list() - except Exception: - return [] - - -def search_memories(query: str) -> list[tuple[str, str]]: - """Search memories by query. Returns matching (key, value) pairs.""" - try: - return get_storage().kv_search(query) - except Exception: - return [] - - # -- Conversation search ------------------------------------------------------- @@ -272,3 +239,96 @@ def search_history_recent(limit: int = 20) -> list[Any]: return get_storage().search_history_recent(limit) except Exception: return [] + + +# -- Structured memories ------------------------------------------------------- + + +def save_structured_memory( + name: str, + content: str, + description: str = "", + mem_type: str = "project", + scope: str = "global", + scope_id: str = "", +) -> tuple[str, str | None]: + """Save a structured memory (upsert by name+scope+scope_id). + + Returns (memory_id, old_content_or_None). Uses create-first to + avoid TOCTOU races under concurrent access. + """ + import uuid + + name = normalize_key(name) + try: + storage = get_storage() + # Try create first — if it hits the unique constraint, fall back to update + memory_id = str(uuid.uuid4()) + try: + storage.create_structured_memory( + memory_id, name, description, mem_type, scope, scope_id, content + ) + return memory_id, None + except sa.exc.IntegrityError: + # Unique constraint violation — row already exists, update it + existing = storage.get_structured_memory_by_name(name, scope, scope_id) + if existing: + old_content = existing["content"] + updates: dict[str, str] = {"content": content} + if description: + updates["description"] = description + if mem_type != "project": + updates["type"] = mem_type + storage.update_structured_memory(existing["memory_id"], **updates) + return existing["memory_id"], old_content + return "", None + except Exception: + return "", None + + +def delete_structured_memory(name: str, scope: str = "global", scope_id: str = "") -> bool: + """Delete a structured memory by name+scope. Returns True if existed.""" + name = normalize_key(name) + try: + return get_storage().delete_structured_memory(name, scope, scope_id) + except Exception: + return False + + +def list_structured_memories( + mem_type: str = "", + scope: str = "", + scope_id: str = "", + limit: int = 100, +) -> list[dict[str, str]]: + """List structured memories with optional filters.""" + try: + return get_storage().list_structured_memories( + mem_type=mem_type, scope=scope, scope_id=scope_id, limit=limit + ) + except Exception: + return [] + + +def search_structured_memories( + query: str, + mem_type: str = "", + scope: str = "", + scope_id: str = "", + limit: int = 20, +) -> list[dict[str, str]]: + """Search structured memories by query.""" + try: + return get_storage().search_structured_memories( + query, mem_type=mem_type, scope=scope, scope_id=scope_id, limit=limit + ) + except Exception: + return [] + + +def count_structured_memories(scope: str = "", scope_id: str = "") -> int: + """Count structured memories with optional scope filter.""" + try: + return get_storage().count_structured_memories(scope=scope, scope_id=scope_id) + except Exception: + return 0 diff --git a/turnstone/core/memory_relevance.py b/turnstone/core/memory_relevance.py new file mode 100644 index 00000000..1089db79 --- /dev/null +++ b/turnstone/core/memory_relevance.py @@ -0,0 +1,85 @@ +"""BM25-based memory relevance scoring and system message formatting.""" + +from __future__ import annotations + +from html import escape as _html_escape +from typing import Any + +from turnstone.core.bm25 import BM25Index + + +def score_memories( + memories: list[dict[str, str]], + query: str, + k: int = 5, +) -> list[dict[str, str]]: + """Return the top-k memories most relevant to *query*. + + Builds a BM25 index over ``name + description + content prefix`` + for each memory and returns matches sorted by relevance. If *query* + is empty, returns the most recent *k* memories (they are already + ordered by ``updated DESC`` from storage). + """ + if not memories: + return [] + if not query or not query.strip(): + return memories[:k] + + documents = [ + f"{m.get('name', '')} {m.get('description', '')} {m.get('content', '')[:200]}" + for m in memories + ] + index = BM25Index(documents) + top_indices = index.search(query, k) + return [memories[i] for i in top_indices] + + +def build_memory_context(memories: list[dict[str, str]]) -> str: + """Format selected memories as an XML block for system message injection. + + Produces a compact ```` section matching the style used + for MCP resources (````). + """ + if not memories: + return "" + lines = [""] + for m in memories: + name = _html_escape(m.get("name", "")) + mem_type = _html_escape(m.get("type", "project")) + scope = _html_escape(m.get("scope", "global")) + desc = m.get("description", "") + content = m.get("content", "") + # Truncate content to avoid bloating system message + if len(content) > 500: + content = content[:500] + "..." + desc_attr = f' description="{_html_escape(desc)}"' if desc else "" + lines.append( + f' ' + f"{_html_escape(content)}" + ) + lines.append("") + return "\n".join(lines) + + +def extract_recent_context(messages: list[dict[str, Any]], max_messages: int = 3) -> str: + """Extract text from the last N user messages for relevance scoring. + + Handles both string and list content formats. + """ + user_texts: list[str] = [] + for msg in reversed(messages): + if msg.get("role") != "user": + continue + content = msg.get("content", "") + if isinstance(content, str): + user_texts.append(content) + elif isinstance(content, list): + # Multi-part content (text + images) + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + user_texts.append(part.get("text", "")) + elif isinstance(part, str): + user_texts.append(part) + if len(user_texts) >= max_messages: + break + return " ".join(user_texts) diff --git a/turnstone/core/metacognition.py b/turnstone/core/metacognition.py new file mode 100644 index 00000000..60b4946e --- /dev/null +++ b/turnstone/core/metacognition.py @@ -0,0 +1,130 @@ +"""Metacognitive prompting — situational nudges for proactive memory use.""" + +from __future__ import annotations + +import re +import time + +_COOLDOWN_SECS = 300 # 5 minutes between nudges of the same type + +# --------------------------------------------------------------------------- +# Nudge messages (brief, model-facing hints) +# --------------------------------------------------------------------------- + +NUDGE_CORRECTION = ( + "Note: The user's message may contain a correction or preference. " + "Pay close attention — if they explain what went wrong or how they'd " + "prefer you to work, consider saving that as a feedback memory " + "(memory action='save', type='feedback') so you don't repeat this." +) + +NUDGE_DENIAL = ( + "Note: The user just rejected a tool action. Their feedback may " + "explain why — pay attention to whether this reflects a persistent " + "preference (e.g. 'never use force-push', 'don't modify that file'). " + "If so, save it as a feedback memory for future sessions." +) + +NUDGE_RESUME = ( + "This workstream has prior conversation history. Before proceeding, " + "use memory(action='search') to check for relevant context — there " + "may be saved preferences, project notes, or prior decisions that " + "apply to this work." +) + +NUDGE_COMPLETION = ( + "The task may be wrapping up. Consider whether there are learnings, " + "decisions, or user preferences from this session worth persisting " + "as memories (memory action='save') so future sessions can benefit." +) + +NUDGE_START = ( + "You have saved memories from prior sessions that may be relevant. " + "Consider using memory(action='search') with keywords from the " + "user's request to find applicable context, preferences, or guidance." +) + +_NUDGE_MAP: dict[str, str] = { + "correction": NUDGE_CORRECTION, + "denial": NUDGE_DENIAL, + "resume": NUDGE_RESUME, + "completion": NUDGE_COMPLETION, + "start": NUDGE_START, +} + +# --------------------------------------------------------------------------- +# Detection heuristics +# --------------------------------------------------------------------------- + +_CORRECTION_PATTERNS: list[re.Pattern[str]] = [ + re.compile(r"(?i)^no[,.\s]"), + re.compile(r"(?i)\bdon'?t\b"), + re.compile(r"(?i)^stop\b"), + re.compile(r"(?i)^actually[,\s]"), + re.compile(r"(?i)^instead[,\s]"), + re.compile(r"(?i)\bnot like that\b"), + re.compile(r"(?i)^wrong\b"), + re.compile(r"(?i)\bthat'?s not\b"), + re.compile(r"(?i)^I said\b"), + re.compile(r"(?i)^I meant\b"), + re.compile(r"(?i)\bnever\b.*\balways\b"), + re.compile(r"(?i)^please don'?t\b"), +] + +_COMPLETION_PATTERNS: list[re.Pattern[str]] = [ + re.compile(r"(?i)^thanks\b"), + re.compile(r"(?i)\bthat'?s all\b"), + re.compile(r"(?i)\blooks good\b"), + re.compile(r"(?i)^perfect\b"), + re.compile(r"(?i)^great job\b"), + re.compile(r"(?i)\bthat works\b"), + re.compile(r"(?i)^done\b"), + re.compile(r"(?i)^lgtm\b"), +] + + +def detect_correction(message: str) -> bool: + """Return True if the message looks like a user correction.""" + if not message: + return False + return any(p.search(message) for p in _CORRECTION_PATTERNS) + + +def detect_completion(message: str) -> bool: + """Return True if the message signals session completion.""" + if not message: + return False + return any(p.search(message) for p in _COMPLETION_PATTERNS) + + +def should_nudge( + nudge_type: str, + state: dict[str, float], + *, + message_count: int = 0, + memory_count: int = 0, +) -> bool: + """Check whether a nudge should fire, respecting cooldowns and context.""" + if nudge_type not in _NUDGE_MAP: + return False + # Don't nudge on the very first message (except resume/start) + if message_count <= 1 and nudge_type not in ("resume", "start"): + return False + # Start nudge only on first message + if nudge_type == "start" and message_count != 1: + return False + # Resume/start nudge only if there are memories to recall + if nudge_type in ("resume", "start") and memory_count == 0: + return False + # Rate limit: one nudge per type per cooldown window + now = time.monotonic() + last = state.get(nudge_type) + if last is not None and now - last < _COOLDOWN_SECS: + return False + state[nudge_type] = now + return True + + +def format_nudge(nudge_type: str) -> str: + """Return the nudge text for the given type.""" + return _NUDGE_MAP.get(nudge_type, "") diff --git a/turnstone/core/session.py b/turnstone/core/session.py index 769caa72..914fa086 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -33,26 +33,38 @@ from turnstone.core.config import get_tavily_key from turnstone.core.edit import find_occurrences, pick_nearest from turnstone.core.log import get_logger from turnstone.core.memory import ( - delete_memory, + count_structured_memories, + delete_structured_memory, delete_workstream, get_prompt_template_by_name, get_workstream_display_name, list_default_templates, + list_structured_memories, list_workstreams_with_history, - load_memories, load_messages, load_workstream_config, normalize_key, resolve_workstream, - save_memory, save_message, + save_structured_memory, save_workstream_config, search_history, search_history_recent, - search_memories, + search_structured_memories, set_workstream_alias, update_workstream_title, ) +from turnstone.core.memory_relevance import ( + build_memory_context, + extract_recent_context, + score_memories, +) +from turnstone.core.metacognition import ( + detect_completion, + detect_correction, + format_nudge, + should_nudge, +) from turnstone.core.providers import create_provider from turnstone.core.safety import is_command_blocked, sanitize_command from turnstone.core.sandbox import execute_math_sandboxed @@ -110,6 +122,7 @@ _IMAGE_SIZE_CAP: int = 4 * 1024 * 1024 # Upper bound on total prompt template content injected into system messages _MAX_TEMPLATE_CONTENT: int = 32768 +_MAX_MEMORY_CONTENT: int = 32768 _TEMPLATE_VAR_RE = re.compile(r"\{\{(\w+)\}\}") @@ -222,6 +235,7 @@ class ChatSession: tool_search_max_results: int = 5, template: str | None = None, judge_config: JudgeConfig | None = None, + user_id: str = "", ): self.client = client self.model = model @@ -255,6 +269,7 @@ class ChatSession: self.debug = False self.auto_approve = False self._node_id = node_id + self._user_id = user_id self._ws_id = ws_id or uuid.uuid4().hex self._title_generated = False self._read_files: set[str] = set() @@ -277,6 +292,9 @@ class ChatSession: self._watch_runner: Any = None # WatchRunner | None self._watch_pending: queue.Queue[dict[str, Any]] = queue.Queue() self._watch_dispatch_depth = 0 + # Metacognitive nudges: ephemeral prompts for proactive memory use + self._metacog_state: dict[str, float] = {} + self._pending_nudge: str | None = None # Cooperative cancellation: set from outside to stop generation self._cancel_event = threading.Event() self._cancelled_partial_msg: dict[str, Any] | None = None @@ -639,7 +657,14 @@ class ChatSession: self._template_name = None if "notify_on_complete" in config: self._notify_on_complete = config["notify_on_complete"] - self._init_system_messages() + if should_nudge( + "resume", + self._metacog_state, + message_count=len(self.messages), + memory_count=self._visible_memory_count(), + ): + self._pending_nudge = format_nudge("resume") + self._init_system_messages() return True def _init_system_messages(self) -> None: @@ -766,13 +791,22 @@ class ChatSession: if self.instructions: dev_parts.append("") dev_parts.append(self.instructions) - memories = load_memories() - if memories: + visible_mems = self._get_visible_memories(limit=50) + if visible_mems: + context = extract_recent_context(self.messages) + relevant = score_memories(visible_mems, context, k=5) + if relevant: + dev_parts.append("") + dev_parts.append(build_memory_context(relevant)) dev_parts.append("") dev_parts.append( - f"REMINDER: You currently have {len(memories)} memories stored. " - "Use recall to see them." + f"You have {len(visible_mems)} memories in scope. " + "Use memory(action='search') or memory(action='list') for more." ) + if self._pending_nudge: + dev_parts.append("") + dev_parts.append(self._pending_nudge) + self._pending_nudge = None new_system_messages.append({"role": "system", "content": "\n".join(dev_parts)}) # Atomic swap — readers see either old or new, never partial self.system_messages = new_system_messages @@ -957,6 +991,12 @@ class ChatSession: self._msg_tokens.append(max(1, int(len(user_input) / self._chars_per_token))) save_message(self._ws_id, "user", user_input) + # Metacognitive nudge: check for correction/completion signals + nudge = self._check_metacognitive_nudge(user_input) + if nudge: + self._pending_nudge = nudge + self._init_system_messages() + try: while True: self._check_cancelled() @@ -996,8 +1036,7 @@ class ChatSession: filtered_tc = [ call for call in tc - if call.get("function", {}).get("name", "") - not in ("remember", "forget", "recall") + if call.get("function", {}).get("name", "") not in ("memory", "recall") ] if filtered_tc: tool_calls_json = json.dumps(filtered_tc) @@ -1066,8 +1105,7 @@ class ChatSession: # Log tool result (skip memory tools to avoid noise) _tname = _tc_names.get(tc_id, "") if _tname not in ( - "remember", - "forget", + "memory", "recall", ): # For image content, store text description only @@ -1626,7 +1664,11 @@ class ChatSession: " - **## Open tasks**: What the user asked for that is not yet done, " "with enough context to continue.\n" " - **## User preferences**: Workflow preferences, constraints, or " - "instructions the user stated.\n\n" + "instructions the user stated.\n" + " - **## Memories to save**: Corrections, preferences, or learnings " + "the user expressed that should be persisted across sessions. " + "Format each as: `name: description — content`. " + "Only include items the user explicitly stated, not inferences.\n\n" "2. **Density rules:**\n" " - Every token should carry information.\n" " - Preserve exact paths, identifiers, and numbers — never paraphrase these.\n" @@ -1821,6 +1863,14 @@ class ChatSession: f"Denied by user: {user_feedback}" if user_feedback else "Denied by user" ) user_feedback = None # feedback is in the denial_msg + if should_nudge( + "denial", + self._metacog_state, + message_count=len(self.messages), + memory_count=self._visible_memory_count(), + ): + self._pending_nudge = format_nudge("denial") + self._init_system_messages() # Phase 3: execute (check cancellation before starting) self._check_cancelled() @@ -1972,9 +2022,8 @@ class ChatSession: "tool_search": self._prepare_tool_search, "task": self._prepare_task, "create_plan": self._prepare_plan, - "remember": self._prepare_remember, + "memory": self._prepare_memory, "recall": self._prepare_recall, - "forget": self._prepare_forget, "notify": self._prepare_notify, "watch": self._prepare_watch, "read_resource": self._prepare_read_resource, @@ -2545,55 +2594,232 @@ class ChatSession: "prompt": goal, } - def _prepare_remember(self, call_id: str, args: dict[str, Any]) -> dict[str, Any]: - """Prepare a remember (save memory) action.""" - key = normalize_key((args.get("key") or "").strip()) - value = (args.get("value") or "").strip() - if not key or not value: - return { - "call_id": call_id, - "func_name": "remember", - "header": "\u2717 remember: requires key and value", - "preview": "", - "needs_approval": False, - "error": "Error: both 'key' and 'value' are required", - } - return { - "call_id": call_id, - "func_name": "remember", - "header": f"\u2699 remember: {key}", - "preview": "", - "needs_approval": False, - "execute": self._exec_remember, - "key": key, - "value": value, - } + def _resolve_scope_id(self, scope: str) -> str: + """Map a scope name to its scope_id.""" + if scope == "workstream": + return self._ws_id + if scope == "user": + return self._user_id + return "" - def _prepare_forget(self, call_id: str, args: dict[str, Any]) -> dict[str, Any]: - """Prepare a forget (delete memory) action.""" - key = normalize_key((args.get("key") or "").strip()) - if not key: + def _validate_scope(self, scope: str, call_id: str) -> dict[str, Any] | None: + """Return an error dict if scope is invalid, None if OK.""" + if scope == "user" and not self._user_id: return { "call_id": call_id, - "func_name": "forget", - "header": "\u2717 forget: empty key", + "func_name": "memory", + "header": "\u2717 memory: user scope requires authentication", "preview": "", "needs_approval": False, - "error": "Error: key is required", + "error": "Error: 'user' scope requires authenticated user identity", } + return None + + def _get_visible_memories(self, limit: int = 50) -> list[dict[str, str]]: + """Return memories visible to this session (scope-filtered).""" + global_mems = list_structured_memories(scope="global", limit=limit) + ws_mems = list_structured_memories(scope="workstream", scope_id=self._ws_id, limit=limit) + user_mems: list[dict[str, str]] = [] + if self._user_id: + user_mems = list_structured_memories(scope="user", scope_id=self._user_id, limit=limit) + combined = global_mems + ws_mems + user_mems + combined.sort(key=lambda m: m.get("updated", ""), reverse=True) + return combined[:limit] + + def _visible_memory_count(self) -> int: + """Count memories visible to this session (cheap — counts only).""" + n = count_structured_memories(scope="global") + n += count_structured_memories(scope="workstream", scope_id=self._ws_id) + if self._user_id: + n += count_structured_memories(scope="user", scope_id=self._user_id) + return n + + def _check_metacognitive_nudge(self, user_message: str) -> str | None: + """Check if a metacognitive nudge should be injected.""" + mem_count = self._visible_memory_count() + msg_count = len(self.messages) + + # First message in a new workstream — nudge to check existing memories + if should_nudge( + "start", self._metacog_state, message_count=msg_count, memory_count=mem_count + ): + return format_nudge("start") + + if detect_correction(user_message) and should_nudge( + "correction", self._metacog_state, message_count=msg_count, memory_count=mem_count + ): + return format_nudge("correction") + + if detect_completion(user_message) and should_nudge( + "completion", self._metacog_state, message_count=msg_count, memory_count=mem_count + ): + return format_nudge("completion") + + return None + + def _prepare_memory(self, call_id: str, args: dict[str, Any]) -> dict[str, Any]: + """Prepare a memory tool action (save/search/delete/list).""" + action = (args.get("action") or "").strip().lower() + + if action == "save": + name = (args.get("name") or args.get("key") or "").strip() + content = (args.get("content") or args.get("value") or "").strip() + name = normalize_key(name) + if not name or not content: + return { + "call_id": call_id, + "func_name": "memory", + "header": "\u2717 memory save: requires name and content", + "preview": "", + "needs_approval": False, + "error": "Error: both 'name' and 'content' are required for save", + } + if len(content) > _MAX_MEMORY_CONTENT: + return { + "call_id": call_id, + "func_name": "memory", + "header": "\u2717 memory save: content too large", + "preview": "", + "needs_approval": False, + "error": f"Error: content exceeds {_MAX_MEMORY_CONTENT} byte limit", + } + description = (args.get("description") or "").strip() + mem_type = (args.get("type") or "project").strip().lower() + if mem_type not in ("user", "project", "feedback", "reference"): + mem_type = "project" + scope = (args.get("scope") or "global").strip().lower() + if scope not in ("global", "workstream", "user"): + scope = "global" + scope_err = self._validate_scope(scope, call_id) + if scope_err: + return scope_err + scope_id = self._resolve_scope_id(scope) + return { + "call_id": call_id, + "func_name": "memory", + "header": f"\u2699 memory save: {name}", + "preview": "", + "needs_approval": False, + "execute": self._exec_memory, + "action": "save", + "name": name, + "content": content, + "description": description, + "mem_type": mem_type, + "scope": scope, + "scope_id": scope_id, + } + + if action == "delete": + name = normalize_key((args.get("name") or args.get("key") or "").strip()) + if not name: + return { + "call_id": call_id, + "func_name": "memory", + "header": "\u2717 memory delete: empty name", + "preview": "", + "needs_approval": False, + "error": "Error: name is required for delete", + } + scope = (args.get("scope") or "global").strip().lower() + if scope not in ("global", "workstream", "user"): + scope = "global" + scope_err = self._validate_scope(scope, call_id) + if scope_err: + return scope_err + scope_id = self._resolve_scope_id(scope) + return { + "call_id": call_id, + "func_name": "memory", + "header": f"\u2699 memory delete: {name}", + "preview": "", + "needs_approval": False, + "execute": self._exec_memory, + "action": "delete", + "name": name, + "scope": scope, + "scope_id": scope_id, + } + + if action == "search": + query = (args.get("query") or "").strip() + mem_type = (args.get("type") or "").strip().lower() + if mem_type and mem_type not in ("user", "project", "feedback", "reference"): + mem_type = "" + scope = (args.get("scope") or "").strip().lower() + if scope and scope not in ("global", "workstream", "user"): + scope = "" + scope_id = self._resolve_scope_id(scope) if scope else "" + limit = args.get("limit", 20) + if isinstance(limit, str): + try: + limit = int(limit) + except ValueError: + limit = 20 + return { + "call_id": call_id, + "func_name": "memory", + "header": f"\u2699 memory search{': ' + query[:80] if query else ''}", + "preview": "", + "needs_approval": False, + "execute": self._exec_memory, + "action": "search", + "query": query, + "mem_type": mem_type, + "scope": scope, + "scope_id": scope_id, + "limit": max(1, min(limit, 50)), + } + + if action == "list": + mem_type = (args.get("type") or "").strip().lower() + if mem_type and mem_type not in ("user", "project", "feedback", "reference"): + mem_type = "" + scope = (args.get("scope") or "").strip().lower() + if scope and scope not in ("global", "workstream", "user"): + scope = "" + scope_id = self._resolve_scope_id(scope) if scope else "" + limit = args.get("limit", 20) + if isinstance(limit, str): + try: + limit = int(limit) + except ValueError: + limit = 20 + return { + "call_id": call_id, + "func_name": "memory", + "header": "\u2699 memory list", + "preview": "", + "needs_approval": False, + "execute": self._exec_memory, + "action": "list", + "mem_type": mem_type, + "scope": scope, + "scope_id": scope_id, + "limit": max(1, min(limit, 50)), + } + return { "call_id": call_id, - "func_name": "forget", - "header": f"\u2699 forget: {key}", + "func_name": "memory", + "header": "\u2717 memory: invalid action", "preview": "", "needs_approval": False, - "execute": self._exec_forget, - "key": key, + "error": f"Error: action must be save/search/delete/list, got '{action}'", } def _prepare_recall(self, call_id: str, args: dict[str, Any]) -> dict[str, Any]: - """Prepare a recall action.""" + """Prepare a conversation history search.""" query = (args.get("query") or "").strip() + if not query: + return { + "call_id": call_id, + "func_name": "recall", + "header": "\u2717 recall: requires query", + "preview": "", + "needs_approval": False, + "error": "Error: query is required", + } limit = args.get("limit", 20) if isinstance(limit, str): try: @@ -2603,12 +2829,12 @@ class ChatSession: return { "call_id": call_id, "func_name": "recall", - "header": f"\u2699 recall{': ' + query[:80] if query else ''}", + "header": f"\u2699 recall: {query[:80]}", "preview": "", "needs_approval": False, "execute": self._exec_recall, "query": query, - "limit": min(limit, 50), + "limit": max(1, min(limit, 50)), } # -- MCP tool prepare/execute ---------------------------------------------- @@ -3500,66 +3726,113 @@ class ChatSession: return content - def _exec_remember(self, item: dict[str, Any]) -> tuple[str, str]: - """Save a persistent memory.""" - call_id, key, value = item["call_id"], item["key"], item["value"] + def _exec_memory(self, item: dict[str, Any]) -> tuple[str, str]: + """Execute a memory tool action.""" + call_id = item["call_id"] + action = item["action"] + try: - old_value = save_memory(key, value) - self._init_system_messages() - if old_value is not None: - msg = f"Updated memory: {key} = {value} (was: {old_value})" - else: - msg = f"Saved memory: {key} = {value}" - self.ui.on_tool_result(call_id, "remember", msg) - return call_id, msg + if action == "save": + memory_id, old = save_structured_memory( + item["name"], + item["content"], + description=item["description"], + mem_type=item["mem_type"], + scope=item["scope"], + scope_id=item["scope_id"], + ) + if not memory_id: + msg = f"Error: failed to save memory '{item['name']}'" + self.ui.on_tool_result(call_id, "memory", msg) + return call_id, msg + self._init_system_messages() + if old is not None: + msg = f"Updated memory '{item['name']}' (type={item['mem_type']}, scope={item['scope']})" + else: + msg = f"Saved memory '{item['name']}' (type={item['mem_type']}, scope={item['scope']})" + self.ui.on_tool_result(call_id, "memory", msg) + return call_id, msg + + if action == "delete": + deleted = delete_structured_memory(item["name"], item["scope"], item["scope_id"]) + if not deleted: + msg = f"Error: memory '{item['name']}' not found (scope={item['scope']})" + else: + self._init_system_messages() + msg = f"Deleted memory '{item['name']}'" + self.ui.on_tool_result(call_id, "memory", msg) + return call_id, msg + + if action == "search": + rows = search_structured_memories( + item["query"], + mem_type=item.get("mem_type", ""), + scope=item.get("scope", ""), + scope_id=item.get("scope_id", ""), + limit=item["limit"], + ) + if rows: + lines = [] + for m in rows: + desc = f" — {m['description']}" if m.get("description") else "" + lines.append( + f" [{m['type']}:{m['scope']}] {m['name']}{desc}\n" + f" {m['content'][:500]}" + ) + msg = f"Memories ({len(rows)} results):\n" + "\n".join(lines) + else: + msg = ( + f"No memories found for '{item['query']}'." + if item["query"] + else "No memories stored." + ) + self.ui.on_tool_result(call_id, "memory", msg) + return call_id, msg + + if action == "list": + rows = list_structured_memories( + mem_type=item.get("mem_type", ""), + scope=item.get("scope", ""), + scope_id=item.get("scope_id", ""), + limit=item["limit"], + ) + if rows: + lines = [] + for m in rows: + desc = f" — {m['description']}" if m.get("description") else "" + lines.append( + f" [{m['type']}:{m['scope']}] {m['name']}{desc}\n" + f" {m['content'][:500]}" + ) + msg = f"Memories ({len(rows)}):\n" + "\n".join(lines) + else: + msg = "No memories stored." + self.ui.on_tool_result(call_id, "memory", msg) + return call_id, msg + except Exception as e: return call_id, f"Error: {e}" - def _exec_forget(self, item: dict[str, Any]) -> tuple[str, str]: - """Remove a persistent memory by key.""" - call_id, key = item["call_id"], item["key"] - try: - deleted = delete_memory(key) - if not deleted: - msg = f"Error: memory '{key}' not found" - else: - self._init_system_messages() - msg = f"Forgot: {key}" - self.ui.on_tool_result(call_id, "forget", msg) - return call_id, msg - except Exception as e: - return call_id, f"Error: {e}" + return call_id, "Error: unexpected action" def _exec_recall(self, item: dict[str, Any]) -> tuple[str, str]: - """Search memories and conversation history.""" + """Search conversation history.""" call_id = item["call_id"] query, limit = item["query"], item["limit"] - parts: list[str] = [] - # Memories: list all (no query) or search (with query) - try: - rows = search_memories(query) if query else load_memories() - if rows: - parts.append("Memories:\n" + "\n".join(f" {k}={v}" for k, v in rows)) - elif not query: - parts.append("No memories stored.") - except Exception: - pass + conv_rows = search_history(query, limit) + if conv_rows: + lines = [] + for ts, sid, role, content, tool_name in conv_rows: + label = f"{role}({tool_name})" if tool_name else role + text = (content or "")[:500] + if content and len(content) > 500: + text += "..." + lines.append(f"[{ts} {sid}] {label}: {text}") + output = f"Conversations ({len(conv_rows)} matches):\n" + "\n".join(lines) + else: + output = f"No conversation history found for '{query}'." - # Conversations: only when a query is provided - if query: - conv_rows = search_history(query, limit) - if conv_rows: - lines = [] - for ts, sid, role, content, tool_name in conv_rows: - label = f"{role}({tool_name})" if tool_name else role - text = (content or "")[:500] - if content and len(content) > 500: - text += "..." - lines.append(f"[{ts} {sid}] {label}: {text}") - parts.append(f"Conversations ({len(conv_rows)} matches):\n" + "\n".join(lines)) - - output = "\n\n".join(parts) if parts else f"No results for '{query}'." self.ui.on_tool_result(call_id, "recall", output) return call_id, output diff --git a/turnstone/core/storage/_postgresql.py b/turnstone/core/storage/_postgresql.py index d81a505c..c2aa2975 100644 --- a/turnstone/core/storage/_postgresql.py +++ b/turnstone/core/storage/_postgresql.py @@ -14,11 +14,11 @@ from turnstone.core.storage._schema import ( audit_events, conversations, intent_verdicts, - memories, metadata, orgs, prompt_templates, roles, + structured_memories, tool_policies, usage_events, user_roles, @@ -37,6 +37,9 @@ from turnstone.core.storage._utils import ( from turnstone.core.storage._utils import ( ROLE_MUTABLE as _ROLE_MUTABLE, ) +from turnstone.core.storage._utils import ( + STRUCTURED_MEMORY_MUTABLE as _SMEM_MUTABLE, +) from turnstone.core.storage._utils import ( TEMPLATE_MUTABLE as _TEMPLATE_MUTABLE, ) @@ -56,6 +59,11 @@ from turnstone.core.storage._utils import ( log = logging.getLogger(__name__) +def _escape_ilike(s: str) -> str: + """Escape ILIKE metacharacters for use with ESCAPE '\\\\'.""" + return s.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + class PostgreSQLBackend: """PostgreSQL implementation of the StorageBackend protocol.""" @@ -274,64 +282,6 @@ class PostgreSQLBackend: ) conn.commit() - # -- Generic key-value store ----------------------------------------------- - - def kv_get(self, key: str) -> str | None: - with self._engine.connect() as conn: - row = conn.execute(sa.select(memories.c.value).where(memories.c.key == key)).fetchone() - return str(row[0]) if row else None - - def kv_set(self, key: str, value: str) -> str | None: - now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") - with self._engine.connect() as conn: - existing = conn.execute( - sa.select(memories.c.value, memories.c.created).where(memories.c.key == key) - ).fetchone() - old_value = str(existing[0]) if existing else None - created = str(existing[1]) if existing else now - # Delete + insert for cross-dialect upsert - conn.execute(sa.delete(memories).where(memories.c.key == key)) - conn.execute( - sa.insert(memories), - {"key": key, "value": value, "created": created, "updated": now}, - ) - conn.commit() - return old_value - - def kv_delete(self, key: str) -> bool: - with self._engine.connect() as conn: - result = conn.execute(sa.delete(memories).where(memories.c.key == key)) - conn.commit() - return result.rowcount > 0 - - def kv_list(self) -> list[tuple[str, str]]: - with self._engine.connect() as conn: - rows = conn.execute( - sa.select(memories.c.key, memories.c.value).order_by(memories.c.key) - ).fetchall() - return [(str(r[0]), str(r[1])) for r in rows] - - def kv_search(self, query: str) -> list[tuple[str, str]]: - if not query or not query.strip(): - return self.kv_list() - terms = query.split() - with self._engine.connect() as conn: - clauses = [] - params: dict[str, str] = {} - for i, t in enumerate(terms): - clauses.append(f"(key ILIKE :k{i} OR value ILIKE :v{i})") - params[f"k{i}"] = f"%{t}%" - params[f"v{i}"] = f"%{t}%" - rows = conn.execute( - sa.text( - "SELECT key, value FROM memories WHERE " - + " AND ".join(clauses) - + " ORDER BY key" - ), - params, - ).fetchall() - return [(str(r[0]), str(r[1])) for r in rows] - # -- Workstream operations ------------------------------------------------- def register_workstream( @@ -2144,6 +2094,166 @@ class PostgreSQLBackend: row = conn.execute(q).fetchone() return row[0] if row else 0 + # -- Structured memories --------------------------------------------------- + + def create_structured_memory( + self, + memory_id: str, + name: str, + description: str, + mem_type: str, + scope: str, + scope_id: str, + content: str, + ) -> None: + now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + with self._engine.connect() as conn: + conn.execute( + sa.insert(structured_memories), + { + "memory_id": memory_id, + "name": name, + "description": description, + "type": mem_type, + "scope": scope, + "scope_id": scope_id, + "content": content, + "created": now, + "updated": now, + "last_accessed": now, + "access_count": 0, + }, + ) + conn.commit() + + def get_structured_memory(self, memory_id: str) -> dict[str, str] | None: + with self._engine.connect() as conn: + row = conn.execute( + sa.select(structured_memories).where(structured_memories.c.memory_id == memory_id) + ).fetchone() + return dict(row._mapping) if row else None + + def get_structured_memory_by_name( + self, name: str, scope: str = "global", scope_id: str = "" + ) -> dict[str, str] | None: + with self._engine.connect() as conn: + row = conn.execute( + sa.select(structured_memories).where( + sa.and_( + structured_memories.c.name == name, + structured_memories.c.scope == scope, + structured_memories.c.scope_id == scope_id, + ) + ) + ).fetchone() + return dict(row._mapping) if row else None + + def update_structured_memory(self, memory_id: str, **fields: str) -> bool: + fields = {k: v for k, v in fields.items() if k in _SMEM_MUTABLE} + if not fields: + return False + now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + fields["updated"] = now + fields["last_accessed"] = now + with self._engine.connect() as conn: + result = conn.execute( + sa.update(structured_memories) + .where(structured_memories.c.memory_id == memory_id) + .values(**fields) + ) + conn.commit() + return result.rowcount > 0 + + def delete_structured_memory( + self, name: str, scope: str = "global", scope_id: str = "" + ) -> bool: + with self._engine.connect() as conn: + result = conn.execute( + sa.delete(structured_memories).where( + sa.and_( + structured_memories.c.name == name, + structured_memories.c.scope == scope, + structured_memories.c.scope_id == scope_id, + ) + ) + ) + conn.commit() + return result.rowcount > 0 + + def list_structured_memories( + self, + mem_type: str = "", + scope: str = "", + scope_id: str = "", + limit: int = 100, + ) -> list[dict[str, str]]: + with self._engine.connect() as conn: + q = sa.select(structured_memories).order_by(structured_memories.c.updated.desc()) + if mem_type: + q = q.where(structured_memories.c.type == mem_type) + if scope: + q = q.where(structured_memories.c.scope == scope) + if scope_id: + q = q.where(structured_memories.c.scope_id == scope_id) + q = q.limit(limit) + rows = conn.execute(q).fetchall() + return [dict(r._mapping) for r in rows] + + def search_structured_memories( + self, + query: str, + mem_type: str = "", + scope: str = "", + scope_id: str = "", + limit: int = 20, + ) -> list[dict[str, str]]: + if not query or not query.strip(): + return self.list_structured_memories( + mem_type=mem_type, scope=scope, scope_id=scope_id, limit=limit + ) + terms = query.split() + with self._engine.connect() as conn: + clauses = [] + params: dict[str, str] = {} + for i, t in enumerate(terms): + escaped = _escape_ilike(t) + clauses.append( + f"(name ILIKE :n{i} ESCAPE '\\' " + f"OR description ILIKE :d{i} ESCAPE '\\' " + f"OR content ILIKE :c{i} ESCAPE '\\')" + ) + params[f"n{i}"] = f"%{escaped}%" + params[f"d{i}"] = f"%{escaped}%" + params[f"c{i}"] = f"%{escaped}%" + where = " AND ".join(clauses) + if mem_type: + where += " AND type = :type_filter" + params["type_filter"] = mem_type + if scope: + where += " AND scope = :scope_filter" + params["scope_filter"] = scope + if scope_id: + where += " AND scope_id = :scope_id_filter" + params["scope_id_filter"] = scope_id + rows = conn.execute( + sa.text( + f"SELECT * FROM structured_memories WHERE {where} " + f"ORDER BY updated DESC LIMIT :lim" + ), + {**params, "lim": limit}, + ).fetchall() + return [dict(r._mapping) for r in rows] + + def count_structured_memories(self, scope: str = "", scope_id: str = "") -> int: + with self._engine.connect() as conn: + q = sa.select(sa.func.count()).select_from(structured_memories) + if scope: + q = q.where(structured_memories.c.scope == scope) + if scope_id: + q = q.where(structured_memories.c.scope_id == scope_id) + result = conn.execute(q).scalar() + return int(result or 0) + # -- Lifecycle ------------------------------------------------------------- def close(self) -> None: diff --git a/turnstone/core/storage/_protocol.py b/turnstone/core/storage/_protocol.py index 28d3aa33..0ceb114c 100644 --- a/turnstone/core/storage/_protocol.py +++ b/turnstone/core/storage/_protocol.py @@ -9,8 +9,8 @@ from typing import Any, Protocol, runtime_checkable class StorageBackend(Protocol): """Protocol that every storage backend adapter must implement. - Provides workstream management, conversation persistence, key-value storage - (for memories), and full-text search. + Provides workstream management, conversation persistence, structured + memories, and full-text search. """ # -- Core conversation operations ------------------------------------------ @@ -71,26 +71,64 @@ class StorageBackend(Protocol): """Set or update the auto-generated title for a workstream.""" ... - # -- Generic key-value store (backs memories table) ------------------------ + # -- Structured memories --------------------------------------------------- - def kv_get(self, key: str) -> str | None: - """Get a value by key. Returns None if not found.""" + def create_structured_memory( + self, + memory_id: str, + name: str, + description: str, + mem_type: str, + scope: str, + scope_id: str, + content: str, + ) -> None: + """Create a structured memory record.""" ... - def kv_set(self, key: str, value: str) -> str | None: - """Set a key-value pair. Returns the previous value if it existed.""" + def get_structured_memory(self, memory_id: str) -> dict[str, str] | None: + """Return structured memory dict or None.""" ... - def kv_delete(self, key: str) -> bool: - """Delete a key. Returns True if the key existed.""" + def get_structured_memory_by_name( + self, name: str, scope: str = "global", scope_id: str = "" + ) -> dict[str, str] | None: + """Lookup structured memory by (name, scope, scope_id). Returns dict or None.""" ... - def kv_list(self) -> list[tuple[str, str]]: - """Return all (key, value) pairs sorted by key.""" + def update_structured_memory(self, memory_id: str, **fields: str) -> bool: + """Update specified fields on a structured memory. Returns True if found.""" ... - def kv_search(self, query: str) -> list[tuple[str, str]]: - """Search key-value pairs by query. Returns matching (key, value) pairs.""" + def delete_structured_memory( + self, name: str, scope: str = "global", scope_id: str = "" + ) -> bool: + """Delete a structured memory by (name, scope, scope_id). Returns True if existed.""" + ... + + def list_structured_memories( + self, + mem_type: str = "", + scope: str = "", + scope_id: str = "", + limit: int = 100, + ) -> list[dict[str, str]]: + """Return structured memories with optional filters, ordered by updated DESC.""" + ... + + def search_structured_memories( + self, + query: str, + mem_type: str = "", + scope: str = "", + scope_id: str = "", + limit: int = 20, + ) -> list[dict[str, str]]: + """Search structured memories by query. Returns matching memory dicts.""" + ... + + def count_structured_memories(self, scope: str = "", scope_id: str = "") -> int: + """Count structured memories with optional scope filter.""" ... # -- Workstream operations ------------------------------------------------- diff --git a/turnstone/core/storage/_schema.py b/turnstone/core/storage/_schema.py index e7a2e7cc..f83d7cc1 100644 --- a/turnstone/core/storage/_schema.py +++ b/turnstone/core/storage/_schema.py @@ -9,13 +9,21 @@ import sqlalchemy as sa metadata = sa.MetaData() -memories = sa.Table( - "memories", +structured_memories = sa.Table( + "structured_memories", metadata, - sa.Column("key", sa.Text, primary_key=True), - sa.Column("value", sa.Text, nullable=False), + sa.Column("memory_id", sa.Text, primary_key=True), + sa.Column("name", sa.Text, nullable=False), + sa.Column("description", sa.Text, nullable=False, server_default=""), + sa.Column("type", sa.Text, nullable=False, server_default="project"), + sa.Column("scope", sa.Text, nullable=False, server_default="global"), + sa.Column("scope_id", sa.Text, nullable=False, server_default=""), + sa.Column("content", sa.Text, nullable=False), sa.Column("created", sa.Text, nullable=False), sa.Column("updated", sa.Text, nullable=False), + sa.Column("last_accessed", sa.Text, nullable=False, server_default=""), + sa.Column("access_count", sa.Integer, nullable=False, server_default="0"), + sa.UniqueConstraint("name", "scope", "scope_id", name="uq_smem_name_scope"), ) conversations = sa.Table( diff --git a/turnstone/core/storage/_sqlite.py b/turnstone/core/storage/_sqlite.py index 0f793475..c0cebae4 100644 --- a/turnstone/core/storage/_sqlite.py +++ b/turnstone/core/storage/_sqlite.py @@ -14,11 +14,11 @@ from turnstone.core.storage._schema import ( audit_events, conversations, intent_verdicts, - memories, metadata, orgs, prompt_templates, roles, + structured_memories, tool_policies, usage_events, user_roles, @@ -37,6 +37,9 @@ from turnstone.core.storage._utils import ( from turnstone.core.storage._utils import ( ROLE_MUTABLE as _ROLE_MUTABLE, ) +from turnstone.core.storage._utils import ( + STRUCTURED_MEMORY_MUTABLE as _SMEM_MUTABLE, +) from turnstone.core.storage._utils import ( TEMPLATE_MUTABLE as _TEMPLATE_MUTABLE, ) @@ -341,68 +344,6 @@ class SQLiteBackend: ) conn.commit() - # -- Generic key-value store ----------------------------------------------- - - def kv_get(self, key: str) -> str | None: - with self._engine.connect() as conn: - row = conn.execute(sa.select(memories.c.value).where(memories.c.key == key)).fetchone() - return str(row[0]) if row else None - - def kv_set(self, key: str, value: str) -> str | None: - with self._engine.connect() as conn: - existing = conn.execute( - sa.select(memories.c.value).where(memories.c.key == key) - ).fetchone() - old_value = str(existing[0]) if existing else None - now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") - conn.execute( - sa.text( - "INSERT OR REPLACE INTO memories (key, value, created, updated) " - "VALUES (:key, :value, " - "COALESCE((SELECT created FROM memories WHERE key = :key), :now), " - ":now)" - ), - {"key": key, "value": value, "now": now}, - ) - conn.commit() - return old_value - - def kv_delete(self, key: str) -> bool: - with self._engine.connect() as conn: - result = conn.execute(sa.delete(memories).where(memories.c.key == key)) - conn.commit() - return result.rowcount > 0 - - def kv_list(self) -> list[tuple[str, str]]: - with self._engine.connect() as conn: - rows = conn.execute( - sa.select(memories.c.key, memories.c.value).order_by(memories.c.key) - ).fetchall() - return [(str(r[0]), str(r[1])) for r in rows] - - def kv_search(self, query: str) -> list[tuple[str, str]]: - if not query or not query.strip(): - return self.kv_list() - terms = query.split() - with self._engine.connect() as conn: - # Build WHERE clause: each term must match key OR value - clauses = [] - params: dict[str, str] = {} - for i, t in enumerate(terms): - escaped = _escape_like(t) - clauses.append(f"(key LIKE :k{i} ESCAPE '\\' OR value LIKE :v{i} ESCAPE '\\')") - params[f"k{i}"] = f"%{escaped}%" - params[f"v{i}"] = f"%{escaped}%" - rows = conn.execute( - sa.text( - "SELECT key, value FROM memories WHERE " - + " AND ".join(clauses) - + " ORDER BY key" - ), - params, - ).fetchall() - return [(str(r[0]), str(r[1])) for r in rows] - # -- Workstream operations ------------------------------------------------- def register_workstream( @@ -2177,6 +2118,166 @@ class SQLiteBackend: row = conn.execute(q).fetchone() return row[0] if row else 0 + # -- Structured memories --------------------------------------------------- + + def create_structured_memory( + self, + memory_id: str, + name: str, + description: str, + mem_type: str, + scope: str, + scope_id: str, + content: str, + ) -> None: + now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + with self._engine.connect() as conn: + conn.execute( + sa.insert(structured_memories), + { + "memory_id": memory_id, + "name": name, + "description": description, + "type": mem_type, + "scope": scope, + "scope_id": scope_id, + "content": content, + "created": now, + "updated": now, + "last_accessed": now, + "access_count": 0, + }, + ) + conn.commit() + + def get_structured_memory(self, memory_id: str) -> dict[str, str] | None: + with self._engine.connect() as conn: + row = conn.execute( + sa.select(structured_memories).where(structured_memories.c.memory_id == memory_id) + ).fetchone() + return dict(row._mapping) if row else None + + def get_structured_memory_by_name( + self, name: str, scope: str = "global", scope_id: str = "" + ) -> dict[str, str] | None: + with self._engine.connect() as conn: + row = conn.execute( + sa.select(structured_memories).where( + sa.and_( + structured_memories.c.name == name, + structured_memories.c.scope == scope, + structured_memories.c.scope_id == scope_id, + ) + ) + ).fetchone() + return dict(row._mapping) if row else None + + def update_structured_memory(self, memory_id: str, **fields: str) -> bool: + fields = {k: v for k, v in fields.items() if k in _SMEM_MUTABLE} + if not fields: + return False + now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S") + fields["updated"] = now + fields["last_accessed"] = now + with self._engine.connect() as conn: + result = conn.execute( + sa.update(structured_memories) + .where(structured_memories.c.memory_id == memory_id) + .values(**fields) + ) + conn.commit() + return result.rowcount > 0 + + def delete_structured_memory( + self, name: str, scope: str = "global", scope_id: str = "" + ) -> bool: + with self._engine.connect() as conn: + result = conn.execute( + sa.delete(structured_memories).where( + sa.and_( + structured_memories.c.name == name, + structured_memories.c.scope == scope, + structured_memories.c.scope_id == scope_id, + ) + ) + ) + conn.commit() + return result.rowcount > 0 + + def list_structured_memories( + self, + mem_type: str = "", + scope: str = "", + scope_id: str = "", + limit: int = 100, + ) -> list[dict[str, str]]: + with self._engine.connect() as conn: + q = sa.select(structured_memories).order_by(structured_memories.c.updated.desc()) + if mem_type: + q = q.where(structured_memories.c.type == mem_type) + if scope: + q = q.where(structured_memories.c.scope == scope) + if scope_id: + q = q.where(structured_memories.c.scope_id == scope_id) + q = q.limit(limit) + rows = conn.execute(q).fetchall() + return [dict(r._mapping) for r in rows] + + def search_structured_memories( + self, + query: str, + mem_type: str = "", + scope: str = "", + scope_id: str = "", + limit: int = 20, + ) -> list[dict[str, str]]: + if not query or not query.strip(): + return self.list_structured_memories( + mem_type=mem_type, scope=scope, scope_id=scope_id, limit=limit + ) + terms = query.split() + with self._engine.connect() as conn: + clauses = [] + params: dict[str, str] = {} + for i, t in enumerate(terms): + escaped = _escape_like(t) + clauses.append( + f"(name LIKE :n{i} ESCAPE '\\' " + f"OR description LIKE :d{i} ESCAPE '\\' " + f"OR content LIKE :c{i} ESCAPE '\\')" + ) + params[f"n{i}"] = f"%{escaped}%" + params[f"d{i}"] = f"%{escaped}%" + params[f"c{i}"] = f"%{escaped}%" + where = " AND ".join(clauses) + if mem_type: + where += " AND type = :type_filter" + params["type_filter"] = mem_type + if scope: + where += " AND scope = :scope_filter" + params["scope_filter"] = scope + if scope_id: + where += " AND scope_id = :scope_id_filter" + params["scope_id_filter"] = scope_id + rows = conn.execute( + sa.text( + f"SELECT * FROM structured_memories WHERE {where} " + f"ORDER BY updated DESC LIMIT :lim" + ), + {**params, "lim": limit}, + ).fetchall() + return [dict(r._mapping) for r in rows] + + def count_structured_memories(self, scope: str = "", scope_id: str = "") -> int: + with self._engine.connect() as conn: + q = sa.select(sa.func.count()).select_from(structured_memories) + if scope: + q = q.where(structured_memories.c.scope == scope) + if scope_id: + q = q.where(structured_memories.c.scope_id == scope_id) + result = conn.execute(q).scalar() + return int(result or 0) + # -- Lifecycle ------------------------------------------------------------- def close(self) -> None: diff --git a/turnstone/core/storage/_utils.py b/turnstone/core/storage/_utils.py index 494fa18d..d6864e34 100644 --- a/turnstone/core/storage/_utils.py +++ b/turnstone/core/storage/_utils.py @@ -47,6 +47,7 @@ WS_TEMPLATE_MUTABLE = frozenset( "enabled", } ) +STRUCTURED_MEMORY_MUTABLE = frozenset({"content", "description", "type"}) VERDICT_MUTABLE = frozenset( { "user_decision", diff --git a/turnstone/core/storage/migrations/versions/014_structured_memories.py b/turnstone/core/storage/migrations/versions/014_structured_memories.py new file mode 100644 index 00000000..f270443f --- /dev/null +++ b/turnstone/core/storage/migrations/versions/014_structured_memories.py @@ -0,0 +1,76 @@ +"""Create structured_memories table and migrate existing flat memories. + +Revision ID: 014 +Revises: 013 +Create Date: 2026-03-13 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "014" +down_revision = "013" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "structured_memories", + sa.Column("memory_id", sa.Text, primary_key=True), + sa.Column("name", sa.Text, nullable=False), + sa.Column("description", sa.Text, nullable=False, server_default=""), + sa.Column("type", sa.Text, nullable=False, server_default="project"), + sa.Column("scope", sa.Text, nullable=False, server_default="global"), + sa.Column("scope_id", sa.Text, nullable=False, server_default=""), + sa.Column("content", sa.Text, nullable=False), + sa.Column("created", sa.Text, nullable=False), + sa.Column("updated", sa.Text, nullable=False), + sa.Column("last_accessed", sa.Text, nullable=False, server_default=""), + sa.Column("access_count", sa.Integer, nullable=False, server_default="0"), + ) + op.create_unique_constraint( + "uq_smem_name_scope", "structured_memories", ["name", "scope", "scope_id"] + ) + op.create_index("idx_smem_type", "structured_memories", ["type"]) + op.create_index("idx_smem_scope", "structured_memories", ["scope", "scope_id"]) + + # Migrate existing flat memories into structured_memories + conn = op.get_bind() + conn.execute( + sa.text( + "INSERT INTO structured_memories " + "(memory_id, name, description, type, scope, scope_id, content, created, updated) " + "SELECT " + " 'migrated-' || key, " + " key, " + " '', " + " 'project', " + " 'global', " + " '', " + " value, " + " created, " + " updated " + "FROM memories" + ) + ) + op.drop_table("memories") + + +def downgrade() -> None: + op.create_table( + "memories", + sa.Column("key", sa.Text, primary_key=True), + sa.Column("value", sa.Text, nullable=False), + sa.Column("created", sa.Text, nullable=False), + sa.Column("updated", sa.Text, nullable=False), + ) + conn = op.get_bind() + conn.execute( + sa.text( + "INSERT INTO memories (key, value, created, updated) " + "SELECT name, content, created, updated " + "FROM structured_memories WHERE scope = 'global'" + ) + ) + op.drop_table("structured_memories") diff --git a/turnstone/core/tool_search.py b/turnstone/core/tool_search.py index e9e411e8..549e431e 100644 --- a/turnstone/core/tool_search.py +++ b/turnstone/core/tool_search.py @@ -8,67 +8,11 @@ models (vLLM, llama.cpp) use the client-side BM25 fallback here. from __future__ import annotations -import math import re from collections import Counter from typing import Any -# --------------------------------------------------------------------------- -# BM25 index — lightweight, pure-Python, zero external deps -# --------------------------------------------------------------------------- - -_SPLIT_RE = re.compile(r"[_\-./\s]+") - - -def _tokenize(text: str) -> list[str]: - """Split text on whitespace, underscores, hyphens, dots.""" - return [t.lower() for t in _SPLIT_RE.split(text) if t] - - -class BM25Index: - """Okapi BM25 index over tool name + description text.""" - - def __init__(self, documents: list[str], *, k1: float = 1.5, b: float = 0.75) -> None: - self.k1 = k1 - self.b = b - self._docs = documents - self._doc_tokens: list[list[str]] = [_tokenize(d) for d in documents] - self._doc_lens = [len(t) for t in self._doc_tokens] - self._avgdl = sum(self._doc_lens) / max(len(self._doc_lens), 1) - self._n = len(documents) - # Document frequency per term - self._df: Counter[str] = Counter() - for tokens in self._doc_tokens: - for term in set(tokens): - self._df[term] += 1 - - def search(self, query: str, k: int = 5) -> list[int]: - """Return indices of top-k documents sorted by descending BM25 score.""" - q_tokens = _tokenize(query) - if not q_tokens: - return [] - scores: list[tuple[float, int]] = [] - for idx, doc_tokens in enumerate(self._doc_tokens): - score = self._score(q_tokens, doc_tokens, self._doc_lens[idx]) - if score > 0: - scores.append((score, idx)) - scores.sort(key=lambda x: (-x[0], x[1])) - return [idx for _, idx in scores[:k]] - - def _score(self, q_tokens: list[str], doc_tokens: list[str], dl: int) -> float: - tf_map: Counter[str] = Counter(doc_tokens) - score = 0.0 - for term in q_tokens: - if term not in tf_map: - continue - tf = tf_map[term] - df = self._df.get(term, 0) - idf = math.log((self._n - df + 0.5) / (df + 0.5) + 1.0) - numerator = tf * (self.k1 + 1) - denominator = tf + self.k1 * (1 - self.b + self.b * dl / self._avgdl) - score += idf * numerator / denominator - return score - +from turnstone.core.bm25 import BM25Index, _tokenize # noqa: F401 # --------------------------------------------------------------------------- # Tool search manager — partitions tools, tracks visibility diff --git a/turnstone/mq/bridge.py b/turnstone/mq/bridge.py index d8df1da4..bd3278a3 100644 --- a/turnstone/mq/bridge.py +++ b/turnstone/mq/bridge.py @@ -55,7 +55,7 @@ if TYPE_CHECKING: log = logging.getLogger("turnstone.mq.bridge") # Server's default safe tools (auto-approved without user confirmation) -DEFAULT_SAFE_TOOLS = frozenset(["read_file", "search", "man", "remember", "recall", "forget"]) +DEFAULT_SAFE_TOOLS = frozenset(["read_file", "search", "man", "memory", "recall"]) class Bridge: diff --git a/turnstone/server.py b/turnstone/server.py index ae3b7dc1..af17c9b2 100644 --- a/turnstone/server.py +++ b/turnstone/server.py @@ -2013,6 +2013,7 @@ def main() -> None: ) -> ChatSession: assert ui is not None r_client, r_model, r_cfg = registry.resolve(model_alias) + uid = getattr(ui, "_user_id", "") or "" return ChatSession( client=r_client, model=r_model, @@ -2038,6 +2039,7 @@ def main() -> None: tool_search_max_results=args.tool_search_max_results, template=args.template, judge_config=judge_config, + user_id=uid, ) # Create WatchRunner (periodic command polling, server-level) diff --git a/turnstone/tools/forget.json b/turnstone/tools/forget.json deleted file mode 100644 index 74a88448..00000000 --- a/turnstone/tools/forget.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "forget", - "description": "Remove a persistent memory by key. Use when the user asks to forget, remove, or delete a stored memory.", - "parameters": { - "type": "object", - "properties": { - "key": { - "type": "string", - "description": "The memory key to remove (e.g. 'user_name')." - } - }, - "required": ["key"] - }, - "primary_key": "key" -} diff --git a/turnstone/tools/memory.json b/turnstone/tools/memory.json new file mode 100644 index 00000000..a05de302 --- /dev/null +++ b/turnstone/tools/memory.json @@ -0,0 +1,46 @@ +{ + "name": "memory", + "description": "Persistent memory across sessions. Actions: 'save' stores a memory, 'search' finds memories by query, 'delete' removes a memory, 'list' shows all memories. Memories have a type (user/project/feedback/reference) and scope (global/workstream/user).", + "parameters": { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": ["save", "search", "delete", "list"], + "description": "Action to perform." + }, + "name": { + "type": "string", + "description": "Memory identifier (required for 'save' and 'delete'). Short snake_case key." + }, + "content": { + "type": "string", + "description": "Memory content (required for 'save')." + }, + "description": { + "type": "string", + "description": "Short description for relevance matching (recommended for 'save')." + }, + "type": { + "type": "string", + "enum": ["user", "project", "feedback", "reference"], + "description": "Memory type. Default: 'project'." + }, + "scope": { + "type": "string", + "enum": ["global", "workstream", "user"], + "description": "Memory scope. Default: 'global'. Use 'workstream' for context private to this workstream, 'user' for context that follows the user across workstreams." + }, + "query": { + "type": "string", + "description": "Search query (for 'search' action)." + }, + "limit": { + "type": "integer", + "description": "Max results for 'search' or 'list'. Default: 20." + } + }, + "required": ["action"] + }, + "primary_key": "name" +} diff --git a/turnstone/tools/recall.json b/turnstone/tools/recall.json index 3be6fbf9..a1f34335 100644 --- a/turnstone/tools/recall.json +++ b/turnstone/tools/recall.json @@ -1,18 +1,19 @@ { "name": "recall", - "description": "Search memories and past conversations. With no query, lists all saved memories. With a query, searches both memories and conversation history.", + "description": "Search conversation history for past messages, tool results, and interactions across sessions.", "parameters": { "type": "object", "properties": { "query": { "type": "string", - "description": "Search term or phrase. Omit to list all memories." + "description": "Search term or phrase to find in conversation history." }, "limit": { "type": "integer", - "description": "Max conversation results to return (default 20)." + "description": "Max results to return (default 20)." } - } + }, + "required": ["query"] }, "primary_key": "query" } diff --git a/turnstone/tools/remember.json b/turnstone/tools/remember.json deleted file mode 100644 index 1693e3f1..00000000 --- a/turnstone/tools/remember.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "remember", - "description": "Save a persistent memory. Memories persist across sessions. Use to remember IPs, paths, commands, conventions, or any fact worth recalling later.", - "parameters": { - "type": "object", - "properties": { - "key": { - "type": "string", - "description": "Short identifier (e.g. 'user_name')." - }, - "value": { - "type": "string", - "description": "Content to remember." - } - }, - "required": ["key", "value"] - }, - "primary_key": "key" -}