mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Patrick Buckley
2169559d6e
* feat(projects): governed project containers — memory scope, grouping, manage UI
A workstream can attach to a project: a first-class, shareable resource
container that owns a `project` memory scope, groups conversations, and is
managed from the console.
Storage / migration 062: projects + project_members tables, workstreams.
project_id, and the memory type default project→general; grants
project.{create,read,write,delete} (admin-default).
Recall + writes: project memory is recalled iff the workstream is attached AND
the user has access (owner ∨ member ∨ public-for-read), resolved once at session
construction; coordinators recall it too. New saves default to the project when
attached + writable; the save and delete paths are write-gated; deleting a
project purges its scoped memory; archived projects aren't recalled.
Access = RBAC capability ∧ per-project ACL (auth.resolve_project_access, a
single-fetch resolver); visibility changes, member management, and delete are
owner-only.
API: project CRUD routes on both the server and console; project_id threaded
through workstream creation, spawn inheritance, the cluster-create proxy, the
dashboard / snapshot / coordinator row builders, and the collector deltas.
UI: a project picker with an inline "+ New project" creator in every creation
box (console launcher + standalone dialog + dashboard); group-by-project in the
rail; a project badge in the composer and on dashboard rows; a console manage
tab (list + create/edit + members shelves). The admin Memories view gains
coordinator/project scope filters and human scope labels (name, not hex). The
memory tool schema documents the project scope and the attach-aware default.
* fix(projects): client refresh hardening, creator race guard, SDK project_id
Addresses PR #724 review feedback plus two bugs found while validating it.
- projects.js refreshProjects: a non-OK status (e.g. 403 when the caller
lacks project.read) or a network/parse error no longer blanks the cache
or masquerades as "no projects" -- the prior cache is preserved, the
failure is recorded (new projectsError()) and warned. Honors the
long-standing "a transient error can't blank the rail" docstring.
- projects.js _fp: the fingerprint separators were raw control bytes,
which made git treat the whole file as binary (no reviewable diff).
Rewritten as escape sequences instead of raw bytes -- behavior is
byte-identical at runtime.
- project_creator.js: createProject() could reject unhandled (authFetch
throws on network/401; r.json() throws on a non-JSON body), leaving the
widget stuck busy/disabled. Added a .catch, plus a generation guard so a
create whose widget was cancelled/reopened mid-flight drops its result
instead of selecting a project the user backed out of.
- types.ts: add project_id to CreateWorkstreamRequest / WorkstreamInfo /
DashboardWorkstream to match the server schemas (was SDK-invisible).
- test_project_api.py: move side-effecting HTTP calls out of asserts so
the requests run even under python -O.
* fix(projects): JSON.stringify the cache fingerprint, drop control-byte separators
_fp joined fields/rows on raw NUL/SOH bytes, which made projects.js read as binary to git. Replace with a collision-proof, escape-free JSON.stringify encoding -- same change-detection semantics, zero embedded control characters.
274 lines
13 KiB
Python
274 lines
13 KiB
Python
"""Tests for structured memory storage backend operations."""
|
|
|
|
|
|
class TestCreateAndGet:
|
|
def test_create_and_get_by_id(self, backend):
|
|
backend.create_structured_memory("m1", "test_key", "desc", "general", "global", "", "data")
|
|
mem = backend.get_structured_memory("m1")
|
|
assert mem is not None
|
|
assert mem["name"] == "test_key"
|
|
assert mem["content"] == "data"
|
|
assert mem["type"] == "general"
|
|
|
|
def test_get_nonexistent(self, backend):
|
|
assert backend.get_structured_memory("nope") is None
|
|
|
|
def test_get_by_name(self, backend):
|
|
backend.create_structured_memory("m1", "mykey", "d", "general", "global", "", "val")
|
|
mem = backend.get_structured_memory_by_name("mykey", "global", "")
|
|
assert mem is not None
|
|
assert mem["memory_id"] == "m1"
|
|
|
|
def test_get_by_name_scoped(self, backend):
|
|
backend.create_structured_memory("m1", "key", "d", "general", "global", "", "g")
|
|
backend.create_structured_memory("m2", "key", "d", "general", "workstream", "ws1", "w")
|
|
g = backend.get_structured_memory_by_name("key", "global", "")
|
|
w = backend.get_structured_memory_by_name("key", "workstream", "ws1")
|
|
assert g["content"] == "g"
|
|
assert w["content"] == "w"
|
|
|
|
|
|
class TestUpdate:
|
|
def test_update_content(self, backend):
|
|
backend.create_structured_memory("m1", "k", "d", "general", "global", "", "old")
|
|
assert backend.update_structured_memory("m1", content="new")
|
|
mem = backend.get_structured_memory("m1")
|
|
assert mem["content"] == "new"
|
|
|
|
def test_update_nonexistent(self, backend):
|
|
assert not backend.update_structured_memory("nope", content="x")
|
|
|
|
def test_update_no_fields(self, backend):
|
|
backend.create_structured_memory("m1", "k", "d", "general", "global", "", "data")
|
|
assert not backend.update_structured_memory("m1", bogus="val")
|
|
|
|
def test_update_bumps_timestamp(self, backend):
|
|
backend.create_structured_memory("m1", "k", "d", "general", "global", "", "data")
|
|
old = backend.get_structured_memory("m1")["updated"]
|
|
import time
|
|
|
|
time.sleep(0.01)
|
|
backend.update_structured_memory("m1", content="new")
|
|
new = backend.get_structured_memory("m1")["updated"]
|
|
assert new >= old
|
|
|
|
|
|
class TestDelete:
|
|
def test_delete_existing(self, backend):
|
|
backend.create_structured_memory("m1", "k", "d", "general", "global", "", "data")
|
|
assert backend.delete_structured_memory("k", "global", "")
|
|
assert backend.get_structured_memory("m1") is None
|
|
|
|
def test_delete_nonexistent(self, backend):
|
|
assert not backend.delete_structured_memory("nope", "global", "")
|
|
|
|
def test_delete_scoped(self, backend):
|
|
backend.create_structured_memory("m1", "k", "d", "general", "workstream", "ws1", "data")
|
|
assert not backend.delete_structured_memory("k", "global", "")
|
|
assert backend.delete_structured_memory("k", "workstream", "ws1")
|
|
|
|
|
|
class TestList:
|
|
def test_list_all(self, backend):
|
|
backend.create_structured_memory("m1", "a", "", "general", "global", "", "1")
|
|
backend.create_structured_memory("m2", "b", "", "user", "global", "", "2")
|
|
mems = backend.list_structured_memories()
|
|
assert len(mems) == 2
|
|
|
|
def test_list_by_type(self, backend):
|
|
backend.create_structured_memory("m1", "a", "", "general", "global", "", "1")
|
|
backend.create_structured_memory("m2", "b", "", "user", "global", "", "2")
|
|
mems = backend.list_structured_memories(mem_type="user")
|
|
assert len(mems) == 1
|
|
assert mems[0]["name"] == "b"
|
|
|
|
def test_list_by_scope(self, backend):
|
|
backend.create_structured_memory("m1", "a", "", "general", "global", "", "1")
|
|
backend.create_structured_memory("m2", "b", "", "general", "workstream", "ws1", "2")
|
|
mems = backend.list_structured_memories(scope="workstream")
|
|
assert len(mems) == 1
|
|
|
|
def test_list_respects_limit(self, backend):
|
|
for i in range(10):
|
|
backend.create_structured_memory(f"m{i}", f"k{i}", "", "general", "global", "", f"{i}")
|
|
mems = backend.list_structured_memories(limit=3)
|
|
assert len(mems) == 3
|
|
|
|
|
|
class TestSearch:
|
|
def test_search_by_name(self, backend):
|
|
backend.create_structured_memory("m1", "database_config", "", "general", "global", "", "pg")
|
|
backend.create_structured_memory("m2", "api_key", "", "general", "global", "", "secret")
|
|
results = backend.search_structured_memories("database")
|
|
assert len(results) == 1
|
|
assert results[0]["name"] == "database_config"
|
|
|
|
def test_search_by_content(self, backend):
|
|
backend.create_structured_memory("m1", "a", "", "general", "global", "", "postgresql host")
|
|
results = backend.search_structured_memories("postgresql")
|
|
assert len(results) == 1
|
|
|
|
def test_search_empty_lists_all(self, backend):
|
|
backend.create_structured_memory("m1", "a", "", "general", "global", "", "1")
|
|
backend.create_structured_memory("m2", "b", "", "general", "global", "", "2")
|
|
results = backend.search_structured_memories("")
|
|
assert len(results) == 2
|
|
|
|
|
|
class TestCount:
|
|
def test_count_all(self, backend):
|
|
backend.create_structured_memory("m1", "a", "", "general", "global", "", "1")
|
|
backend.create_structured_memory("m2", "b", "", "general", "global", "", "2")
|
|
assert backend.count_structured_memories() == 2
|
|
|
|
def test_count_by_scope(self, backend):
|
|
backend.create_structured_memory("m1", "a", "", "general", "global", "", "1")
|
|
backend.create_structured_memory("m2", "b", "", "general", "workstream", "ws1", "2")
|
|
assert backend.count_structured_memories(scope="global") == 1
|
|
assert backend.count_structured_memories(scope="workstream") == 1
|
|
|
|
|
|
class TestSearchOrOfTerms:
|
|
"""Verify that multi-word search uses OR-of-terms (any term matches → row included)."""
|
|
|
|
def test_single_matching_term_in_multi_word_query(self, backend):
|
|
"""Memory with content 'apple' found when query is 'apple banana cherry'."""
|
|
backend.create_structured_memory("m1", "apple_mem", "", "general", "global", "", "apple")
|
|
backend.create_structured_memory("m2", "other_mem", "", "general", "global", "", "grape")
|
|
|
|
results = backend.search_structured_memories("apple banana cherry")
|
|
names = {r["name"] for r in results}
|
|
assert "apple_mem" in names # matches "apple" — OR-of-terms keeps it
|
|
assert "other_mem" not in names # "grape" matches nothing in the query
|
|
|
|
def test_partial_overlap_across_memories(self, backend):
|
|
"""Each memory matches one of three terms; all three are returned."""
|
|
backend.create_structured_memory("m1", "alpha_doc", "", "general", "global", "", "alpha")
|
|
backend.create_structured_memory("m2", "beta_doc", "", "general", "global", "", "beta")
|
|
backend.create_structured_memory("m3", "gamma_doc", "", "general", "global", "", "gamma")
|
|
backend.create_structured_memory("m4", "unrelated", "", "general", "global", "", "delta")
|
|
|
|
results = backend.search_structured_memories("alpha beta gamma")
|
|
names = {r["name"] for r in results}
|
|
assert "alpha_doc" in names
|
|
assert "beta_doc" in names
|
|
assert "gamma_doc" in names
|
|
assert "unrelated" not in names # "delta" doesn't appear in the query
|
|
|
|
def test_scope_filter_preserved(self, backend):
|
|
"""OR-of-terms search still respects scope / scope_id filters."""
|
|
backend.create_structured_memory(
|
|
"m1", "ws1_note", "", "general", "workstream", "ws1", "info"
|
|
)
|
|
backend.create_structured_memory(
|
|
"m2", "ws2_note", "", "general", "workstream", "ws2", "info"
|
|
)
|
|
backend.create_structured_memory("m3", "global_note", "", "general", "global", "", "info")
|
|
|
|
results = backend.search_structured_memories("info", scope="workstream", scope_id="ws1")
|
|
names = {r["name"] for r in results}
|
|
assert "ws1_note" in names
|
|
assert "ws2_note" not in names
|
|
assert "global_note" not in names
|
|
|
|
def test_term_cap_normalizes_unbounded_query(self, backend):
|
|
"""A multi-KB query collapses to <= MAX terms (de-dupe + length filter)."""
|
|
backend.create_structured_memory("m1", "alpha_doc", "", "general", "global", "", "alpha")
|
|
backend.create_structured_memory(
|
|
"m2", "other_doc", "", "general", "global", "", "irrelevant"
|
|
)
|
|
|
|
# Build a noisy query: same word repeated, plus 1-char tokens that
|
|
# the normalizer drops, plus the actual signal "alpha".
|
|
noisy = " ".join(["x"] * 100 + ["alpha"] * 50)
|
|
results = backend.search_structured_memories(noisy)
|
|
names = {r["name"] for r in results}
|
|
assert "alpha_doc" in names
|
|
|
|
|
|
class TestVisibleStructuredMemories:
|
|
"""Single-query union helpers used by the composition path."""
|
|
|
|
def test_list_visible_unions_global_workstream_user(self, backend):
|
|
backend.create_structured_memory("m1", "g_note", "", "general", "global", "", "g")
|
|
backend.create_structured_memory("m2", "ws_note", "", "general", "workstream", "ws1", "w")
|
|
backend.create_structured_memory("m3", "u_note", "", "general", "user", "u1", "u")
|
|
backend.create_structured_memory("m4", "other_ws", "", "general", "workstream", "ws2", "x")
|
|
|
|
scopes = [("global", ""), ("workstream", "ws1"), ("user", "u1")]
|
|
rows = backend.list_visible_structured_memories(scopes)
|
|
names = {r["name"] for r in rows}
|
|
assert names == {"g_note", "ws_note", "u_note"} # ws2 excluded
|
|
|
|
def test_search_visible_unions_scopes_and_terms(self, backend):
|
|
backend.create_structured_memory("m1", "g_alpha", "", "general", "global", "", "alpha")
|
|
backend.create_structured_memory(
|
|
"m2", "ws_beta", "", "general", "workstream", "ws1", "beta"
|
|
)
|
|
backend.create_structured_memory(
|
|
"m3", "ws_other", "", "general", "workstream", "ws2", "alpha"
|
|
)
|
|
|
|
scopes = [("global", ""), ("workstream", "ws1")]
|
|
rows = backend.search_visible_structured_memories("alpha beta", scopes)
|
|
names = {r["name"] for r in rows}
|
|
assert "g_alpha" in names # global, matches "alpha"
|
|
assert "ws_beta" in names # ws1, matches "beta"
|
|
assert "ws_other" not in names # ws2 -> outside visibility
|
|
|
|
def test_visible_helpers_handle_empty_scopes(self, backend):
|
|
backend.create_structured_memory("m1", "anything", "", "general", "global", "", "x")
|
|
assert backend.list_visible_structured_memories([]) == []
|
|
assert backend.search_visible_structured_memories("x", []) == []
|
|
|
|
|
|
class TestStableOrderingOnTimestampTies:
|
|
"""When two memories share an `updated` timestamp, secondary sort on
|
|
memory_id keeps the order deterministic across calls.
|
|
|
|
`updated` is second-precision, and touch_structured_memories() can bump
|
|
a batch to identical timestamps — without a tie-breaker BM25 input
|
|
order shuffles run-to-run, busting the LLM-side prompt cache.
|
|
"""
|
|
|
|
def _seed_with_shared_timestamp(self, backend):
|
|
# Create three memories then force their `updated` columns equal —
|
|
# mirrors the real-world case where a touch_structured_memories
|
|
# batch lands them in the same second.
|
|
for mid in ("zebra_id", "apple_id", "mango_id"):
|
|
backend.create_structured_memory(
|
|
mid, f"name_{mid}", "", "general", "global", "", "shared content"
|
|
)
|
|
import sqlalchemy as sa
|
|
|
|
with backend._conn() as conn:
|
|
conn.execute(sa.text("UPDATE structured_memories SET updated = '2024-01-01T00:00:00'"))
|
|
conn.commit()
|
|
|
|
def test_list_stable_order_under_tied_updated(self, backend):
|
|
self._seed_with_shared_timestamp(backend)
|
|
first = [r["memory_id"] for r in backend.list_structured_memories()]
|
|
second = [r["memory_id"] for r in backend.list_structured_memories()]
|
|
# Deterministic across calls AND sorted by memory_id ASC for ties
|
|
assert first == second
|
|
assert first == ["apple_id", "mango_id", "zebra_id"]
|
|
|
|
def test_search_stable_order_under_tied_updated(self, backend):
|
|
self._seed_with_shared_timestamp(backend)
|
|
first = [r["memory_id"] for r in backend.search_structured_memories("shared")]
|
|
second = [r["memory_id"] for r in backend.search_structured_memories("shared")]
|
|
assert first == second
|
|
assert first == ["apple_id", "mango_id", "zebra_id"]
|
|
|
|
def test_visible_search_stable_order_under_tied_updated(self, backend):
|
|
self._seed_with_shared_timestamp(backend)
|
|
scopes = [("global", "")]
|
|
first = [
|
|
r["memory_id"] for r in backend.search_visible_structured_memories("shared", scopes)
|
|
]
|
|
second = [
|
|
r["memory_id"] for r in backend.search_visible_structured_memories("shared", scopes)
|
|
]
|
|
assert first == second
|
|
assert first == ["apple_id", "mango_id", "zebra_id"]
|