Files
turnstone/tests/test_rendezvous.py
Patrick Buckley 7c16b0dfa8 refactor(routing): replace hash-ring rebalancer with rendezvous (HRW)… (#384)
* refactor(routing): replace hash-ring rebalancer with rendezvous (HRW) hashing

Routing was a stored bucket table maintained by a central rebalancer
daemon, which shared its liveness primitive (services.last_heartbeat)
with the collector — when a heartbeat-fresh node went into a zombie
HTTP-handler-broken state, neither the collector nor the rebalancer
could self-correct, and the router kept directing traffic at it.
Rendezvous hashing makes the route a pure function of (ws_id,
live_services) so the heartbeat is the single source of truth and any
liveness-eviction propagates to the next route call without a separate
state-publication step.

The rebalancer's central state has no analogue: the new router computes
the per-key node winner on every call, the collector pushes membership
updates into the router cache from its discovery thread, and per-route
overrides survive on workstream_overrides. Eager workstream migration
goes away; in-flight workstreams lazily rehydrate from storage on the
new owner — already the dead-node behaviour.

* fix(tools): describe rendezvous re-routing on spawn/inspect node_id

The first pass overclaimed `node_id` "stays canonical for this
workstream's lifetime" — under rendezvous routing the active owner
re-derives per-call from live membership, so a node join/drop after
spawn can shift it.  Tool descriptions now say `node_id` is the
spawn-time binding; subsequent ops re-route via rendezvous over the
current live-node set; the new owner lazily rehydrates from shared
storage; coordinators should re-read with inspect_workstream rather
than caching the value.
2026-04-18 19:02:52 -07:00

139 lines
5.1 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Tests for turnstone.core.rendezvous (HRW routing primitive)."""
from __future__ import annotations
import pytest
from turnstone.core.rendezvous import NoAvailableNodeError, NodeRef, fnv1a_32, select, select_all
class TestFnv1aVectors:
"""Pin the FNV-1a-32 implementation against the documented test
vectors so cross-language readers (Go, TS) stay in sync."""
def test_empty_input(self) -> None:
assert fnv1a_32(b"") == 0x811C9DC5 # basis
def test_foobar(self) -> None:
assert fnv1a_32(b"foobar") == 0xBF9CF968
def test_single_byte(self) -> None:
# Hand-computed: (basis ^ 0x61) * prime, masked to 32 bits.
expected = ((0x811C9DC5 ^ 0x61) * 0x01000193) & 0xFFFFFFFF
assert fnv1a_32(b"a") == expected
class TestSelect:
def test_empty_node_list_raises(self) -> None:
with pytest.raises(NoAvailableNodeError):
select("any-key", [])
def test_single_node_always_wins(self) -> None:
only = NodeRef("solo", "http://solo")
for key in ("a", "b", "00ff" + "0" * 28):
assert select(key, [only]) is only
def test_deterministic(self) -> None:
nodes = [NodeRef(f"n{i}", f"http://n{i}") for i in range(5)]
key = "deadbeef" * 4
first = select(key, nodes)
for _ in range(20):
assert select(key, nodes) is first
def test_independent_of_node_list_order(self) -> None:
nodes = [NodeRef(f"n{i}", f"http://n{i}") for i in range(5)]
key = "feedface" * 4
forward = select(key, nodes)
backward = select(key, list(reversed(nodes)))
assert forward.node_id == backward.node_id
def test_distribution_roughly_uniform(self) -> None:
nodes = [NodeRef(f"n{i}", f"http://n{i}") for i in range(4)]
counts = {n.node_id: 0 for n in nodes}
# Use sequential keys — 32 hex chars is what the router actually
# passes in. Sequential isn't a problem because FNV-1a smears.
for i in range(4000):
key = f"{i:08x}" + "0" * 24
counts[select(key, nodes).node_id] += 1
# Each node should win ~25% (1000); allow ±15% drift.
for c in counts.values():
assert 850 < c < 1150, counts
class TestMinimalMoves:
def test_join_only_moves_to_new_node(self) -> None:
old = [NodeRef(f"n{i}", f"http://n{i}") for i in range(3)]
new = [*old, NodeRef("n3", "http://n3")]
moved_correctly = 0
moved_incorrectly = 0
for i in range(2000):
key = f"{i:08x}" + "0" * 24
before = select(key, old).node_id
after = select(key, new).node_id
if before == after:
continue
if after == "n3":
moved_correctly += 1
else:
moved_incorrectly += 1
# Strict invariant: a join must never move a key between two
# surviving nodes.
assert moved_incorrectly == 0
# Sanity: some keys did move.
assert moved_correctly > 0
def test_leave_does_not_disturb_surviving_nodes(self) -> None:
old = [NodeRef(f"n{i}", f"http://n{i}") for i in range(4)]
new = old[:-1] # n3 leaves
for i in range(2000):
key = f"{i:08x}" + "0" * 24
before = select(key, old).node_id
after = select(key, new).node_id
if before == "n3":
# Must rehome to a survivor.
assert after in {"n0", "n1", "n2"}
else:
# Must not move.
assert after == before
class TestWeights:
def test_higher_weight_wins_more_often(self) -> None:
nodes = [
NodeRef("light", "http://l", weight=1),
NodeRef("heavy", "http://h", weight=4),
]
on_heavy = 0
for i in range(5000):
key = f"{i:08x}" + "0" * 24
if select(key, nodes).node_id == "heavy":
on_heavy += 1
# Heavy gets clearly more than half; tolerance for the simple
# hash×weight formulation is wide.
assert on_heavy / 5000 > 0.65
def test_zero_weight_clamped_to_one(self) -> None:
# A weight-0 node still participates as if weight 1 — defended
# at both NodeRef construction and _score(). Use the public
# surface to sanity check.
nodes = [
NodeRef("a", "http://a", weight=0),
NodeRef("b", "http://b", weight=0),
]
# Just confirms it doesn't divide-by-zero or score to 0.
winner = select("any-key", nodes)
assert winner.node_id in {"a", "b"}
class TestSelectAll:
def test_returns_all_nodes_in_score_order(self) -> None:
nodes = [NodeRef(f"n{i}", f"http://n{i}") for i in range(4)]
ranked = select_all("some-key", nodes)
assert len(ranked) == 4
assert {n.node_id for n in ranked} == {"n0", "n1", "n2", "n3"}
# Top of the ranked list matches the single-select winner.
assert ranked[0] is select("some-key", nodes)
def test_empty_list_returns_empty(self) -> None:
assert select_all("any-key", []) == []