mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
Patrick Buckley
7c16b0dfa8
* refactor(routing): replace hash-ring rebalancer with rendezvous (HRW) hashing Routing was a stored bucket table maintained by a central rebalancer daemon, which shared its liveness primitive (services.last_heartbeat) with the collector — when a heartbeat-fresh node went into a zombie HTTP-handler-broken state, neither the collector nor the rebalancer could self-correct, and the router kept directing traffic at it. Rendezvous hashing makes the route a pure function of (ws_id, live_services) so the heartbeat is the single source of truth and any liveness-eviction propagates to the next route call without a separate state-publication step. The rebalancer's central state has no analogue: the new router computes the per-key node winner on every call, the collector pushes membership updates into the router cache from its discovery thread, and per-route overrides survive on workstream_overrides. Eager workstream migration goes away; in-flight workstreams lazily rehydrate from storage on the new owner — already the dead-node behaviour. * fix(tools): describe rendezvous re-routing on spawn/inspect node_id The first pass overclaimed `node_id` "stays canonical for this workstream's lifetime" — under rendezvous routing the active owner re-derives per-call from live membership, so a node join/drop after spawn can shift it. Tool descriptions now say `node_id` is the spawn-time binding; subsequent ops re-route via rendezvous over the current live-node set; the new owner lazily rehydrates from shared storage; coordinators should re-read with inspect_workstream rather than caching the value.
100 lines
3.5 KiB
Python
100 lines
3.5 KiB
Python
"""Tests for turnstone.console.metrics."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from turnstone.console.metrics import ConsoleMetrics
|
|
|
|
|
|
class TestRecordRoute:
|
|
"""Recording routed requests."""
|
|
|
|
def test_single_request(self) -> None:
|
|
m = ConsoleMetrics()
|
|
m.record_route("send", 200, 0.05)
|
|
|
|
text = m.generate_text()
|
|
assert 'turnstone_router_requests_total{method="send",status="2xx"} 1' in text
|
|
|
|
def test_multiple_methods(self) -> None:
|
|
m = ConsoleMetrics()
|
|
m.record_route("send", 200, 0.01)
|
|
m.record_route("create", 200, 0.02)
|
|
m.record_route("send", 502, 0.5)
|
|
|
|
text = m.generate_text()
|
|
assert 'turnstone_router_requests_total{method="send",status="2xx"} 1' in text
|
|
assert 'turnstone_router_requests_total{method="create",status="2xx"} 1' in text
|
|
assert 'turnstone_router_requests_total{method="send",status="5xx"} 1' in text
|
|
|
|
def test_duration_recorded(self) -> None:
|
|
m = ConsoleMetrics()
|
|
m.record_route("send", 200, 0.123)
|
|
m.record_route("send", 200, 0.456)
|
|
|
|
text = m.generate_text()
|
|
assert 'turnstone_router_request_duration_seconds_count{method="send"} 2' in text
|
|
# Sum should be 0.579
|
|
assert "turnstone_router_request_duration_seconds_sum" in text
|
|
|
|
|
|
class TestRouterInfo:
|
|
"""Live-membership gauge + refresh counter."""
|
|
|
|
def test_defaults_zero(self) -> None:
|
|
m = ConsoleMetrics()
|
|
text = m.generate_text()
|
|
assert "turnstone_router_membership_size 0" in text
|
|
assert "turnstone_router_refresh_total 0" in text
|
|
|
|
def test_set_router_info(self) -> None:
|
|
m = ConsoleMetrics()
|
|
m.set_router_info(3, 7)
|
|
|
|
text = m.generate_text()
|
|
assert "turnstone_router_membership_size 3" in text
|
|
assert "turnstone_router_refresh_total 7" in text
|
|
|
|
|
|
class TestGenerateText:
|
|
"""Output format validation."""
|
|
|
|
def test_contains_all_metric_names(self) -> None:
|
|
m = ConsoleMetrics()
|
|
text = m.generate_text()
|
|
expected = [
|
|
"turnstone_router_requests_total",
|
|
"turnstone_router_request_duration_seconds",
|
|
"turnstone_router_membership_size",
|
|
"turnstone_router_refresh_total",
|
|
]
|
|
for name in expected:
|
|
assert name in text, f"Missing metric: {name}"
|
|
|
|
def test_has_help_and_type(self) -> None:
|
|
m = ConsoleMetrics()
|
|
text = m.generate_text()
|
|
assert "# HELP turnstone_router_requests_total" in text
|
|
assert "# TYPE turnstone_router_requests_total counter" in text
|
|
assert "# HELP turnstone_router_membership_size" in text
|
|
assert "# TYPE turnstone_router_membership_size gauge" in text
|
|
|
|
def test_ends_with_newline(self) -> None:
|
|
m = ConsoleMetrics()
|
|
text = m.generate_text()
|
|
assert text.endswith("\n")
|
|
|
|
def test_combined_scenario(self) -> None:
|
|
"""Full scenario: routes + router info."""
|
|
m = ConsoleMetrics()
|
|
m.record_route("create", 200, 0.1)
|
|
m.record_route("send", 200, 0.05)
|
|
m.record_route("send", 502, 1.2)
|
|
m.set_router_info(3, 12)
|
|
|
|
text = m.generate_text()
|
|
assert 'turnstone_router_requests_total{method="create",status="2xx"} 1' in text
|
|
assert 'turnstone_router_requests_total{method="send",status="2xx"} 1' in text
|
|
assert 'turnstone_router_requests_total{method="send",status="5xx"} 1' in text
|
|
assert "turnstone_router_membership_size 3" in text
|
|
assert "turnstone_router_refresh_total 12" in text
|