diff --git a/README.md b/README.md index 78c6d8ed..b6642b58 100644 --- a/README.md +++ b/README.md @@ -308,7 +308,7 @@ search_max_results = 5 # max tools returned per search query [server] host = "0.0.0.0" port = 8080 -max_workstreams = 10 # auto-evicts oldest idle when full +max_workstreams = 50 # auto-evicts oldest idle when full [redis] host = "localhost" @@ -340,7 +340,7 @@ burst = 20 backend = "sqlite" # "sqlite" (default) or "postgresql" path = ".turnstone.db" # SQLite file path (relative to working directory) # url = "postgresql+psycopg://user:pass@host:5432/turnstone" # PostgreSQL -# pool_size = 5 # PostgreSQL connection pool size +# pool_size = 2 # PostgreSQL connection pool size (per process) [judge] enabled = true # intent validation for tool approvals (--no-judge to disable) @@ -394,7 +394,7 @@ Idle workstreams are automatically cleaned up after 2 hours (configurable). In m - `turnstone_judge_llm_latency_seconds` — LLM judge evaluation latency histogram - `turnstone_judge_enabled` — whether the intent validation judge is active (0/1) -Per-workstream metrics are labeled by `ws_id` (bounded to 10 max workstreams). +Per-workstream metrics are labeled by `ws_id` (bounded by `[server].max_workstreams`). ### Health & Rate Limiting @@ -404,7 +404,7 @@ Per-workstream metrics are labeled by `ws_id` (bounded to 10 max workstreams). **Per-IP rate limiting.** When `[ratelimit].enabled` is true, each client IP is tracked with a token-bucket limiter (`requests_per_second` / `burst`). Rate limiting is applied in `do_GET`/`do_POST` after authentication but before route dispatch. `/health` and `/metrics` are exempt. Requests that exceed the limit receive HTTP 429 with a `Retry-After` header. -**Workstream eviction.** When `WorkstreamManager.create()` would exceed `max_workstreams`, the oldest IDLE workstream is automatically evicted and the `turnstone_workstreams_evicted_total` counter is incremented. Configure via `[server].max_workstreams` (default 10). +**Workstream eviction.** When `WorkstreamManager.create()` would exceed `max_workstreams`, the oldest IDLE workstream is automatically evicted and the `turnstone_workstreams_evicted_total` counter is incremented. Configure via `[server].max_workstreams` (default 50). ## Requirements diff --git a/compose.yaml b/compose.yaml index ed3df0f6..87d8e944 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,6 +7,7 @@ # Production (PG): DB_BACKEND=postgresql docker compose --profile production up # 10-node cluster: docker compose --profile cluster up # Cluster + DDG: docker compose --profile ddgCluster up +# 100-node stress: docker compose --profile ddgStressCluster up # With simulator: docker compose --profile sim up # ============================================================================= @@ -31,6 +32,13 @@ services: - production - cluster - ddgCluster + - ddgStressCluster + command: + - postgres + - -c + - max_connections=${POSTGRES_MAX_CONNECTIONS:-300} + - -c + - shared_buffers=128MB environment: POSTGRES_DB: turnstone POSTGRES_USER: ${POSTGRES_USER:-turnstone} @@ -49,7 +57,7 @@ services: deploy: resources: limits: - memory: 512M + memory: 1G cpus: '1.0' restart: unless-stopped @@ -165,6 +173,8 @@ services: - REDIS_PASSWORD=${REDIS_PASSWORD:-} - TURNSTONE_AUTH_TOKEN=${TURNSTONE_AUTH_TOKEN:-} - TURNSTONE_JWT_SECRET=${TURNSTONE_JWT_SECRET:-} + - TURNSTONE_DB_BACKEND=${DB_BACKEND:-sqlite} + - TURNSTONE_DB_URL=${DATABASE_URL:-} networks: - turnstone-net depends_on: @@ -261,6 +271,7 @@ services: image: python:3.14-slim profiles: - ddgCluster + - ddgStressCluster command: - sh - -c @@ -334,6 +345,7 @@ services: # -- cluster servers ------------------------------------------------ server-1: &cluster-server + image: turnstone:local build: { context: ., dockerfile: Dockerfile } profiles: [cluster, ddgCluster] command: &cluster-server-cmd @@ -412,6 +424,7 @@ services: # -- cluster bridges ------------------------------------------------ bridge-1: &cluster-bridge + image: turnstone:local build: { context: ., dockerfile: Dockerfile } profiles: [cluster, ddgCluster] command: @@ -425,6 +438,8 @@ services: REDIS_PASSWORD: ${REDIS_PASSWORD:-} TURNSTONE_AUTH_TOKEN: ${TURNSTONE_AUTH_TOKEN:-} TURNSTONE_JWT_SECRET: ${TURNSTONE_JWT_SECRET:-} + TURNSTONE_DB_BACKEND: ${DB_BACKEND:-postgresql} + TURNSTONE_DB_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER:-turnstone}:${POSTGRES_PASSWORD:?}@postgres:5432/turnstone} networks: [turnstone-net] depends_on: server-1: { condition: service_healthy } @@ -542,3 +557,2337 @@ services: depends_on: server-10: { condition: service_healthy } redis: { condition: service_healthy } + + # =================================================================== + # 100-node stress cluster (profile: ddgStressCluster) + # + # 10 groups x 10 nodes = 100 server+bridge pairs. + # Groups: alpha, beta, gamma, delta, epsilon, zeta, eta, theta, + # iota, kappa + # + # Start: docker compose --profile ddgStressCluster up + # =================================================================== + + # -- stress cluster servers ---------------------------------------- + + server-alpha-1: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: alpha-1 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-alpha-2: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: alpha-2 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-alpha-3: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: alpha-3 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-alpha-4: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: alpha-4 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-alpha-5: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: alpha-5 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-alpha-6: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: alpha-6 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-alpha-7: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: alpha-7 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-alpha-8: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: alpha-8 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-alpha-9: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: alpha-9 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-alpha-10: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: alpha-10 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + + server-beta-1: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: beta-1 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-beta-2: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: beta-2 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-beta-3: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: beta-3 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-beta-4: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: beta-4 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-beta-5: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: beta-5 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-beta-6: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: beta-6 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-beta-7: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: beta-7 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-beta-8: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: beta-8 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-beta-9: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: beta-9 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-beta-10: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: beta-10 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + + server-gamma-1: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: gamma-1 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-gamma-2: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: gamma-2 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-gamma-3: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: gamma-3 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-gamma-4: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: gamma-4 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-gamma-5: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: gamma-5 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-gamma-6: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: gamma-6 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-gamma-7: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: gamma-7 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-gamma-8: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: gamma-8 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-gamma-9: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: gamma-9 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-gamma-10: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: gamma-10 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + + server-delta-1: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: delta-1 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-delta-2: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: delta-2 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-delta-3: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: delta-3 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-delta-4: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: delta-4 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-delta-5: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: delta-5 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-delta-6: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: delta-6 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-delta-7: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: delta-7 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-delta-8: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: delta-8 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-delta-9: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: delta-9 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-delta-10: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: delta-10 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + + server-epsilon-1: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: epsilon-1 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-epsilon-2: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: epsilon-2 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-epsilon-3: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: epsilon-3 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-epsilon-4: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: epsilon-4 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-epsilon-5: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: epsilon-5 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-epsilon-6: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: epsilon-6 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-epsilon-7: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: epsilon-7 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-epsilon-8: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: epsilon-8 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-epsilon-9: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: epsilon-9 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-epsilon-10: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: epsilon-10 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + + server-zeta-1: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: zeta-1 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-zeta-2: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: zeta-2 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-zeta-3: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: zeta-3 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-zeta-4: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: zeta-4 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-zeta-5: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: zeta-5 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-zeta-6: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: zeta-6 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-zeta-7: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: zeta-7 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-zeta-8: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: zeta-8 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-zeta-9: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: zeta-9 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-zeta-10: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: zeta-10 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + + server-eta-1: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: eta-1 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-eta-2: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: eta-2 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-eta-3: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: eta-3 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-eta-4: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: eta-4 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-eta-5: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: eta-5 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-eta-6: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: eta-6 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-eta-7: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: eta-7 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-eta-8: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: eta-8 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-eta-9: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: eta-9 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-eta-10: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: eta-10 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + + server-theta-1: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: theta-1 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-theta-2: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: theta-2 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-theta-3: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: theta-3 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-theta-4: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: theta-4 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-theta-5: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: theta-5 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-theta-6: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: theta-6 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-theta-7: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: theta-7 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-theta-8: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: theta-8 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-theta-9: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: theta-9 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-theta-10: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: theta-10 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + + server-iota-1: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: iota-1 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-iota-2: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: iota-2 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-iota-3: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: iota-3 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-iota-4: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: iota-4 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-iota-5: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: iota-5 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-iota-6: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: iota-6 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-iota-7: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: iota-7 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-iota-8: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: iota-8 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-iota-9: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: iota-9 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-iota-10: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: iota-10 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + + server-kappa-1: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: kappa-1 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-kappa-2: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: kappa-2 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-kappa-3: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: kappa-3 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-kappa-4: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: kappa-4 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-kappa-5: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: kappa-5 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-kappa-6: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: kappa-6 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-kappa-7: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: kappa-7 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-kappa-8: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: kappa-8 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-kappa-9: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: kappa-9 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + server-kappa-10: + <<: *cluster-server + profiles: [ddgStressCluster] + environment: { <<: *cluster-server-env, TURNSTONE_NODE_ID: kappa-10 } + deploy: + resources: + limits: { memory: 256M, cpus: '0.25' } + + # -- stress cluster bridges ---------------------------------------- + + bridge-alpha-1: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-alpha-1:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-alpha-1: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-alpha-2: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-alpha-2:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-alpha-2: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-alpha-3: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-alpha-3:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-alpha-3: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-alpha-4: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-alpha-4:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-alpha-4: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-alpha-5: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-alpha-5:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-alpha-5: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-alpha-6: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-alpha-6:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-alpha-6: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-alpha-7: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-alpha-7:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-alpha-7: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-alpha-8: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-alpha-8:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-alpha-8: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-alpha-9: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-alpha-9:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-alpha-9: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-alpha-10: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-alpha-10:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-alpha-10: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + + bridge-beta-1: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-beta-1:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-beta-1: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-beta-2: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-beta-2:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-beta-2: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-beta-3: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-beta-3:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-beta-3: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-beta-4: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-beta-4:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-beta-4: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-beta-5: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-beta-5:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-beta-5: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-beta-6: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-beta-6:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-beta-6: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-beta-7: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-beta-7:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-beta-7: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-beta-8: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-beta-8:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-beta-8: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-beta-9: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-beta-9:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-beta-9: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-beta-10: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-beta-10:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-beta-10: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + + bridge-gamma-1: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-gamma-1:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-gamma-1: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-gamma-2: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-gamma-2:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-gamma-2: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-gamma-3: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-gamma-3:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-gamma-3: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-gamma-4: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-gamma-4:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-gamma-4: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-gamma-5: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-gamma-5:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-gamma-5: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-gamma-6: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-gamma-6:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-gamma-6: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-gamma-7: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-gamma-7:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-gamma-7: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-gamma-8: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-gamma-8:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-gamma-8: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-gamma-9: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-gamma-9:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-gamma-9: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-gamma-10: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-gamma-10:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-gamma-10: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + + bridge-delta-1: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-delta-1:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-delta-1: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-delta-2: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-delta-2:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-delta-2: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-delta-3: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-delta-3:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-delta-3: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-delta-4: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-delta-4:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-delta-4: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-delta-5: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-delta-5:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-delta-5: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-delta-6: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-delta-6:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-delta-6: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-delta-7: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-delta-7:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-delta-7: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-delta-8: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-delta-8:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-delta-8: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-delta-9: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-delta-9:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-delta-9: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-delta-10: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-delta-10:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-delta-10: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + + bridge-epsilon-1: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-epsilon-1:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-epsilon-1: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-epsilon-2: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-epsilon-2:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-epsilon-2: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-epsilon-3: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-epsilon-3:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-epsilon-3: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-epsilon-4: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-epsilon-4:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-epsilon-4: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-epsilon-5: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-epsilon-5:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-epsilon-5: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-epsilon-6: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-epsilon-6:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-epsilon-6: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-epsilon-7: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-epsilon-7:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-epsilon-7: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-epsilon-8: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-epsilon-8:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-epsilon-8: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-epsilon-9: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-epsilon-9:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-epsilon-9: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-epsilon-10: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-epsilon-10:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-epsilon-10: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + + bridge-zeta-1: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-zeta-1:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-zeta-1: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-zeta-2: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-zeta-2:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-zeta-2: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-zeta-3: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-zeta-3:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-zeta-3: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-zeta-4: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-zeta-4:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-zeta-4: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-zeta-5: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-zeta-5:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-zeta-5: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-zeta-6: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-zeta-6:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-zeta-6: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-zeta-7: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-zeta-7:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-zeta-7: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-zeta-8: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-zeta-8:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-zeta-8: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-zeta-9: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-zeta-9:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-zeta-9: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-zeta-10: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-zeta-10:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-zeta-10: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + + bridge-eta-1: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-eta-1:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-eta-1: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-eta-2: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-eta-2:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-eta-2: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-eta-3: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-eta-3:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-eta-3: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-eta-4: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-eta-4:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-eta-4: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-eta-5: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-eta-5:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-eta-5: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-eta-6: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-eta-6:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-eta-6: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-eta-7: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-eta-7:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-eta-7: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-eta-8: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-eta-8:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-eta-8: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-eta-9: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-eta-9:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-eta-9: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-eta-10: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-eta-10:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-eta-10: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + + bridge-theta-1: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-theta-1:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-theta-1: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-theta-2: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-theta-2:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-theta-2: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-theta-3: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-theta-3:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-theta-3: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-theta-4: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-theta-4:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-theta-4: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-theta-5: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-theta-5:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-theta-5: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-theta-6: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-theta-6:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-theta-6: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-theta-7: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-theta-7:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-theta-7: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-theta-8: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-theta-8:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-theta-8: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-theta-9: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-theta-9:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-theta-9: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-theta-10: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-theta-10:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-theta-10: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + + bridge-iota-1: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-iota-1:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-iota-1: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-iota-2: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-iota-2:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-iota-2: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-iota-3: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-iota-3:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-iota-3: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-iota-4: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-iota-4:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-iota-4: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-iota-5: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-iota-5:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-iota-5: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-iota-6: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-iota-6:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-iota-6: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-iota-7: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-iota-7:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-iota-7: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-iota-8: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-iota-8:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-iota-8: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-iota-9: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-iota-9:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-iota-9: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-iota-10: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-iota-10:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-iota-10: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + + bridge-kappa-1: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-kappa-1:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-kappa-1: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-kappa-2: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-kappa-2:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-kappa-2: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-kappa-3: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-kappa-3:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-kappa-3: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-kappa-4: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-kappa-4:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-kappa-4: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-kappa-5: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-kappa-5:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-kappa-5: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-kappa-6: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-kappa-6:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-kappa-6: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-kappa-7: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-kappa-7:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-kappa-7: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-kappa-8: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-kappa-8:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-kappa-8: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-kappa-9: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-kappa-9:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-kappa-9: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } + bridge-kappa-10: + <<: *cluster-bridge + profiles: [ddgStressCluster] + command: + - turnstone-bridge + - --server-url=http://server-kappa-10:8080 + - --redis-host=redis + - --redis-port=6379 + - --heartbeat-ttl=${HEARTBEAT_TTL:-60} + - --approval-timeout=${APPROVAL_TIMEOUT:-3600} + depends_on: + server-kappa-10: { condition: service_healthy } + redis: { condition: service_healthy } + deploy: + resources: + limits: { memory: 128M, cpus: '0.1' } diff --git a/docs/api-reference.md b/docs/api-reference.md index e39025cb..500065bc 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -554,7 +554,7 @@ Possible `state` values: | `error` | An error occurred | **Fan-out pattern:** Each connected client receives its own bounded queue -(`maxsize=500`). A dedicated fan-out thread reads from the shared global queue +(`maxsize=1000`). A dedicated fan-out thread reads from the shared global queue and copies each event to every client queue. If a client queue is full, the event is silently dropped for that client. diff --git a/docs/architecture.md b/docs/architecture.md index ee66b632..f4520f23 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -353,7 +353,7 @@ remove the tab immediately. Controlled by `--workstream-idle-timeout` (default: **Workstream eviction at capacity:** When `WorkstreamManager.create()` would exceed `max_workstreams` (configurable via `[server].max_workstreams`, default -10), the oldest IDLE workstream is automatically evicted to make room. The +50), the oldest IDLE workstream is automatically evicted to make room. The `turnstone_workstreams_evicted_total` counter is incremented on each eviction. If no IDLE workstream is available the create request fails as before. @@ -835,10 +835,16 @@ and are the single source of truth for both backends and Alembic migrations. backend = "sqlite" # "sqlite" | "postgresql" path = ".turnstone.db" # SQLite file path url = "" # PostgreSQL connection URL -pool_size = 5 # PostgreSQL connection pool size +pool_size = 2 # PostgreSQL connection pool size (per process) ``` -Environment variables: `TURNSTONE_DB_BACKEND`, `TURNSTONE_DB_URL`, `TURNSTONE_DB_PATH`. +Environment variables: `TURNSTONE_DB_BACKEND`, `TURNSTONE_DB_URL`, `TURNSTONE_DB_PATH`, +`TURNSTONE_DB_POOL_SIZE`. + +The default pool is intentionally small (2 base + 3 overflow = 5 per process) +because all database operations are short-burst queries that hold connections for +milliseconds. For clusters with many nodes sharing a PostgreSQL instance, use +[PgBouncer](pgbouncer.md) in transaction pooling mode. ### Persistence and Resume diff --git a/docs/console.md b/docs/console.md index 6b8d368b..f7db6d7a 100644 --- a/docs/console.md +++ b/docs/console.md @@ -69,10 +69,11 @@ All reads and writes to the node/workstream map are protected by a single `threa ### Scale Considerations -- **10,000 workstreams** at ~500 bytes each = ~5 MB in memory -- **1,000 nodes** polled in parallel with 50 threads at ~100ms each = ~2 second poll cycle +- **50,000 workstreams** (1,000 nodes × 50 per node) at ~500 bytes each = ~25 MB in memory +- **1,000 nodes** polled in parallel — fan-out concurrency is configurable via `cluster.node_fan_out_limit` (default 200), yielding 5 batches at ~100ms each = ~0.5 second poll cycle - **Filtering and pagination** run in-memory on the full workstream list — sub-millisecond at this scale -- **SSE fan-out** uses the same per-client queue pattern as the per-node server — backed-up clients get events dropped, not blocking +- **SSE fan-out** uses per-client queues (2,000 events) — backed-up clients get events dropped, not blocking +- **Database** — for clusters sharing PostgreSQL, use [PgBouncer](pgbouncer.md) in transaction pooling mode --- diff --git a/docs/diagrams/11-console-data-flow.puml b/docs/diagrams/11-console-data-flow.puml index 9bf12deb..26895b40 100644 --- a/docs/diagrams/11-console-data-flow.puml +++ b/docs/diagrams/11-console-data-flow.puml @@ -105,7 +105,7 @@ Server -> CC : get_snapshot() CC --> Server : ClusterSnapshot\n(full current state) Server -> CC : register_listener(queue) -note right : Per-client queue.Queue(maxsize=500)\nSSE via EventSourceResponse + run_in_executor() +note right : Per-client queue.Queue(maxsize=2000)\nSSE via EventSourceResponse + run_in_executor() Server -> Browser : data: {"type":"snapshot",...}\n(full state as first SSE event) diff --git a/docs/diagrams/12-deployment.puml b/docs/diagrams/12-deployment.puml index f2e3e361..940756eb 100644 --- a/docs/diagrams/12-deployment.puml +++ b/docs/diagrams/12-deployment.puml @@ -56,6 +56,26 @@ node "Docker Host" as host { end note } + node "postgres (profile: production)" <> as pg_node { + component [PostgreSQL\nport 5432] as postgres + note bottom of postgres + Healthcheck: pg_isready + Volume: postgres-data + Required for cluster + and production profiles + end note + } + + node "pgbouncer (optional)" <> as pgb_node { + component [PgBouncer\nport 6432] as pgbouncer + note bottom of pgbouncer + pool_mode: transaction + Recommended for clusters + > 50 nodes + See docs/pgbouncer.md + end note + } + node "sim (profile: sim)" <> as sim_node { component [turnstone-sim] as sim note bottom of sim @@ -92,6 +112,11 @@ console --> server : HTTP polling + proxy\n(GET /v1/api/dashboard,\nproxy /node/ sim --> redis : Redis protocol\n(queues + pubsub + keys) +' Database connections (production/cluster profiles) +server ..> pgbouncer : PostgreSQL\n(pool_size=2) +console ..> pgbouncer : PostgreSQL\n(auth/admin) +pgbouncer --> postgres : transaction\npooling + ' Environment variables note right of host **Environment Variables:** @@ -99,13 +124,17 @@ note right of host • OPENAI_API_KEY — API key • REDIS_PASSWORD — Redis auth • TURNSTONE_AUTH_TOKEN — API auth + • TURNSTONE_DB_URL — PostgreSQL URL + • POSTGRES_PASSWORD — DB password end note ' Volumes database "redis-data" as rv database "turnstone-data" as tv +database "postgres-data" as pv redis_node --> rv server_node --> tv +pg_node --> pv @enduml diff --git a/docs/diagrams/14-storage-architecture.puml b/docs/diagrams/14-storage-architecture.puml index 5cf8b849..5d11f24b 100644 --- a/docs/diagrams/14-storage-architecture.puml +++ b/docs/diagrams/14-storage-architecture.puml @@ -53,10 +53,10 @@ class "SQLiteBackend" as SQLite <> { class "PostgreSQLBackend" as PG <> { -_engine: sa.Engine - +__init__(url: str, pool_size: int) + +__init__(url: str, pool_size: int = 2,\n max_overflow: int = 3) -- tsvector + ILIKE search - Connection pooling + Connection pooling (5 max per process) } ' -- Schema -- @@ -151,7 +151,7 @@ note right of Registry backend = "sqlite" | "postgresql" url = "postgresql+psycopg://..." path = ".turnstone.db" - pool_size = 5 + pool_size = 2 (+ 3 overflow) end note note bottom of SQLite @@ -162,8 +162,9 @@ end note note bottom of PG Production backend. - Multi-node / Docker - default. + Multi-node / Docker default. + Use PgBouncer (transaction mode) + for clusters > 50 nodes. end note @enduml diff --git a/docs/diagrams/png/11-console-data-flow.png b/docs/diagrams/png/11-console-data-flow.png index 858427e8..a8e6c3d3 100644 --- a/docs/diagrams/png/11-console-data-flow.png +++ b/docs/diagrams/png/11-console-data-flow.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a74b4b8b5dbfb1a51a01100b731477968942b01218bad9451a3d5a9cb3003294 -size 411665 +oid sha256:e3f1ad0fcd55eaca3b8ad9c5abc07432803641c54ede9fc93c79df144cf77d1c +size 407761 diff --git a/docs/diagrams/png/12-deployment.png b/docs/diagrams/png/12-deployment.png index 7f114453..d134dbae 100644 --- a/docs/diagrams/png/12-deployment.png +++ b/docs/diagrams/png/12-deployment.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:84524f4bc900708ac8adf081591d336f862830188eb8505e71a0f071b339d923 -size 252599 +oid sha256:09065fef028d05e6df425fd8abefaf5a2ca04b66802f2e3975f289fa597f63ed +size 309656 diff --git a/docs/diagrams/png/14-storage-architecture.png b/docs/diagrams/png/14-storage-architecture.png index 1f8767b1..eea08a2e 100644 --- a/docs/diagrams/png/14-storage-architecture.png +++ b/docs/diagrams/png/14-storage-architecture.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c94556889abb382cd5b818639fc0a4706beef3d9c7a0b4cbedc763943d657dd0 -size 244998 +oid sha256:b047cdc318c505f0f0895a65e14c5cc7552716053055cca57fa0a77db150e618 +size 255458 diff --git a/docs/docker.md b/docs/docker.md index 05ede1e5..9e080bff 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -109,9 +109,12 @@ All configuration is via environment variables in `.env` (copy from `.env.exampl |----------|---------|-------------| | `TURNSTONE_DB_BACKEND` | `sqlite` | Storage backend: `sqlite` or `postgresql` | | `TURNSTONE_DB_URL` | — | Database URL (e.g. `postgresql://user:pass@db:5432/turnstone`). For SQLite, defaults to `/data/.turnstone.db` | +| `TURNSTONE_DB_POOL_SIZE` | `2` | PostgreSQL connection pool size per process (default: 2 base + 3 overflow = 5 max) | The database stores workstream history, user accounts, and API tokens. When using JWT auth, a database backend is required for user storage. +> **Large clusters:** Each turnstone process maintains a small connection pool (5 max). At hundreds of nodes this adds up — use [PgBouncer](pgbouncer.md) in transaction pooling mode between turnstone and PostgreSQL. + > **First-time setup:** After deploying with auth enabled, create an initial admin user by running `turnstone-admin create-user` inside the container: > > ```bash @@ -151,6 +154,8 @@ POSTGRES_PASSWORD=secret docker compose --profile cluster up The default `server` and `bridge` also run alongside the cluster nodes (11 total). All nodes are accessible via the console dashboard at `:8090`. +For production clusters beyond ~50 nodes, add PgBouncer between turnstone services and PostgreSQL. See [PgBouncer Connection Pooling](pgbouncer.md) for Docker Compose and Helm configuration. + ## Volumes | Volume | Mount | Purpose | diff --git a/docs/pgbouncer.md b/docs/pgbouncer.md new file mode 100644 index 00000000..13e55547 --- /dev/null +++ b/docs/pgbouncer.md @@ -0,0 +1,200 @@ +# PgBouncer Connection Pooling + +Turnstone cluster deployments share a single PostgreSQL instance across +all server nodes, bridge processes, and the console. Each process +maintains a small connection pool (2 base + 3 overflow = 5 max). At +scale this adds up — a 100-node cluster opens up to 500 connections, +and a 1000-node cluster up to 5,000. + +PostgreSQL's default `max_connections` is 100, and each real connection +allocates ~5–10 MB of backend memory. PgBouncer sits between turnstone +and PostgreSQL, multiplexing thousands of lightweight client connections +down to a small number of real database connections. + +--- + +## Why PgBouncer works well with turnstone + +All turnstone database operations are short-burst queries: acquire a +connection, execute 1–3 statements, commit, release. No operation holds +a connection for more than a few milliseconds. This makes **transaction +pooling mode** ideal — PgBouncer assigns a real connection only for the +duration of each transaction, then returns it to the pool. + +| Cluster size | Client connections (max) | PgBouncer server connections needed | +|--------------|------------------------|-------------------------------------| +| 10 nodes | 50 | 10–20 | +| 100 nodes | 500 | 20–40 | +| 500 nodes | 2,500 | 30–60 | +| 1,000 nodes | 5,000 | 40–80 | + +The server connection count stays low because most client connections +are idle at any given moment. + +--- + +## Docker Compose + +Add PgBouncer between turnstone services and PostgreSQL: + +```yaml +services: + pgbouncer: + image: bitnami/pgbouncer:latest + environment: + POSTGRESQL_HOST: postgres + POSTGRESQL_PORT: "5432" + POSTGRESQL_DATABASE: turnstone + POSTGRESQL_USERNAME: ${POSTGRES_USER:-turnstone} + POSTGRESQL_PASSWORD: ${POSTGRES_PASSWORD:?} + PGBOUNCER_POOL_MODE: transaction + PGBOUNCER_DEFAULT_POOL_SIZE: "40" + PGBOUNCER_MAX_CLIENT_CONN: "5000" + PGBOUNCER_MAX_DB_CONNECTIONS: "80" + PGBOUNCER_SERVER_IDLE_TIMEOUT: "300" + ports: + - "6432:6432" + networks: + - turnstone-net + depends_on: + postgres: + condition: service_healthy + healthcheck: + test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-p", "6432"] + interval: 5s + timeout: 3s + retries: 5 +``` + +Then point turnstone services at PgBouncer instead of PostgreSQL +directly by changing the `DATABASE_URL` (or `TURNSTONE_DB_URL`): + +```bash +# Before (direct) +TURNSTONE_DB_URL=postgresql://turnstone:secret@postgres:5432/turnstone + +# After (via PgBouncer) +TURNSTONE_DB_URL=postgresql://turnstone:secret@pgbouncer:6432/turnstone +``` + +--- + +## Helm / Kubernetes + +Add a PgBouncer deployment or use a Helm chart like +[bitnami/pgbouncer](https://github.com/bitnami/charts/tree/main/bitnami/pgbouncer). + +In `values.yaml`, point the database at PgBouncer: + +```yaml +database: + backend: postgresql + external: + host: pgbouncer + port: 6432 + database: turnstone + username: turnstone + existingSecret: turnstone-db-secret +``` + +PgBouncer configuration: + +```yaml +pgbouncer: + poolMode: transaction + defaultPoolSize: 40 + maxClientConn: 5000 + maxDbConnections: 80 +``` + +--- + +## Configuration reference + +| PgBouncer setting | Recommended | Notes | +|-------------------|-------------|-------| +| `pool_mode` | `transaction` | Required — turnstone uses short-burst queries with no session state | +| `default_pool_size` | 40 | Real PostgreSQL connections per database. Start here, increase if you see `no more connections allowed` | +| `max_client_conn` | 5000 | Upper bound on client connections. Set to `cluster_nodes × 5` | +| `max_db_connections` | 80 | Hard cap on real connections to PostgreSQL. Keep below PG `max_connections` minus headroom for admin/monitoring | +| `server_idle_timeout` | 300 | Close idle server connections after 5 minutes | +| `server_lifetime` | 3600 | Recycle server connections after 1 hour | + +On the PostgreSQL side: + +| PostgreSQL setting | Recommended | Notes | +|--------------------|-------------|-------| +| `max_connections` | 100 | Default is fine — PgBouncer is the only client. Set higher than `max_db_connections` to leave room for admin connections | +| `shared_buffers` | 25% of RAM | Standard PostgreSQL tuning | + +--- + +## Turnstone pool settings + +Each turnstone process maintains its own SQLAlchemy connection pool to +PgBouncer (which then multiplexes to PostgreSQL): + +| Environment variable | Default | Description | +|---------------------|---------|-------------| +| `TURNSTONE_DB_POOL_SIZE` | 2 | Base pool size per process | +| `TURNSTONE_DB_BACKEND` | sqlite | Set to `postgresql` for cluster deployments | +| `TURNSTONE_DB_URL` | — | Connection URL (point at PgBouncer, not PostgreSQL directly) | + +The default pool of 2 + 3 overflow = 5 connections per process is +intentionally small to support large clusters. You should not need to +increase this — turnstone's database operations are all short-burst +context-managed queries that hold connections for milliseconds. + +SQLAlchemy `pool_pre_ping` is enabled, so stale connections (e.g. after +PgBouncer restarts) are automatically detected and replaced. + +--- + +## Monitoring + +PgBouncer exposes stats via its admin console (connect to +PgBouncer port with user `pgbouncer`): + +```sql +-- Active and waiting clients +SHOW POOLS; + +-- Per-database stats +SHOW STATS; + +-- Current client connections +SHOW CLIENTS; +``` + +Key metrics to watch: + +- **`cl_active`** — clients with a server connection assigned. Should be + well below `max_db_connections`. +- **`cl_waiting`** — clients waiting for a server connection. Sustained + non-zero values mean you need more `default_pool_size`. +- **`sv_active`** — active server (PostgreSQL) connections. Should stay + below PostgreSQL `max_connections`. + +--- + +## Troubleshooting + +**"no more connections allowed (max_client_conn)"** — PgBouncer is +rejecting new client connections. Increase `max_client_conn` to match +your cluster size × 5. + +**"no more connections allowed (max_db_connections)"** — PgBouncer +cannot open more connections to PostgreSQL. Increase +`max_db_connections` and ensure PostgreSQL `max_connections` is higher. + +**Connections timing out on startup** — If all nodes start +simultaneously, the burst of initial connections (migrations, health +checks) can temporarily exceed the pool. PgBouncer queues excess +clients by default — this resolves itself within seconds. + +**Prepared statements not supported** — PgBouncer in `transaction` mode +does not support prepared statements. Turnstone's SQLAlchemy layer does +not use server-side prepared statements by default, so this is not an +issue. + +See also: [Docker deployment](docker.md) · [Security](security.md) diff --git a/docs/settings.md b/docs/settings.md index f1798a2c..faf94744 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -60,6 +60,7 @@ storage initialization: | `session` | instructions, retention_days, compact_max_tokens, auto_compact_pct | | `tools` | timeout, truncation, agent_max_turns, skip_permissions, search, search_threshold, search_max_results | | `server` | workstream_idle_timeout, max_workstreams | +| `cluster` | node_fan_out_limit, mcp_max_servers | | `mcp` | config_path, refresh_interval, registry_url | | `ratelimit` | enabled, requests_per_second, burst | | `health` | backend_probe_interval, backend_probe_timeout, circuit_breaker_threshold, circuit_breaker_cooldown | diff --git a/tests/test_console.py b/tests/test_console.py index fd52dd08..031ae183 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -47,8 +47,8 @@ class MockBroker: # --------------------------------------------------------------------------- -def _make_collector(broker=None, poll_interval=999, discovery_interval=999): - """Create a collector with long intervals so threads don't auto-fire.""" +def _make_collector(broker=None, poll_interval=0, discovery_interval=999): + """Create a collector with zero poll interval (no jitter delay in tests).""" b = broker or MockBroker() return ClusterCollector( broker=b, @@ -953,6 +953,8 @@ class TestConsoleWorkstreamCreation: ], 2, ) + # get_all_nodes delegates to get_nodes (mirrors real implementation) + collector.get_all_nodes.side_effect = lambda: collector.get_nodes.return_value[0] return collector @pytest.fixture() @@ -1266,47 +1268,49 @@ class TestProxyRewriting: class TestPickBestNode: """Test the _pick_best_node helper.""" + @staticmethod + def _mock_collector(nodes: list) -> MagicMock: + collector = MagicMock(spec=ClusterCollector) + collector.get_nodes.return_value = (nodes, len(nodes)) + collector.get_all_nodes.side_effect = lambda: collector.get_nodes.return_value[0] + return collector + def test_picks_node_with_most_headroom(self): from turnstone.console.server import _pick_best_node - collector = MagicMock(spec=ClusterCollector) - collector.get_nodes.return_value = ( + collector = self._mock_collector( [ {"node_id": "busy", "reachable": True, "max_ws": 10, "ws_total": 9}, {"node_id": "free", "reachable": True, "max_ws": 10, "ws_total": 2}, {"node_id": "mid", "reachable": True, "max_ws": 10, "ws_total": 5}, - ], - 3, + ] ) assert _pick_best_node(collector) == "free" def test_skips_unreachable_nodes(self): from turnstone.console.server import _pick_best_node - collector = MagicMock(spec=ClusterCollector) - collector.get_nodes.return_value = ( + collector = self._mock_collector( [ {"node_id": "down", "reachable": False, "max_ws": 10, "ws_total": 0}, {"node_id": "up", "reachable": True, "max_ws": 10, "ws_total": 5}, - ], - 2, + ] ) assert _pick_best_node(collector) == "up" def test_returns_empty_when_no_nodes(self): from turnstone.console.server import _pick_best_node - collector = MagicMock(spec=ClusterCollector) - collector.get_nodes.return_value = ([], 0) + collector = self._mock_collector([]) assert _pick_best_node(collector) == "" def test_returns_empty_when_all_unreachable(self): from turnstone.console.server import _pick_best_node - collector = MagicMock(spec=ClusterCollector) - collector.get_nodes.return_value = ( - [{"node_id": "down", "reachable": False, "max_ws": 10, "ws_total": 0}], - 1, + collector = self._mock_collector( + [ + {"node_id": "down", "reachable": False, "max_ws": 10, "ws_total": 0}, + ] ) assert _pick_best_node(collector) == "" diff --git a/tests/test_mcp_admin_api.py b/tests/test_mcp_admin_api.py index 5ad62d91..f0eaa348 100644 --- a/tests/test_mcp_admin_api.py +++ b/tests/test_mcp_admin_api.py @@ -550,6 +550,7 @@ def _fake_request(*nodes: dict[str, Any], proxy_client: Any = None) -> MagicMock """Build a minimal mock request with collector and proxy_client.""" collector = MagicMock() collector.get_nodes.return_value = (list(nodes), len(nodes)) + collector.get_all_nodes.side_effect = lambda: collector.get_nodes.return_value[0] req = MagicMock() req.app.state.collector = collector req.app.state.proxy_client = proxy_client or AsyncMock() diff --git a/tests/test_mcp_registry_api.py b/tests/test_mcp_registry_api.py index 8dcf3185..a5a2ba73 100644 --- a/tests/test_mcp_registry_api.py +++ b/tests/test_mcp_registry_api.py @@ -362,7 +362,7 @@ class TestRegistryInstall: def test_install_max_servers(self, client: TestClient, storage: SQLiteBackend) -> None: import uuid - for i in range(50): + for i in range(200): storage.create_mcp_server( server_id=uuid.uuid4().hex, name=f"server-{i}", diff --git a/turnstone/cli.py b/turnstone/cli.py index a147dfdf..035fdef2 100644 --- a/turnstone/cli.py +++ b/turnstone/cli.py @@ -752,10 +752,15 @@ def _handle_cluster_command(cmd_line: str, console_url: str | None, auth_token: def detect_model(client: Any, provider: str = "openai") -> tuple[str, int | None]: - """Auto-detect model — delegates to :func:`turnstone.core.model_registry.detect_model`.""" + """Auto-detect model — delegates to :func:`turnstone.core.model_registry.detect_model`. + + CLI always uses fatal=True, so model is never None. + """ from turnstone.core.model_registry import detect_model as _detect - return _detect(client, provider=provider) + model, ctx = _detect(client, provider=provider) + assert model is not None # fatal=True guarantees non-None or SystemExit + return model, ctx # ─── Main ────────────────────────────────────────────────────────────────── @@ -980,7 +985,7 @@ def main() -> None: db_url = getattr(args, "db_url", None) or os.environ.get("TURNSTONE_DB_URL", "") db_path = getattr(args, "db_path", None) or os.environ.get("TURNSTONE_DB_PATH", "") db_pool_size = int( - getattr(args, "db_pool_size", None) or os.environ.get("TURNSTONE_DB_POOL_SIZE", "5") + getattr(args, "db_pool_size", None) or os.environ.get("TURNSTONE_DB_POOL_SIZE", "2") ) init_storage(db_backend, path=db_path, url=db_url, pool_size=db_pool_size) diff --git a/turnstone/console/collector.py b/turnstone/console/collector.py index 75e2173a..b7f1d0b9 100644 --- a/turnstone/console/collector.py +++ b/turnstone/console/collector.py @@ -54,10 +54,10 @@ class ClusterCollector: self, broker: RedisBroker, prefix: str = "turnstone", - poll_interval: float = 10.0, + poll_interval: float = 15.0, discovery_interval: float = 15.0, - max_poll_workers: int = 50, - http_timeout: float = 5.0, + max_poll_workers: int = 200, + http_timeout: float = 30.0, auth_token: str = "", token_manager: ServiceTokenManager | None = None, ): @@ -80,7 +80,13 @@ class ClusterCollector: self._running = False self._threads: list[threading.Thread] = [] self._poll_pool = ThreadPoolExecutor(max_workers=max_poll_workers) - self._http_client = httpx.Client(timeout=http_timeout) + self._http_client = httpx.Client( + timeout=httpx.Timeout(connect=10, read=http_timeout, write=5, pool=http_timeout), + limits=httpx.Limits( + max_connections=max_poll_workers + 10, + max_keepalive_connections=min(max_poll_workers, 200), + ), + ) # SSE fan-out to browser clients self._listeners: list[queue.Queue[dict[str, Any]]] = [] @@ -240,8 +246,27 @@ class ClusterCollector: log.exception("Poll loop error") time.sleep(self._poll_interval) + @staticmethod + def _node_jitter(node_id: str, window: float) -> float: + """Deterministic per-node delay within a sliding window. + + Uses a Mersenne prime (2^31 - 1) to hash the node_id into a + stable offset so each node is polled at a different point in + the cycle. The offset is consistent across restarts for the + same node_id, giving an even spread without randomness. + """ + h = hash(node_id) & 0x7FFFFFFF # positive 31-bit + return (h % 2147483647) / 2147483647 * window # M31 = 2^31 - 1 + def _poll_all_nodes(self) -> None: - """Fetch dashboard data from all known nodes in parallel.""" + """Fetch dashboard data from all known nodes in parallel. + + Submissions are throttled by the thread pool size to avoid a + thundering herd — at most ``max_poll_workers`` concurrent HTTP + requests are in flight at any time. Each worker sleeps a + deterministic per-node jitter (derived from its node_id) to + spread requests across the first half of the poll interval. + """ # Snapshot current auth header for this poll cycle. Per-request # headers avoid mutating shared client state (thread-safe). if self._token_manager is not None: @@ -260,8 +285,18 @@ class ClusterCollector: if not targets: return + jitter_window = self._poll_interval / 2 + + def _jittered_fetch( + nid: str, url: str, headers: dict[str, str] | None + ) -> tuple[dict[str, Any], dict[str, Any]]: + delay = self._node_jitter(nid, jitter_window) + if delay > 0.1: + time.sleep(delay) + return self._fetch_node(nid, url, headers) + futures = { - self._poll_pool.submit(self._fetch_node, nid, url, poll_headers): nid + self._poll_pool.submit(_jittered_fetch, nid, url, poll_headers): nid for nid, url in targets } for future in as_completed(futures): @@ -280,7 +315,7 @@ class ClusterCollector: if nid in self._nodes: self._nodes[nid].reachable = False except Exception: - log.debug("Failed to poll node %s", nid) + log.warning("Failed to poll node %s", nid, exc_info=True) with self._lock: if nid in self._nodes: self._nodes[nid].reachable = False @@ -300,6 +335,7 @@ class ClusterCollector: health_resp = self._http_client.get(f"{base}/health", headers=extra_headers) health_data: dict[str, Any] = health_resp.json() except Exception: + log.debug("Failed to fetch health from %s", node_id, exc_info=True) health_data = {} return dash_data, health_data @@ -407,9 +443,12 @@ class ClusterCollector: } def get_nodes( - self, sort_by: str = "activity", limit: int = 100, offset: int = 0 + self, sort_by: str = "activity", limit: int | None = 100, offset: int = 0 ) -> tuple[list[dict[str, Any]], int]: - """Return sorted, paginated node list with per-node counts.""" + """Return sorted, paginated node list with per-node counts. + + Pass ``limit=None`` to return all nodes (no pagination). + """ with self._lock: items = [] for node in self._nodes.values(): @@ -457,8 +496,15 @@ class ClusterCollector: elif sort_by == "name": items.sort(key=lambda n: n["node_id"]) + if limit is None: + return items[offset:], total return items[offset : offset + limit], total + def get_all_nodes(self) -> list[dict[str, Any]]: + """Return all nodes without pagination (for fan-out operations).""" + nodes, _ = self.get_nodes(sort_by="activity", limit=None) + return nodes + def get_workstreams( self, state: str | None = None, diff --git a/turnstone/console/server.py b/turnstone/console/server.py index c315c533..ea52cb66 100644 --- a/turnstone/console/server.py +++ b/turnstone/console/server.py @@ -168,7 +168,7 @@ def _get_server_url(request: Request, node_id: str) -> str | None: def _pick_best_node(collector: ClusterCollector) -> str: """Select the reachable node with the most available capacity.""" - nodes, _ = collector.get_nodes(sort_by="activity", limit=1000, offset=0) + nodes = collector.get_all_nodes() best_id = "" best_headroom = -1 for n in nodes: @@ -252,7 +252,7 @@ async def cluster_snapshot(request: Request) -> JSONResponse: async def cluster_events_sse(request: Request) -> Response: collector: ClusterCollector = request.app.state.collector - client_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=500) + client_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=2000) async def event_generator() -> AsyncGenerator[dict[str, str], None]: loop = asyncio.get_running_loop() @@ -682,10 +682,34 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]: # Create async HTTP clients for proxy routes. Auth headers are NOT baked # in — _proxy_auth_headers() injects a fresh token per-request so JWTs # auto-rotate via ServiceTokenManager instead of expiring after 1 hour. - app.state.proxy_client = httpx.AsyncClient(timeout=30) + # Size the pool above the fan-out limit to leave headroom for non-fan-out + # proxy traffic (UI proxying, SSE streams, etc.). + fan_out = _NODE_FAN_OUT_LIMIT + storage = getattr(app.state, "auth_storage", None) + if storage: + try: + row = storage.get_system_setting("cluster.node_fan_out_limit") + if row: + fan_out = int(row["value"]) + except Exception: + log.warning( + "Failed to read cluster.node_fan_out_limit, using default %d", + fan_out, + exc_info=True, + ) + app.state.fan_out_limit = fan_out + app.state.proxy_client = httpx.AsyncClient( + timeout=30, + limits=httpx.Limits( + max_connections=fan_out + 50, + max_keepalive_connections=min(fan_out // 4, 100), + ), + ) app.state.proxy_sse_client = httpx.AsyncClient( timeout=httpx.Timeout(connect=5, read=30, write=5, pool=5), - limits=httpx.Limits(keepalive_expiry=30), + limits=httpx.Limits( + max_connections=1100, max_keepalive_connections=100, keepalive_expiry=30 + ), ) # Start scheduler if configured scheduler = getattr(app.state, "scheduler", None) @@ -1472,10 +1496,10 @@ async def admin_list_watches(request: Request) -> JSONResponse: if err: return err collector: ClusterCollector = request.app.state.collector - nodes, _ = collector.get_nodes(limit=500) + nodes = collector.get_all_nodes() client: httpx.AsyncClient = request.app.state.proxy_client headers = _proxy_auth_headers(request) - sem = asyncio.Semaphore(_NODE_FAN_OUT_LIMIT) + sem = asyncio.Semaphore(_get_fan_out_limit(request)) async def _fetch_node(node: dict[str, Any]) -> list[dict[str, Any]]: server_url = (node.get("server_url") or "").rstrip("/") @@ -1514,9 +1538,14 @@ async def admin_list_watches(request: Request) -> JSONResponse: _VALID_WATCH_ID = re.compile(r"^[a-fA-F0-9]+$") # Max concurrent outbound requests when fanning out to cluster nodes. -# Sized below the default httpx pool limit (100) to leave headroom for -# other proxy traffic (UI proxying, SSE streams, etc.). -_NODE_FAN_OUT_LIMIT = 50 +# Must stay below the httpx pool limit (set in _lifespan) to leave +# headroom for non-fan-out proxy traffic (UI proxying, SSE streams). +_NODE_FAN_OUT_LIMIT = 200 # fallback; prefer cluster.node_fan_out_limit from storage + + +def _get_fan_out_limit(request: Request) -> int: + """Return the fan-out limit cached at startup on app.state.""" + return int(getattr(request.app.state, "fan_out_limit", _NODE_FAN_OUT_LIMIT)) async def admin_cancel_watch(request: Request) -> Response: @@ -3435,31 +3464,34 @@ async def admin_delete_memory(request: Request) -> JSONResponse: # --------------------------------------------------------------------------- -def _publish_config_change(request: Request, *, key: str, node_id: str, action: str) -> None: - """Fan out config-reload to all known server nodes (best-effort). +async def _publish_config_change(request: Request) -> None: + """Fan out config-reload to all known server nodes (best-effort, async). - Uses the collector's node registry and the existing proxy auth - mechanism — no MQ dependency. + Uses the collector's node registry, the shared async proxy client, + and bounded concurrency via the fan-out semaphore. """ - import contextlib - - import httpx - collector = getattr(request.app.state, "collector", None) if not collector: return + client: httpx.AsyncClient = request.app.state.proxy_client headers = _proxy_auth_headers(request) - with contextlib.suppress(Exception): - nodes = collector.get_nodes() - for node in nodes.get("nodes", []): - url = node.get("url", "") - if url: - with contextlib.suppress(Exception): - httpx.post( - f"{url}/v1/api/_internal/config-reload", - headers=headers, - timeout=5.0, - ) + sem = asyncio.Semaphore(_get_fan_out_limit(request)) + + async def _notify(url: str) -> None: + async with sem: + try: + await client.post( + f"{url.rstrip('/')}/v1/api/_internal/config-reload", + headers=headers, + timeout=5.0, + ) + except Exception: + log.warning("Config reload failed for %s", url, exc_info=True) + + nodes = collector.get_all_nodes() + tasks = [_notify(n["server_url"]) for n in nodes if n.get("server_url")] + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) async def admin_list_settings(request: Request) -> JSONResponse: @@ -3615,7 +3647,7 @@ async def admin_update_setting(request: Request) -> JSONResponse: ip, ) - _publish_config_change(request, key=key, node_id=node_id, action="set") + await _publish_config_change(request) return JSONResponse( { @@ -3670,7 +3702,7 @@ async def admin_delete_setting(request: Request) -> JSONResponse: ip, ) - _publish_config_change(request, key=key, node_id=node_id, action="delete") + await _publish_config_change(request) return JSONResponse({"status": "ok", "key": key}) @@ -3846,8 +3878,9 @@ async def admin_registry_install(request: Request) -> JSONResponse: # Check max servers current = storage.list_mcp_servers() - if len(current) >= _MCP_MAX_SERVERS: - return JSONResponse({"error": f"Maximum {_MCP_MAX_SERVERS} servers"}, status_code=400) + max_servers = _get_mcp_max_servers(request) + if len(current) >= max_servers: + return JSONResponse({"error": f"Maximum {max_servers} servers"}, status_code=400) # Fetch the specific server from the registry registry_url = _get_registry_url(request) @@ -3943,7 +3976,24 @@ async def admin_registry_install(request: Request) -> JSONResponse: # --------------------------------------------------------------------------- _MCP_NAME_RE = re.compile(r"^[a-zA-Z0-9._-]+$") -_MCP_MAX_SERVERS = 50 +_MCP_MAX_SERVERS = 200 # fallback; prefer cluster.mcp_max_servers from storage + + +def _get_mcp_max_servers(request: Request) -> int: + """Read cluster.mcp_max_servers from storage, falling back to the constant.""" + storage = getattr(request.app.state, "auth_storage", None) + if storage: + try: + row = storage.get_system_setting("cluster.mcp_max_servers") + if row: + return int(row["value"]) + except Exception: + log.warning( + "Failed to read cluster.mcp_max_servers, using default %d", + _MCP_MAX_SERVERS, + exc_info=True, + ) + return _MCP_MAX_SERVERS def _mask_mcp_secrets(server: dict[str, Any], reveal: bool = False) -> dict[str, Any]: @@ -3981,10 +4031,10 @@ async def _collect_mcp_status( ) -> dict[str, dict[str, dict[str, Any]]]: """Query all nodes for MCP status. Returns {node_id: {server_name: status}}.""" collector: ClusterCollector = request.app.state.collector - nodes, _ = collector.get_nodes(sort_by="activity", limit=1000, offset=0) + nodes = collector.get_all_nodes() client: httpx.AsyncClient = request.app.state.proxy_client headers = _proxy_auth_headers(request) - sem = asyncio.Semaphore(_NODE_FAN_OUT_LIMIT) + sem = asyncio.Semaphore(_get_fan_out_limit(request)) async def _fetch(node: dict[str, Any]) -> tuple[str, dict[str, dict[str, Any]] | None]: node_id = node.get("node_id", "") @@ -4128,9 +4178,10 @@ async def admin_create_mcp_server(request: Request) -> JSONResponse: # Check max servers existing = storage.list_mcp_servers() - if len(existing) >= _MCP_MAX_SERVERS: + max_servers = _get_mcp_max_servers(request) + if len(existing) >= max_servers: return JSONResponse( - {"error": f"Maximum {_MCP_MAX_SERVERS} servers"}, + {"error": f"Maximum {max_servers} servers"}, status_code=400, ) @@ -4332,10 +4383,10 @@ async def admin_delete_mcp_server(request: Request) -> JSONResponse: async def _notify_nodes_mcp_reload(request: Request) -> dict[str, Any]: """Tell all nodes to re-read the mcp_servers DB table and reconcile.""" collector: ClusterCollector = request.app.state.collector - nodes, _ = collector.get_nodes(sort_by="activity", limit=1000, offset=0) + nodes = collector.get_all_nodes() client: httpx.AsyncClient = request.app.state.proxy_client headers = _proxy_auth_headers(request) - sem = asyncio.Semaphore(_NODE_FAN_OUT_LIMIT) + sem = asyncio.Semaphore(_get_fan_out_limit(request)) async def _notify(node: dict[str, Any]) -> tuple[str, Any]: node_id = node.get("node_id", "") @@ -4411,6 +4462,7 @@ async def admin_import_mcp_config(request: Request) -> JSONResponse: errors: list[str] = [] audit_uid, ip = _audit_context(request) current_count = len(storage.list_mcp_servers()) + max_servers = _get_mcp_max_servers(request) for srv_name, cfg in servers.items(): srv_name = str(srv_name).strip()[:64] @@ -4420,7 +4472,7 @@ async def admin_import_mcp_config(request: Request) -> JSONResponse: if storage.get_mcp_server_by_name(srv_name): skipped.append(srv_name) continue - if current_count >= _MCP_MAX_SERVERS: + if current_count >= max_servers: errors.append(f"{srv_name}: max servers reached") break diff --git a/turnstone/core/auth.py b/turnstone/core/auth.py index b88be00e..ce71a2a8 100644 --- a/turnstone/core/auth.py +++ b/turnstone/core/auth.py @@ -18,7 +18,6 @@ always accessible without authentication. from __future__ import annotations -import contextlib import hashlib import hmac import json @@ -1010,7 +1009,7 @@ async def handle_auth_status(request: Request) -> Response: users = storage.list_users() has_users = len(users) > 0 except Exception: - pass + log.warning("Failed to check user existence for auth status", exc_info=True) # OIDC configuration oidc_config = getattr(request.app.state, "oidc_config", None) @@ -1079,8 +1078,10 @@ async def handle_auth_setup(request: Request, audience: str) -> Response: except Exception: log.error("Failed to assign admin role to first user %s — aborting setup", user_id) # Roll back the user creation so setup can be retried - with contextlib.suppress(Exception): + try: storage.delete_user(user_id) + except Exception: + log.error("Failed to roll back user %s during setup abort", user_id, exc_info=True) return JSONResponse( {"error": "Failed to assign admin role. Ensure migrations have run."}, status_code=503, @@ -1092,8 +1093,10 @@ async def handle_auth_setup(request: Request, audience: str) -> Response: log.error( "First user %s has no permissions after role assignment — aborting setup", user_id ) - with contextlib.suppress(Exception): + try: storage.delete_user(user_id) + except Exception: + log.error("Failed to roll back user %s during setup abort", user_id, exc_info=True) return JSONResponse( {"error": "Failed to load permissions. Ensure migrations have run."}, status_code=503, @@ -1229,8 +1232,10 @@ async def handle_oidc_callback(request: Request, audience: str) -> Response: return RedirectResponse("/?oidc_error=Too+many+login+attempts", status_code=302) # Lazy cleanup of expired pending states - with contextlib.suppress(Exception): + try: storage.cleanup_expired_oidc_states(300) + except Exception: + log.debug("OIDC state cleanup failed", exc_info=True) def _record_oidc_failure() -> None: if login_limiter is not None: diff --git a/turnstone/core/healthcheck.py b/turnstone/core/healthcheck.py index a7d77610..95ab8fef 100644 --- a/turnstone/core/healthcheck.py +++ b/turnstone/core/healthcheck.py @@ -149,7 +149,16 @@ class BackendHealthMonitor: # ------------------------------------------------------------------ def _probe_loop(self) -> None: - """Background: probe backend every interval.""" + """Background: probe backend every interval. + + An initial jitter (derived from the PID) staggers probes across + cluster nodes so they don't all hit the LLM backend at once. + """ + import os + + # Deterministic per-process jitter: spread across half the interval + jitter = ((os.getpid() * 2654435761) & 0x7FFFFFFF) / 0x7FFFFFFF * (self._probe_interval / 2) + self._stop_event.wait(jitter) while not self._stop_event.is_set(): self._stop_event.wait(self._probe_interval) if self._stop_event.is_set(): diff --git a/turnstone/core/mcp_client.py b/turnstone/core/mcp_client.py index 249cbdc3..d6dc529d 100644 --- a/turnstone/core/mcp_client.py +++ b/turnstone/core/mcp_client.py @@ -1415,7 +1415,7 @@ def create_mcp_client( if rows: db_names = {r["name"] for r in rows} except Exception: - pass + log.warning("Failed to load DB-managed MCP servers", exc_info=True) servers = load_mcp_config(config_path, storage=storage) if not servers: diff --git a/turnstone/core/memory.py b/turnstone/core/memory.py index e6ff2be8..f3aa71f2 100644 --- a/turnstone/core/memory.py +++ b/turnstone/core/memory.py @@ -3,11 +3,15 @@ All functions maintain their existing signatures for consumers (session.py, server.py, cli.py). The actual storage implementation lives in ``turnstone.core.storage``. + +The no-raise contract is preserved — callers never see exceptions from this +module. All failures are logged so storage issues are visible in logs +rather than silently swallowed. """ from __future__ import annotations -import contextlib +import logging from typing import TYPE_CHECKING, Any import sqlalchemy as sa @@ -17,6 +21,8 @@ from turnstone.core.storage import get_storage if TYPE_CHECKING: from collections.abc import Callable +log = logging.getLogger(__name__) + def normalize_key(key: str) -> str: """Normalize a memory key for consistent lookup.""" @@ -37,7 +43,7 @@ def save_message( tool_calls: str | None = None, ) -> None: """Log a message to the conversations table.""" - with contextlib.suppress(Exception): + try: get_storage().save_message( ws_id, role, @@ -48,6 +54,8 @@ def save_message( provider_data, tool_calls=tool_calls, ) + except Exception: + log.warning("Failed to save message for ws=%s role=%s", ws_id, role, exc_info=True) def load_messages(ws_id: str) -> list[dict[str, Any]]: @@ -55,6 +63,7 @@ def load_messages(ws_id: str) -> list[dict[str, Any]]: try: return get_storage().load_messages(ws_id) except Exception: + log.warning("Failed to load messages for ws=%s", ws_id, exc_info=True) return [] @@ -70,22 +79,28 @@ def register_workstream( skill_version: int = 0, ) -> None: """Persist a new workstream (no-op if already exists).""" - with contextlib.suppress(Exception): + try: get_storage().register_workstream( ws_id, node_id, name, state, skill_id=skill_id, skill_version=skill_version ) + except Exception: + log.warning("Failed to register workstream ws=%s", ws_id, exc_info=True) def update_workstream_state(ws_id: str, state: str) -> None: """Update a workstream's state.""" - with contextlib.suppress(Exception): + try: get_storage().update_workstream_state(ws_id, state) + except Exception: + log.warning("Failed to update workstream state ws=%s state=%s", ws_id, state, exc_info=True) def update_workstream_name(ws_id: str, name: str) -> None: """Update a workstream's display name.""" - with contextlib.suppress(Exception): + try: get_storage().update_workstream_name(ws_id, name) + except Exception: + log.warning("Failed to update workstream name ws=%s", ws_id, exc_info=True) def list_workstreams(node_id: str | None = None, limit: int = 100) -> list[Any]: @@ -93,6 +108,7 @@ def list_workstreams(node_id: str | None = None, limit: int = 100) -> list[Any]: try: return get_storage().list_workstreams(node_id, limit) except Exception: + log.warning("Failed to list workstreams", exc_info=True) return [] @@ -101,6 +117,7 @@ def list_workstreams_with_history(limit: int = 20) -> list[Any]: try: return get_storage().list_workstreams_with_history(limit) except Exception: + log.warning("Failed to list workstreams with history", exc_info=True) return [] @@ -109,6 +126,7 @@ def delete_workstream(ws_id: str) -> bool: try: return get_storage().delete_workstream(ws_id) except Exception: + log.warning("Failed to delete workstream ws=%s", ws_id, exc_info=True) return False @@ -120,6 +138,7 @@ def prune_workstreams( try: orphans, stale = get_storage().prune_workstreams(retention_days) except Exception: + log.warning("Failed to prune workstreams", exc_info=True) return (0, 0) if log_fn and (orphans or stale): @@ -140,6 +159,7 @@ def resolve_workstream(alias_or_id: str) -> str | None: try: return get_storage().resolve_workstream(alias_or_id) except Exception: + log.warning("Failed to resolve workstream alias=%s", alias_or_id, exc_info=True) return None @@ -148,8 +168,10 @@ def resolve_workstream(alias_or_id: str) -> str | None: def save_workstream_config(ws_id: str, config: dict[str, str]) -> None: """Persist workstream configuration key/value pairs.""" - with contextlib.suppress(Exception): + try: get_storage().save_workstream_config(ws_id, config) + except Exception: + log.warning("Failed to save workstream config ws=%s", ws_id, exc_info=True) def load_workstream_config(ws_id: str) -> dict[str, str]: @@ -157,6 +179,7 @@ def load_workstream_config(ws_id: str) -> dict[str, str]: try: return get_storage().load_workstream_config(ws_id) except Exception: + log.warning("Failed to load workstream config ws=%s", ws_id, exc_info=True) return {} @@ -168,6 +191,7 @@ def get_skill_by_name(name: str) -> dict[str, Any] | None: try: return get_storage().get_prompt_template_by_name(name) except Exception: + log.warning("Failed to get skill name=%s", name, exc_info=True) return None @@ -176,6 +200,7 @@ def list_default_skills(org_id: str = "") -> list[dict[str, Any]]: try: return get_storage().list_default_templates(org_id) except Exception: + log.warning("Failed to list default skills", exc_info=True) return [] @@ -191,6 +216,7 @@ def list_skills_by_activation( activation, enabled_only=enabled_only, limit=limit ) except Exception: + log.warning("Failed to list skills by activation=%s", activation, exc_info=True) return [] @@ -202,6 +228,7 @@ def set_workstream_alias(ws_id: str, alias: str) -> bool: try: return get_storage().set_workstream_alias(ws_id, alias) except Exception: + log.warning("Failed to set alias ws=%s alias=%s", ws_id, alias, exc_info=True) return False @@ -210,13 +237,16 @@ def get_workstream_display_name(ws_id: str) -> str | None: try: return get_storage().get_workstream_display_name(ws_id) except Exception: + log.warning("Failed to get display name ws=%s", ws_id, exc_info=True) return None def update_workstream_title(ws_id: str, title: str) -> None: """Set or update the auto-generated title for a workstream.""" - with contextlib.suppress(Exception): + try: get_storage().update_workstream_title(ws_id, title) + except Exception: + log.warning("Failed to update title ws=%s", ws_id, exc_info=True) # -- Conversation search ------------------------------------------------------- @@ -227,6 +257,7 @@ def search_history(query: str, limit: int = 20) -> list[Any]: try: return get_storage().search_history(query, limit) except Exception: + log.warning("Failed to search history", exc_info=True) return [] @@ -235,6 +266,7 @@ def search_history_recent(limit: int = 20) -> list[Any]: try: return get_storage().search_history_recent(limit) except Exception: + log.warning("Failed to search recent history", exc_info=True) return [] @@ -280,6 +312,7 @@ def save_structured_memory( return existing["memory_id"], old_content return "", None except Exception: + log.warning("Failed to save structured memory name=%s", name, exc_info=True) return "", None @@ -289,6 +322,7 @@ def delete_structured_memory(name: str, scope: str = "global", scope_id: str = " try: return get_storage().delete_structured_memory(name, scope, scope_id) except Exception: + log.warning("Failed to delete structured memory name=%s", name, exc_info=True) return False @@ -297,6 +331,7 @@ def delete_structured_memory_by_id(memory_id: str) -> bool: try: return get_storage().delete_structured_memory_by_id(memory_id) except Exception: + log.warning("Failed to delete structured memory id=%s", memory_id, exc_info=True) return False @@ -312,6 +347,7 @@ def list_structured_memories( mem_type=mem_type, scope=scope, scope_id=scope_id, limit=limit ) except Exception: + log.warning("Failed to list structured memories", exc_info=True) return [] @@ -328,6 +364,7 @@ def search_structured_memories( query, mem_type=mem_type, scope=scope, scope_id=scope_id, limit=limit ) except Exception: + log.warning("Failed to search structured memories", exc_info=True) return [] @@ -338,4 +375,5 @@ def count_structured_memories(mem_type: str = "", scope: str = "", scope_id: str mem_type=mem_type, scope=scope, scope_id=scope_id ) except Exception: + log.warning("Failed to count structured memories", exc_info=True) return 0 diff --git a/turnstone/core/model_registry.py b/turnstone/core/model_registry.py index 5af96b9b..fd5fd42f 100644 --- a/turnstone/core/model_registry.py +++ b/turnstone/core/model_registry.py @@ -278,7 +278,9 @@ def detect_model( client: Any, log_fn: Any = print, provider: str = "openai", -) -> tuple[str, int | None]: + *, + fatal: bool = True, +) -> tuple[str | None, int | None]: """Auto-detect the model and context window from the API's models endpoint. Returns ``(model_id, context_window)`` where *context_window* is @@ -289,13 +291,20 @@ def detect_model( For local single-model servers (vLLM, llama.cpp), uses the first model. Calls ``log_fn`` for informational messages (defaults to ``print``). - Raises ``SystemExit`` on failure. + + When *fatal* is ``True`` (default), raises ``SystemExit`` on failure. + When ``False``, returns ``(None, None)`` so the server can start in + degraded mode (useful for cluster deployments where the LLM backend + may not be available at startup). """ try: models = client.models.list() if not models.data: - log_fn("Error: No models found at server. Use --model to specify.") - raise SystemExit(1) + if fatal: + log_fn("Error: No models found at server. Use --model to specify.") + raise SystemExit(1) + log_fn("Warning: No models found at server — starting in degraded mode.") + return None, None all_ids = [x.id for x in models.data] selected_id = _select_best_model(all_ids, provider) @@ -321,6 +330,10 @@ def detect_model( except SystemExit: raise except Exception as e: - log_fn(f"Error: Could not connect to server: {e}") - log_fn("Is the model server running? Start it or use --base-url to point elsewhere.") - raise SystemExit(1) from e + if fatal: + log_fn(f"Error: Could not connect to server: {e}") + log_fn("Is the model server running? Start it or use --base-url to point elsewhere.") + raise SystemExit(1) from e + log_fn(f"Warning: Could not connect to LLM backend: {e}") + log_fn("Starting in degraded mode — requests will fail until backend is reachable.") + return None, None diff --git a/turnstone/core/session.py b/turnstone/core/session.py index c0636bcc..52957509 100644 --- a/turnstone/core/session.py +++ b/turnstone/core/session.py @@ -3497,8 +3497,10 @@ class ChatSession: assert proc.stdout is not None for line in proc.stdout: stdout_parts.append(line) - with contextlib.suppress(Exception): + try: self.ui.on_tool_output_chunk(call_id, line) + except Exception: + log.debug("UI callback error during tool output", exc_info=True) # Check cancellation during long-running commands if self._cancel_event.is_set(): with contextlib.suppress(OSError, ProcessLookupError): diff --git a/turnstone/core/settings_registry.py b/turnstone/core/settings_registry.py index baadb172..37f59bd7 100644 --- a/turnstone/core/settings_registry.py +++ b/turnstone/core/settings_registry.py @@ -212,13 +212,42 @@ def _build_registry() -> dict[str, SettingDef]: SettingDef( "server.max_workstreams", "int", - 10, + 50, "Max concurrent workstreams", "server", min_value=1, restart_required=True, help="Maximum number of active conversation threads on this server node. " - "When the limit is reached, the oldest idle workstream is evicted to make room.", + "When the limit is reached, the oldest idle workstream is evicted to make room. " + "Each workstream uses memory proportional to its conversation history.", + ), + # -- cluster -------------------------------------------------------- + SettingDef( + "cluster.node_fan_out_limit", + "int", + 200, + "Max concurrent outbound requests during cluster-wide operations", + "cluster", + min_value=10, + max_value=1000, + restart_required=True, + help="Controls how many nodes the console queries in parallel during " + "fan-out operations (watch listing, MCP status, reload notifications). " + "Higher values speed up large-cluster admin operations at the cost of " + "more concurrent connections. The httpx proxy pool is sized to match " + "this value (requires console restart to take effect).", + ), + SettingDef( + "cluster.mcp_max_servers", + "int", + 200, + "Max MCP server definitions in the cluster", + "cluster", + min_value=1, + max_value=2000, + help="Hard cap on the total number of MCP server definitions stored in the " + "database. Each node only connects to the servers it needs, so this " + "limit is on definitions, not active connections.", ), # -- mcp ------------------------------------------------------------ SettingDef( diff --git a/turnstone/core/storage/_migrate.py b/turnstone/core/storage/_migrate.py index dc31df60..4ab01100 100644 --- a/turnstone/core/storage/_migrate.py +++ b/turnstone/core/storage/_migrate.py @@ -49,17 +49,42 @@ def _run_with_pg_lock(engine: Any, cfg: Any) -> None: Advisory lock ID 7_475_283 (arbitrary, derived from 'turnstone'). ``pg_advisory_lock`` blocks until the lock is available, so concurrent containers wait in line rather than racing. + + Retries with jittered backoff if PostgreSQL is temporarily at + max_connections (common during large-cluster startup stampedes). """ + import random + import time + import sqlalchemy as sa from alembic import command - with engine.connect() as conn: - conn.execute(sa.text("SELECT pg_advisory_lock(7475283)")) + max_retries = 10 + for attempt in range(max_retries): try: - command.upgrade(cfg, "head") - finally: - conn.execute(sa.text("SELECT pg_advisory_unlock(7475283)")) - conn.commit() + with engine.connect() as conn: + conn.execute(sa.text("SELECT pg_advisory_lock(7475283)")) + try: + command.upgrade(cfg, "head") + finally: + conn.execute(sa.text("SELECT pg_advisory_unlock(7475283)")) + conn.commit() + return + except Exception as exc: + err_str = str(exc).lower() + if "too many clients" not in err_str and "connection" not in err_str: + raise + if attempt == max_retries - 1: + raise + delay = min(2**attempt + random.uniform(0, 1), 30) # noqa: S311 + log.warning( + "PG connection failed (attempt %d/%d), retrying in %.1fs: %s", + attempt + 1, + max_retries, + delay, + exc, + ) + time.sleep(delay) def _bootstrap_existing_sqlite(engine: Any, cfg: Any) -> None: diff --git a/turnstone/core/storage/_postgresql.py b/turnstone/core/storage/_postgresql.py index 3b796dc3..8fa92ae5 100644 --- a/turnstone/core/storage/_postgresql.py +++ b/turnstone/core/storage/_postgresql.py @@ -73,7 +73,7 @@ class PostgreSQLBackend: """PostgreSQL implementation of the StorageBackend protocol.""" def __init__( - self, url: str, pool_size: int = 5, max_overflow: int = 10, *, create_tables: bool = True + self, url: str, pool_size: int = 2, max_overflow: int = 3, *, create_tables: bool = True ) -> None: self._engine = sa.create_engine( url, diff --git a/turnstone/core/storage/_registry.py b/turnstone/core/storage/_registry.py index e9699cbc..d810544d 100644 --- a/turnstone/core/storage/_registry.py +++ b/turnstone/core/storage/_registry.py @@ -19,7 +19,7 @@ def init_storage( *, path: str = "", url: str = "", - pool_size: int = 5, + pool_size: int = 2, run_migrations: bool = True, ) -> StorageBackend: """Initialize the storage backend singleton. diff --git a/turnstone/core/workstream.py b/turnstone/core/workstream.py index 5b4a6c70..156363e9 100644 --- a/turnstone/core/workstream.py +++ b/turnstone/core/workstream.py @@ -78,7 +78,7 @@ class WorkstreamManager: self, session_factory: _SessionFactory, *, - max_workstreams: int = 10, + max_workstreams: int = 50, node_id: str | None = None, ): """ diff --git a/turnstone/eval.py b/turnstone/eval.py index 823ab32d..defe11f8 100644 --- a/turnstone/eval.py +++ b/turnstone/eval.py @@ -1513,7 +1513,9 @@ def run_optimization( if not model: from turnstone.core.model_registry import detect_model - model, _ = detect_model(client) + detected, _ = detect_model(client) + assert detected is not None # fatal=True guarantees non-None or SystemExit + model = detected # --- Optimizer model (inherits from test if not specified) --- opt_base = optimizer_base_url or base_url diff --git a/turnstone/mq/async_broker.py b/turnstone/mq/async_broker.py index 78dfb23c..fc577cae 100644 --- a/turnstone/mq/async_broker.py +++ b/turnstone/mq/async_broker.py @@ -82,6 +82,7 @@ class AsyncRedisBroker: password=self._password, decode_responses=True, retry_on_timeout=True, + max_connections=200, ) self._pubsub = self._redis.pubsub(ignore_subscribe_messages=True) @@ -264,9 +265,16 @@ class AsyncRedisBroker: await self._ensure_connected() pattern = f"{self._prefix}:node:*" prefix_len = len(f"{self._prefix}:node:") - nodes: list[dict[str, Any]] = [] + # Collect all keys first, then batch-fetch with MGET to avoid + # N+1 round-trips (1 GET per node). + keys: list[str] = [] async for key in self._r.scan_iter(match=pattern, count=100): - raw = await self._r.get(key) + keys.append(key) + if not keys: + return [] + values = await self._r.mget(keys) + nodes: list[dict[str, Any]] = [] + for key, raw in zip(keys, values, strict=True): if raw: try: meta: dict[str, Any] = json.loads(raw) diff --git a/turnstone/mq/bridge.py b/turnstone/mq/bridge.py index 13a301e0..a50b707a 100644 --- a/turnstone/mq/bridge.py +++ b/turnstone/mq/bridge.py @@ -9,7 +9,6 @@ Run as: ``turnstone-bridge --server-url http://localhost:8080`` from __future__ import annotations -import contextlib import json import logging import os @@ -800,11 +799,13 @@ class Bridge: with self._lock: self._pending_plan_reviews.pop(ws_id, None) # Best-effort rejection so the server doesn't hang - with contextlib.suppress(Exception): + try: self._http.post( "/v1/api/plan", json={"feedback": "reject", "ws_id": ws_id}, ) + except Exception: + log.warning("Failed to reject plan for ws=%s", ws_id, exc_info=True) raise threading.Thread(target=self._run_in_context(_wait_plan), daemon=True).start() @@ -894,7 +895,16 @@ class Bridge: # -- heartbeat ----------------------------------------------------------- def _heartbeat_loop(self) -> None: - """Periodically register this node in the broker.""" + """Periodically register this node in the broker. + + An initial jitter (derived from the node_id) staggers heartbeats + across cluster nodes so they don't all hit Redis at the same instant. + """ + # Deterministic per-node jitter: spread across first quarter of TTL + h = hash(self._node_id) & 0x7FFFFFFF + jitter = (h % 2147483647) / 2147483647 * (self._heartbeat_ttl / 4) + if jitter > 0.1: + time.sleep(jitter) while self._running: self._broker.register_node( self._node_id, @@ -938,9 +948,11 @@ def _iter_sse_data(resp: httpx.Response) -> Iterator[dict[str, Any]]: source = EventSource(resp) for sse in source.iter_sse(): if sse.data: - with contextlib.suppress(json.JSONDecodeError): + try: data: dict[str, Any] = json.loads(sse.data) yield data + except json.JSONDecodeError: + log.debug("Skipping malformed SSE data: %.200s", sse.data) # --------------------------------------------------------------------------- diff --git a/turnstone/mq/broker.py b/turnstone/mq/broker.py index 81969c92..e07e9103 100644 --- a/turnstone/mq/broker.py +++ b/turnstone/mq/broker.py @@ -134,6 +134,7 @@ class RedisBroker: password=password, decode_responses=True, retry_on_timeout=True, + max_connections=200, ) self._redis: _redis_t.Redis[str] = cast( "_redis_t.Redis[str]", @@ -213,9 +214,14 @@ class RedisBroker: def list_nodes(self) -> list[dict[str, Any]]: pattern = f"{self._prefix}:node:*" prefix_len = len(f"{self._prefix}:node:") + # Collect all keys first, then batch-fetch with MGET to avoid + # N+1 round-trips (1 GET per node). + keys = list(self._redis.scan_iter(match=pattern, count=100)) + if not keys: + return [] + values = self._redis.mget(keys) nodes: list[dict[str, Any]] = [] - for key in self._redis.scan_iter(match=pattern, count=100): - raw = self._redis.get(key) + for key, raw in zip(keys, values, strict=True): if raw: try: meta: dict[str, Any] = json.loads(raw) diff --git a/turnstone/server.py b/turnstone/server.py index 0bf1a184..8aee1849 100644 --- a/turnstone/server.py +++ b/turnstone/server.py @@ -25,6 +25,7 @@ import textwrap import threading import time import uuid +from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager from pathlib import Path from typing import TYPE_CHECKING, Any @@ -430,7 +431,7 @@ class WebUI: cache_read_tokens=cache_read, ) except Exception: - pass # Non-critical — never break the response pipeline + log.warning("Failed to record usage event", exc_info=True) def on_plan_review(self, content: str) -> str: self._plan_event.clear() @@ -798,10 +799,11 @@ async def events_sse(request: Request) -> Response: _metrics.record_sse_connect() try: loop = asyncio.get_running_loop() + executor = request.app.state.sse_executor while True: try: event = await loop.run_in_executor( - None, functools.partial(client_queue.get, timeout=5) + executor, functools.partial(client_queue.get, timeout=5) ) if event.get("type") == "ws_closed": return @@ -817,7 +819,7 @@ async def events_sse(request: Request) -> Response: async def global_events_sse(request: Request) -> Response: """GET /v1/api/events/global — global SSE event stream.""" - client_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=500) + client_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=1000) listeners = request.app.state.global_listeners listeners_lock = request.app.state.global_listeners_lock with listeners_lock: @@ -827,10 +829,11 @@ async def global_events_sse(request: Request) -> Response: _metrics.record_sse_connect() try: loop = asyncio.get_running_loop() + executor = request.app.state.sse_executor while True: try: event = await loop.run_in_executor( - None, functools.partial(client_queue.get, timeout=5) + executor, functools.partial(client_queue.get, timeout=5) ) yield {"data": json.dumps(event)} except queue.Empty: @@ -1744,7 +1747,7 @@ def _global_fanout_thread( with contextlib.suppress(queue.Full): lq.put_nowait(event) # drop if a listener is backed up except Exception: - pass + log.debug("Global fan-out error", exc_info=True) # --------------------------------------------------------------------------- @@ -1755,6 +1758,9 @@ def _global_fanout_thread( @asynccontextmanager async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]: """Start background threads and handle shutdown.""" + # Dedicated executor for SSE queue polling so it doesn't compete + # with the default asyncio executor (which caps at ~32 workers). + app.state.sse_executor = ThreadPoolExecutor(max_workers=200, thread_name_prefix="sse") # Start global event fan-out thread fanout = threading.Thread( target=_global_fanout_thread, @@ -1817,6 +1823,7 @@ async def _lifespan(app: Starlette) -> AsyncGenerator[None, None]: app.state.mcp_client.shutdown() if app.state.registry: app.state.registry.shutdown() + app.state.sse_executor.shutdown(wait=True, cancel_futures=True) # --------------------------------------------------------------------------- @@ -2038,7 +2045,7 @@ def main() -> None: db_url = getattr(args, "db_url", None) or os.environ.get("TURNSTONE_DB_URL", "") db_path = getattr(args, "db_path", None) or os.environ.get("TURNSTONE_DB_PATH", "") db_pool_size = int( - getattr(args, "db_pool_size", None) or os.environ.get("TURNSTONE_DB_POOL_SIZE", "5") + getattr(args, "db_pool_size", None) or os.environ.get("TURNSTONE_DB_POOL_SIZE", "2") ) init_storage(db_backend, path=db_path, url=db_url, pool_size=db_pool_size) @@ -2100,7 +2107,12 @@ def main() -> None: else: from turnstone.core.model_registry import detect_model - model, detected_ctx = detect_model(client, provider=provider_name) + model, detected_ctx = detect_model(client, provider=provider_name, fatal=False) + if model is None: + # LLM backend unreachable — start with a placeholder model name. + # The health monitor will report degraded and the circuit breaker + # will prevent requests until the backend comes up. + model = "unavailable" # Use detected context window, fall back to ConfigStore override or 32768 cfg_ctx = config_store.get("model.context_window")