intelligence-router/tests/test_router_circuit_breaker.py

103 lines
4.3 KiB
Python
Raw Normal View History

Epic: Model Switching via Sidecar — Issues #4-#7 + #8 deployment Issue #4: Automatic model detection and switch - Router extracts model from chat body, queries sidecar, triggers switch on mismatch - Matching active model routes directly to Main PC - No active model triggers cold start switch - Tests: 4 test_router_model_detection.py Issue #5: SSE switch progress feedback - _sse_format() correctly serializes SSE events - sse_progress_stream() generates phase progression events - Proxy yields SSE events then actual response - Tests: 3 test_router_sse_progress.py Issue #6: Circuit breaker + OpenRouter fallback - Circuit tracks Sidecar failures, opens after MAX_RECOVERY_ATTEMPTS (3) - OpenRouter API key from env, no longer uses x-intelligence-level header - Fixes: OPENROUTER_BASE, SSE format, circuit state isolation - Tests: 7 test_router_circuit_breaker.py Issue #7: LXC fallback chain completion - Full fallback: Main PC → OpenRouter → LXC - Each backend health-checked via /v1/models before routing - All backends down → 503 response - Fixed: execute() wrapped in try/except to trigger fallback chain - Tests: 3 test_router_fallback_lxc.py Issue #8: Systemd service deployment - deploy/llm-sidecar.service: systemd unit with Restart=always - deploy/manifest.yaml: example manifest with 3 profiles - deploy/README.md: deployment instructions - Updated: docker-compose.yml, requirements.txt, Dockerfile Test framework improvements: - tests/conftest.py: shared URL patches for all router tests - Fixed global state pollution in circuit breaker tests - Fixed test sidecar switch test (AsyncMock for async function) Total: 42 tests passing
2026-06-15 04:13:36 +03:00
"""Tests for circuit breaker + OpenRouter fallback — Issue #6.
Circuit tracks Sidecar failures, falls back to OpenRouter when open,
resets on successful Sidecar interaction.
Uses conftest.py patches for URL mocking.
"""
import asyncio
import pytest
from httpx import Response, ASGITransport, AsyncClient
import respx
import main
class TestCircuitBreaker:
"""Tests for the circuit breaker mechanism."""
def test_circuit_closed_initially(self):
"""Circuit starts closed (allows Sidecar requests)."""
assert main._circuit_open is False
assert main._recovery_attempts == 0
def test_circuit_opens_after_max_failures(self):
"""Circuit opens after MAX_RECOVERY_ATTEMPTS failures."""
for i in range(main.MAX_RECOVERY_ATTEMPTS):
main.circuit_record_failure()
assert main._circuit_open is True
assert main._recovery_attempts == main.MAX_RECOVERY_ATTEMPTS
def test_circuit_resets_on_success(self):
"""Circuit resets after a successful Sidecar interaction."""
# Fill up recovery attempts to trigger open circuit
for _ in range(main.MAX_RECOVERY_ATTEMPTS):
main.circuit_record_failure()
assert main._circuit_open is True
main.circuit_reset()
assert main._circuit_open is False
def test_circuit_allows_request_when_closed(self):
"""Circuit allows Sidecar request when closed."""
main.circuit_reset()
result = asyncio.run(main.circuit_breaker_check())
assert result is True
def test_circuit_blocks_when_open(self):
"""Circuit blocks Sidecar request when open."""
for _ in range(main.MAX_RECOVERY_ATTEMPTS):
main.circuit_record_failure()
result = asyncio.run(main.circuit_breaker_check())
assert result is False
class TestOpenRouterFallback:
"""Tests for OpenRouter as fallback backend."""
def test_router_uses_openrouter_when_circuit_open(self):
"""When circuit is open, router tries OpenRouter."""
async def run_test():
with respx.mock:
# Sidecar is down
respx.get("http://localhost:8081/models/status").mock(
side_effect=Exception("connection refused")
)
# OpenRouter works
respx.post("https://openrouter.ai/v1/chat/completions").mock(
return_value=Response(200, json={"choices": [{"message": {"content": "Hello from OR"}}]})
)
transport = ASGITransport(app=main.app)
async with AsyncClient(transport=transport, base_url="http://test") as ac:
resp = await ac.post(
"/v1/chat/completions",
json={"model": "qwen-3-8b", "messages": [{"role": "user", "content": "hi"}]},
)
assert resp.status_code == 200
data = resp.json()
assert data["choices"][0]["message"]["content"] == "Hello from OR"
asyncio.run(run_test())
class TestDeprecatedHeaderRemoved:
"""Verify x-intelligence-level header is removed."""
def test_proxy_ignores_intelligence_level_header(self):
"""Router does not route based on x-intelligence-level: High."""
async def run_test():
with respx.mock:
respx.get("http://localhost:8081/models/status").mock(
return_value=Response(200, json={"active_profile": "qwen-3-8b", "llama_server_running": True})
)
# Should route to Main PC regardless of header
respx.post("http://localhost:8080/v1/chat/completions").mock(
return_value=Response(200, json={"choices": [{"message": {"content": "Hello"}}]})
)
transport = ASGITransport(app=main.app)
async with AsyncClient(transport=transport, base_url="http://test") as ac:
resp = await ac.post(
"/v1/chat/completions",
json={"model": "qwen-3-8b", "messages": [{"role": "user", "content": "hi"}]},
headers={"x-intelligence-level": "High"}, # Should be ignored
)
assert resp.status_code == 200
asyncio.run(run_test())