"""The multi-server Ollama fallback.
A connection error is a statement about the moment, not about the server: on
2026-10-08 the only configured server (ollama.com) timed out 27 times in a day,
and because complete() made no second attempt, every one of those aborted the
caller — memory summarisation and planning included — with
"All backends exhausted: ReadTimeout".
"""
import pytest
from navi.config import Settings
from navi.exceptions import LLMBackendError, LLMConnectionError, LLMModelNotFoundError
from navi.llm import fallback as fb
from navi.llm.base import LLMResponse, Message
from navi.llm.fallback import FallbackOllamaBackend, ServerEntry
CLOUD = "https://ollama.com"
LOCAL = "http://localhost:11434"
class FakeClient:
"""Fails the first `fail_times` calls with `error`, then succeeds."""
def __init__(self, fail_times: int = 0, error: Exception | None = None):
self.calls = 0
self.fail_times = fail_times
self.error = error or LLMConnectionError("ReadTimeout")
async def complete(self, messages, **kwargs):
self.calls += 1
if self.calls <= self.fail_times:
raise self.error
return LLMResponse(content="ok", tool_calls=None, finish_reason="stop", model=None)
async def embed(self, texts, model=None):
self.calls += 1
if self.calls <= self.fail_times:
raise self.error
return [[0.1] * 4 for _ in texts]
@pytest.fixture(autouse=True)
def _fast_retries(monkeypatch):
"""No sleeping between attempts, and the retry count the prod config means.
Settings is a frozen model, so the fixture swaps in a fresh instance rather
than poking attributes at the shared singleton.
"""
monkeypatch.setattr(fb, "settings", retry_settings(2, 0.0))
def retry_settings(retries: int, backoff: float) -> Settings:
return Settings(
_env_file=None, llm_complete_retries=retries, llm_retry_backoff_sec=backoff
)
@pytest.fixture(autouse=True)
def _clean_blacklists():
fb._dead_servers.clear()
fb._dead_models.clear()
yield
fb._dead_servers.clear()
fb._dead_models.clear()
def backend(monkeypatch, servers, clients):
b = FallbackOllamaBackend([ServerEntry(host=h) for h in servers])
monkeypatch.setattr(b, "_get_client", lambda server: clients[server.host])
return b
class TestCompleteRetry:
async def test_a_transient_timeout_is_retried_on_the_same_server(self, monkeypatch):
client = FakeClient(fail_times=1)
b = backend(monkeypatch, [CLOUD], {CLOUD: client})
resp = await b.complete([Message(role="user", content="hi")], model="m")
assert resp.content == "ok"
assert client.calls == 2
async def test_persistent_failure_still_exhausts_the_backend(self, monkeypatch):
client = FakeClient(fail_times=99)
b = backend(monkeypatch, [CLOUD], {CLOUD: client})
with pytest.raises(LLMBackendError):
await b.complete([Message(role="user", content="hi")], model="m")
assert client.calls == fb.settings.llm_complete_retries
async def test_the_only_server_is_not_blacklisted(self, monkeypatch):
"""Blacklisting it would block the next request for _TTL even though the
timeout says nothing about the server's health."""
client = FakeClient(fail_times=99)
b = backend(monkeypatch, [CLOUD], {CLOUD: client})
with pytest.raises(LLMBackendError):
await b.complete([Message(role="user", content="hi")], model="m")
assert fb._is_dead_server(CLOUD) is False
async def test_a_dead_server_falls_through_to_the_next_one(self, monkeypatch):
dead, alive = FakeClient(fail_times=99), FakeClient()
b = backend(monkeypatch, [CLOUD, LOCAL], {CLOUD: dead, LOCAL: alive})
resp = await b.complete([Message(role="user", content="hi")], model="m")
assert resp.content == "ok"
assert dead.calls == fb.settings.llm_complete_retries
assert alive.calls == 1
assert fb._is_dead_server(CLOUD) is True
async def test_a_missing_model_is_not_retried(self, monkeypatch):
"""A 404 is an answer, not a hiccup — retrying it just wastes a timeout."""
client = FakeClient(fail_times=99, error=LLMModelNotFoundError("model not found"))
b = backend(monkeypatch, [CLOUD], {CLOUD: client})
with pytest.raises(LLMBackendError):
await b.complete([Message(role="user", content="hi")], model="m")
assert client.calls == 1
async def test_retries_can_be_switched_off(self, monkeypatch):
monkeypatch.setattr(fb, "settings", retry_settings(1, 0.0))
client = FakeClient(fail_times=99)
b = backend(monkeypatch, [CLOUD], {CLOUD: client})
with pytest.raises(LLMBackendError):
await b.complete([Message(role="user", content="hi")], model="m")
assert client.calls == 1
async def test_the_resolved_model_is_still_stamped(self, monkeypatch):
"""The retry loop must not lose the model the agent surfaces to the UI."""
client = FakeClient(fail_times=1)
b = backend(monkeypatch, [CLOUD], {CLOUD: client})
resp = await b.complete([Message(role="user", content="hi")], model="glm-5.3-flash:cloud")
assert resp.model == "glm-5.3-flash:cloud"
class TestEmbedRetry:
async def test_a_transient_timeout_is_retried(self, monkeypatch):
client = FakeClient(fail_times=1)
b = backend(monkeypatch, [CLOUD], {CLOUD: client})
vectors = await b.embed(["hello"], model="nomic-embed-text")
assert len(vectors) == 1
assert client.calls == 2