"""End-to-end harness: a throwaway backend, the built SPA and a real browser.
Run it with::
uv run pytest e2e -q
The directory sits outside ``testpaths = ["tests"]`` so the regular backend run
never pays for it. One run gets:
* a throwaway PostgreSQL database (created and dropped by the fixture);
* a throwaway working directory used as the server's CWD, which also means
``mcp_servers.d/`` is not found and no MCP server is started;
* its own temp ``workspace/`` and ``session_files/`` directories;
* ``NAVI_AUTH_ENABLED=false``, so every request is the local anonymous admin;
* the real LLM route from ``llm.json`` and the real credentials from ``.env``.
Nothing here touches the developer's running stack, the clone database or any
file in the repository.
"""
from __future__ import annotations
import os
import shutil
import socket
import subprocess
import sys
import time
import uuid
from pathlib import Path
from types import SimpleNamespace
from urllib import error, request
from urllib.parse import urlsplit
import pytest
from dotenv import dotenv_values
from support import SERVER_TIMEOUT
REPO = Path(__file__).resolve().parents[1]
DIST = REPO / "webclient" / "dist"
ENV_FILE = REPO / ".env"
CHROMIUM = os.environ.get("E2E_CHROMIUM", "/usr/bin/chromium")
SHOTS_DIR = Path(os.environ.get("E2E_SHOTS_DIR", "/tmp/navi-e2e-shots"))
def _free_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.bind(("127.0.0.1", 0))
return sock.getsockname()[1]
def _database_url() -> str:
"""The DSN the local dev stack uses, so the e2e DB sits on the same host."""
dotenv = dotenv_values(ENV_FILE)
return (
os.environ.get("E2E_DATABASE_URL")
or dotenv.get("DATABASE_URL")
or os.environ.get("DATABASE_URL")
or ""
).strip()
def _psql(dsn: str, sql: str, database: str = "postgres") -> subprocess.CompletedProcess:
parts = urlsplit(dsn)
env = os.environ.copy()
if parts.password:
env["PGPASSWORD"] = parts.password
command = [
"psql",
"-h", parts.hostname or "127.0.0.1",
"-p", str(parts.port or 5432),
"-U", parts.username or os.environ.get("USER", ""),
"-d", database,
"-v", "ON_ERROR_STOP=1",
"-c", sql,
]
return subprocess.run(
command, env=env, capture_output=True, text=True, timeout=60, check=False
)
def _tail(path: Path, lines: int = 80) -> str:
try:
return "\n".join(path.read_text(errors="replace").splitlines()[-lines:])
except OSError:
return "(no log)"
def _assert_frontend_built() -> None:
"""Fail loudly on a stale SPA build instead of quietly testing old assets."""
index = DIST / "index.html"
if not index.exists():
pytest.fail("webclient/dist is missing — run `npm run build` in webclient/")
newest = 0.0
for path in (REPO / "webclient" / "src").rglob("*"):
if path.is_file():
newest = max(newest, path.stat().st_mtime)
for name in ("package.json", "vite.config.js"):
candidate = REPO / "webclient" / name
if candidate.exists():
newest = max(newest, candidate.stat().st_mtime)
if newest > index.stat().st_mtime + 1:
pytest.fail(
"webclient/dist is older than webclient/src — "
"run `npm run build` in webclient/ before the e2e run"
)
@pytest.fixture(scope="session")
def e2e_database() -> str:
dsn = _database_url()
if not dsn:
pytest.skip("no DATABASE_URL (set E2E_DATABASE_URL or put it in .env)")
if not shutil.which("psql"):
pytest.skip("psql not found (install the postgresql client)")
name = f"navidb_e2e_{os.getpid()}_{uuid.uuid4().hex[:8]}"
created = _psql(dsn, f'CREATE DATABASE "{name}"')
if created.returncode != 0:
pytest.fail(f"could not create the e2e database: {created.stderr.strip()}")
parts = urlsplit(dsn)
e2e_dsn = parts._replace(path="/" + name).geturl()
try:
yield e2e_dsn
finally:
dropped = _psql(dsn, f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)')
if dropped.returncode != 0:
# PostgreSQL < 13 has no FORCE; disconnect the server first.
_psql(
dsn,
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
f"WHERE datname = '{name}'",
)
_psql(dsn, f'DROP DATABASE IF EXISTS "{name}"')
@pytest.fixture(scope="session")
def e2e_server(e2e_database: str, tmp_path_factory) -> SimpleNamespace:
_assert_frontend_built()
run_root = tmp_path_factory.mktemp("navi-e2e")
cwd = run_root / "cwd"
workspace = run_root / "workspace"
session_files = run_root / "session_files"
for directory in (cwd, workspace, session_files):
directory.mkdir()
port = _free_port()
base_url = f"http://127.0.0.1:{port}"
env = os.environ.copy()
# Mirror the developer's .env first (that is where the Ollama key lives),
# then override everything the harness must isolate.
env.update({key: value for key, value in dotenv_values(ENV_FILE).items() if value is not None})
env.update({
"NAVI_AUTH_ENABLED": "false",
"NAVI_OVERLAY": "off",
"NAVI_UI_MCP_ENABLED": "false",
"NAVI_HOST": "127.0.0.1",
"NAVI_PORT": str(port),
"PUBLIC_URL": base_url,
"DATABASE_URL": e2e_database,
"FS_ALLOWED_PATHS": str(workspace),
"SESSION_FILES_DIR": str(session_files),
"PYTHONUNBUFFERED": "1",
"LOG_LEVEL": os.environ.get("E2E_LOG_LEVEL", "INFO"),
# Even if the developer's .env has an export consumer configured, the
# throwaway database must not produce export files.
"NAVI_EXPORT_AGE_RECIPIENT": "",
"NAVI_EXPORT_TOKEN": "",
})
log_path = run_root / "server.log"
log_file = log_path.open("wb")
proc = subprocess.Popen(
[sys.executable, "-m", "navi.server"],
cwd=str(cwd),
env=env,
stdout=log_file,
stderr=subprocess.STDOUT,
start_new_session=True,
)
try:
deadline = time.monotonic() + SERVER_TIMEOUT
while time.monotonic() < deadline:
if proc.poll() is not None:
pytest.fail(
f"navi-server exited during startup (code {proc.returncode})\n"
f"{_tail(log_path)}"
)
try:
with request.urlopen(f"{base_url}/health", timeout=2) as resp:
if resp.status == 200:
break
except (error.URLError, OSError):
pass
time.sleep(0.5)
else:
pytest.fail(
f"navi-server did not become healthy in {SERVER_TIMEOUT:.0f}s\n"
f"{_tail(log_path)}"
)
yield SimpleNamespace(
base_url=base_url,
workspace=workspace,
session_files=session_files,
log_path=log_path,
proc=proc,
cwd=cwd,
)
finally:
proc.terminate()
try:
proc.wait(timeout=15)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait(timeout=10)
log_file.close()
@pytest.fixture(scope="session")
def browser():
if not Path(CHROMIUM).exists():
pytest.skip(f"chromium not found at {CHROMIUM} (set E2E_CHROMIUM)")
playwright = pytest.importorskip("playwright.sync_api")
with playwright.sync_playwright() as p:
instance = p.chromium.launch(
headless=True,
executable_path=CHROMIUM,
args=["--no-sandbox"],
)
try:
yield instance
finally:
instance.close()
@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
"""Save a screenshot, the DOM and the browser log when a test fails.
pytest does not throw the test's exception into fixture finalizers, so the
artefacts have to be captured from the report hook. The ``page`` fixture
leaves itself and its console log on the item for this hook to find.
"""
outcome = yield
report = outcome.get_result()
if report.when != "call" or not report.failed:
return
page = getattr(item, "_e2e_page", None)
if page is None:
return
SHOTS_DIR.mkdir(parents=True, exist_ok=True)
stem = f"{item.name}-{time.strftime('%H%M%S')}"
screenshot = SHOTS_DIR / f"{stem}.png"
try:
page.screenshot(path=str(screenshot), full_page=True)
except Exception: # noqa: BLE001 — best-effort artefact capture
screenshot = None
dom = SHOTS_DIR / f"{stem}.html"
try:
dom.write_text(page.content())
except Exception: # noqa: BLE001 — best-effort artefact capture
dom = None
events = getattr(item, "_e2e_events", [])
diagnostics = SHOTS_DIR / f"{stem}.txt"
diagnostics.write_text("\n".join(events) or "(no browser errors)")
print(
f"\n[e2e] screenshot: {screenshot}\n[e2e] DOM: {dom}\n"
f"[e2e] browser log: {diagnostics}"
)
@pytest.fixture
def page(browser, request):
"""A fresh browser context per test.
Service workers are blocked so a previous build's cache never answers a
request during the run.
"""
context = browser.new_context(
viewport={"width": 1440, "height": 900},
locale="en-US",
service_workers="block",
)
page = context.new_page()
page.set_default_timeout(30_000)
request.node._e2e_page = page
events: list[str] = []
request.node._e2e_events = events
page.on(
"console",
lambda message: events.append(f"console.{message.type}: {message.text}")
if message.type == "error" else None,
)
page.on("pageerror", lambda exc: events.append(f"pageerror: {exc}"))
page.on(
"requestfailed",
lambda req: events.append(f"requestfailed: {req.method} {req.url}: {req.failure}"),
)
try:
yield page
finally:
context.close()
@pytest.fixture
def workdir(e2e_server: SimpleNamespace) -> Path:
"""A private directory inside the agent's allowed workspace."""
path = e2e_server.workspace / uuid.uuid4().hex[:10]
path.mkdir()
return path