Newer
Older
navi-1 / e2e / conftest.py
"""End-to-end harness: a throwaway backend, the built SPA and a real browser.

Run it with::

    uv run pytest e2e -q

The directory sits outside ``testpaths = ["tests"]`` so the regular backend run
never pays for it. One run gets:

* a throwaway PostgreSQL database (created and dropped by the fixture);
* a throwaway working directory used as the server's CWD, which also means
  ``mcp_servers.d/`` is not found and no MCP server is started;
* its own temp ``workspace/`` and ``session_files/`` directories;
* ``NAVI_AUTH_ENABLED=false``, so every request is the local anonymous admin;
* the real LLM route from ``llm.json`` and the real credentials from ``.env``.

Nothing here touches the developer's running stack, the clone database or any
file in the repository.
"""

from __future__ import annotations

import os
import shutil
import socket
import subprocess
import sys
import time
import uuid
from pathlib import Path
from types import SimpleNamespace
from urllib import error, request
from urllib.parse import urlsplit

import pytest
from dotenv import dotenv_values
from support import SERVER_TIMEOUT

REPO = Path(__file__).resolve().parents[1]
DIST = REPO / "webclient" / "dist"
ENV_FILE = REPO / ".env"
CHROMIUM = os.environ.get("E2E_CHROMIUM", "/usr/bin/chromium")
SHOTS_DIR = Path(os.environ.get("E2E_SHOTS_DIR", "/tmp/navi-e2e-shots"))


def _free_port() -> int:
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
        sock.bind(("127.0.0.1", 0))
        return sock.getsockname()[1]


def _database_url() -> str:
    """The DSN the local dev stack uses, so the e2e DB sits on the same host."""
    dotenv = dotenv_values(ENV_FILE)
    return (
        os.environ.get("E2E_DATABASE_URL")
        or dotenv.get("DATABASE_URL")
        or os.environ.get("DATABASE_URL")
        or ""
    ).strip()


def _psql(dsn: str, sql: str, database: str = "postgres") -> subprocess.CompletedProcess:
    parts = urlsplit(dsn)
    env = os.environ.copy()
    if parts.password:
        env["PGPASSWORD"] = parts.password
    command = [
        "psql",
        "-h", parts.hostname or "127.0.0.1",
        "-p", str(parts.port or 5432),
        "-U", parts.username or os.environ.get("USER", ""),
        "-d", database,
        "-v", "ON_ERROR_STOP=1",
        "-c", sql,
    ]
    return subprocess.run(
        command, env=env, capture_output=True, text=True, timeout=60, check=False
    )


def _tail(path: Path, lines: int = 80) -> str:
    try:
        return "\n".join(path.read_text(errors="replace").splitlines()[-lines:])
    except OSError:
        return "(no log)"


def _assert_frontend_built() -> None:
    """Fail loudly on a stale SPA build instead of quietly testing old assets."""
    index = DIST / "index.html"
    if not index.exists():
        pytest.fail("webclient/dist is missing — run `npm run build` in webclient/")
    newest = 0.0
    for path in (REPO / "webclient" / "src").rglob("*"):
        if path.is_file():
            newest = max(newest, path.stat().st_mtime)
    for name in ("package.json", "vite.config.js"):
        candidate = REPO / "webclient" / name
        if candidate.exists():
            newest = max(newest, candidate.stat().st_mtime)
    if newest > index.stat().st_mtime + 1:
        pytest.fail(
            "webclient/dist is older than webclient/src — "
            "run `npm run build` in webclient/ before the e2e run"
        )


@pytest.fixture(scope="session")
def e2e_database() -> str:
    dsn = _database_url()
    if not dsn:
        pytest.skip("no DATABASE_URL (set E2E_DATABASE_URL or put it in .env)")
    if not shutil.which("psql"):
        pytest.skip("psql not found (install the postgresql client)")

    name = f"navidb_e2e_{os.getpid()}_{uuid.uuid4().hex[:8]}"
    created = _psql(dsn, f'CREATE DATABASE "{name}"')
    if created.returncode != 0:
        pytest.fail(f"could not create the e2e database: {created.stderr.strip()}")

    parts = urlsplit(dsn)
    e2e_dsn = parts._replace(path="/" + name).geturl()
    try:
        yield e2e_dsn
    finally:
        dropped = _psql(dsn, f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)')
        if dropped.returncode != 0:
            # PostgreSQL < 13 has no FORCE; disconnect the server first.
            _psql(
                dsn,
                "SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
                f"WHERE datname = '{name}'",
            )
            _psql(dsn, f'DROP DATABASE IF EXISTS "{name}"')


@pytest.fixture(scope="session")
def e2e_server(e2e_database: str, tmp_path_factory) -> SimpleNamespace:
    _assert_frontend_built()

    run_root = tmp_path_factory.mktemp("navi-e2e")
    cwd = run_root / "cwd"
    workspace = run_root / "workspace"
    session_files = run_root / "session_files"
    for directory in (cwd, workspace, session_files):
        directory.mkdir()

    port = _free_port()
    base_url = f"http://127.0.0.1:{port}"

    env = os.environ.copy()
    # Mirror the developer's .env first (that is where the Ollama key lives),
    # then override everything the harness must isolate.
    env.update({key: value for key, value in dotenv_values(ENV_FILE).items() if value is not None})
    env.update({
        "NAVI_AUTH_ENABLED": "false",
        "NAVI_OVERLAY": "off",
        "NAVI_UI_MCP_ENABLED": "false",
        "NAVI_HOST": "127.0.0.1",
        "NAVI_PORT": str(port),
        "PUBLIC_URL": base_url,
        "DATABASE_URL": e2e_database,
        "FS_ALLOWED_PATHS": str(workspace),
        "SESSION_FILES_DIR": str(session_files),
        "PYTHONUNBUFFERED": "1",
        "LOG_LEVEL": os.environ.get("E2E_LOG_LEVEL", "INFO"),
        # Even if the developer's .env has an export consumer configured, the
        # throwaway database must not produce export files.
        "NAVI_EXPORT_AGE_RECIPIENT": "",
        "NAVI_EXPORT_TOKEN": "",
    })

    log_path = run_root / "server.log"
    log_file = log_path.open("wb")
    proc = subprocess.Popen(
        [sys.executable, "-m", "navi.server"],
        cwd=str(cwd),
        env=env,
        stdout=log_file,
        stderr=subprocess.STDOUT,
        start_new_session=True,
    )

    try:
        deadline = time.monotonic() + SERVER_TIMEOUT
        while time.monotonic() < deadline:
            if proc.poll() is not None:
                pytest.fail(
                    f"navi-server exited during startup (code {proc.returncode})\n"
                    f"{_tail(log_path)}"
                )
            try:
                with request.urlopen(f"{base_url}/health", timeout=2) as resp:
                    if resp.status == 200:
                        break
            except (error.URLError, OSError):
                pass
            time.sleep(0.5)
        else:
            pytest.fail(
                f"navi-server did not become healthy in {SERVER_TIMEOUT:.0f}s\n"
                f"{_tail(log_path)}"
            )

        yield SimpleNamespace(
            base_url=base_url,
            workspace=workspace,
            session_files=session_files,
            log_path=log_path,
            proc=proc,
            cwd=cwd,
        )
    finally:
        proc.terminate()
        try:
            proc.wait(timeout=15)
        except subprocess.TimeoutExpired:
            proc.kill()
            proc.wait(timeout=10)
        log_file.close()


@pytest.fixture(scope="session")
def browser():
    if not Path(CHROMIUM).exists():
        pytest.skip(f"chromium not found at {CHROMIUM} (set E2E_CHROMIUM)")
    playwright = pytest.importorskip("playwright.sync_api")
    with playwright.sync_playwright() as p:
        instance = p.chromium.launch(
            headless=True,
            executable_path=CHROMIUM,
            args=["--no-sandbox"],
        )
        try:
            yield instance
        finally:
            instance.close()


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
    """Save a screenshot, the DOM and the browser log when a test fails.

    pytest does not throw the test's exception into fixture finalizers, so the
    artefacts have to be captured from the report hook. The ``page`` fixture
    leaves itself and its console log on the item for this hook to find.
    """
    outcome = yield
    report = outcome.get_result()
    if report.when != "call" or not report.failed:
        return
    page = getattr(item, "_e2e_page", None)
    if page is None:
        return
    SHOTS_DIR.mkdir(parents=True, exist_ok=True)
    stem = f"{item.name}-{time.strftime('%H%M%S')}"
    screenshot = SHOTS_DIR / f"{stem}.png"
    try:
        page.screenshot(path=str(screenshot), full_page=True)
    except Exception:  # noqa: BLE001 — best-effort artefact capture
        screenshot = None
    dom = SHOTS_DIR / f"{stem}.html"
    try:
        dom.write_text(page.content())
    except Exception:  # noqa: BLE001 — best-effort artefact capture
        dom = None
    events = getattr(item, "_e2e_events", [])
    diagnostics = SHOTS_DIR / f"{stem}.txt"
    diagnostics.write_text("\n".join(events) or "(no browser errors)")
    print(
        f"\n[e2e] screenshot: {screenshot}\n[e2e] DOM: {dom}\n"
        f"[e2e] browser log: {diagnostics}"
    )


@pytest.fixture
def page(browser, request):
    """A fresh browser context per test.

    Service workers are blocked so a previous build's cache never answers a
    request during the run.
    """
    context = browser.new_context(
        viewport={"width": 1440, "height": 900},
        locale="en-US",
        service_workers="block",
    )
    page = context.new_page()
    page.set_default_timeout(30_000)
    request.node._e2e_page = page

    events: list[str] = []
    request.node._e2e_events = events
    page.on(
        "console",
        lambda message: events.append(f"console.{message.type}: {message.text}")
        if message.type == "error" else None,
    )
    page.on("pageerror", lambda exc: events.append(f"pageerror: {exc}"))
    page.on(
        "requestfailed",
        lambda req: events.append(f"requestfailed: {req.method} {req.url}: {req.failure}"),
    )

    try:
        yield page
    finally:
        context.close()


@pytest.fixture
def workdir(e2e_server: SimpleNamespace) -> Path:
    """A private directory inside the agent's allowed workspace."""
    path = e2e_server.workspace / uuid.uuid4().hex[:10]
    path.mkdir()
    return path