diff --git a/docs/e2e.md b/docs/e2e.md new file mode 100644 index 0000000..ba9643e --- /dev/null +++ b/docs/e2e.md @@ -0,0 +1,82 @@ +# End-to-end tests + +The e2e suite drives the real stack in a browser: a real Navi backend, the +built webclient, a real model from `llm.json`, and the actual file tools. A +test sends a message, the agent calls `filesystem_read` / `filesystem_write` / +`file_ai`, and the assertions check both the rendered tool cards and the file +on disk. + +It lives in `e2e/`, outside `testpaths = ["tests"]`, so the normal suite never +collects it. Run it explicitly: + +```bash +uv run pytest e2e -q +``` + +## What one run does + +``` +e2e/conftest.py # database, server process, browser, per-test screenshots +e2e/support.py # session/HTTP/UI helpers +e2e/test_agent_file_tools.py +``` + +Per run the fixtures: + +1. Create a throwaway PostgreSQL database `navidb_e2e__` from the + DSN in `.env` (or `E2E_DATABASE_URL`) and drop it afterwards. +2. Start `navi-server` on a free port with a temporary working directory, its + own temp `workspace/` and `session_files/`, and + `NAVI_AUTH_ENABLED=false` (the local anonymous admin). Because the working + directory is temporary, `mcp_servers.d/` is not found, so no MCP server is + started and the run does not depend on their credentials. +3. Serve the built SPA from `webclient/dist` — the same assets production + serves. The suite refuses to run when `webclient/src` is newer than + `dist/index.html`; rebuild first: + + ```bash + cd webclient && npm run build + ``` + +4. Launch Chromium through Playwright with `/usr/bin/chromium` + (`E2E_CHROMIUM` overrides the path) and a fresh context per test. Service + workers are blocked so a cached build cannot answer. + +## Scenarios + +| Test | What it proves | +|------|----------------| +| `test_agent_writes_then_reads_file` | `filesystem_write write` creates the file, `filesystem_read read` renders its numbered content, the file matches on disk | +| `test_agent_edits_file_and_reads_diff` | `filesystem_write edit` changes the file, `filesystem_read diff` renders the unified diff | +| `test_agent_uses_file_ai_to_edit` | `file_ai edit` applies an instruction through the AI helper and renders its diff | + +## Requirements + +- Local PostgreSQL reachable through `DATABASE_URL`, and a role allowed to + `CREATE DATABASE` (the local dev role is). +- A working real LLM route: `llm.json` plus the credentials in `.env`. +- `psql` on `PATH`. +- Playwright's Python package (already a dependency) and Chromium at + `/usr/bin/chromium`. +- A fresh `webclient/dist`. + +## Tuning and diagnostics + +| Variable | Default | Meaning | +|----------|---------|---------| +| `E2E_DATABASE_URL` | `.env`'s `DATABASE_URL` | DSN used to create/drop the throwaway DB | +| `E2E_CHROMIUM` | `/usr/bin/chromium` | Browser binary | +| `E2E_SERVER_TIMEOUT` | `120` | Seconds to wait for `/health` | +| `E2E_TURN_TIMEOUT_MS` | `300000` | Milliseconds one agent turn may take | +| `E2E_SHOTS_DIR` | `/tmp/navi-e2e-shots` | Screenshots and browser logs on failure | +| `E2E_LOG_LEVEL` | `INFO` | Backend log level; the log lives in the run's temp dir | + +## Why the tests can fail without a code change + +The model is the production route, not a stub. It can pick `terminal` instead +of `filesystem_write`, refuse, or ask a question. The prompts are explicit +("use the tools yourself", exact action/arguments), but this is the price of an +honest end-to-end check. On failure the suite saves a screenshot and the +browser console log under `E2E_SHOTS_DIR`, and the backend log is in the run's +temp directory (`/tmp/pytest-*/navi-e2e*/server.log`); both make the model's +actual choice visible. diff --git a/docs/index.md b/docs/index.md index 77cc8af..f82bc3a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -53,6 +53,7 @@ | [`config.md`](config.md) | All environment variables with types and defaults | | [`api.md`](api.md) | REST API endpoints + full WebSocket event schemas and sequences | | [`testing.md`](testing.md) | Test layout, fixtures, how to run the suite | +| [`e2e.md`](e2e.md) | End-to-end suite: real backend, built SPA, real model, throwaway DB | Historical design notes and tech-debt reviews live in [`archive/`](archive/). diff --git a/e2e/conftest.py b/e2e/conftest.py new file mode 100644 index 0000000..dc4d1c6 --- /dev/null +++ b/e2e/conftest.py @@ -0,0 +1,317 @@ +"""End-to-end harness: a throwaway backend, the built SPA and a real browser. + +Run it with:: + + uv run pytest e2e -q + +The directory sits outside ``testpaths = ["tests"]`` so the regular backend run +never pays for it. One run gets: + +* a throwaway PostgreSQL database (created and dropped by the fixture); +* a throwaway working directory used as the server's CWD, which also means + ``mcp_servers.d/`` is not found and no MCP server is started; +* its own temp ``workspace/`` and ``session_files/`` directories; +* ``NAVI_AUTH_ENABLED=false``, so every request is the local anonymous admin; +* the real LLM route from ``llm.json`` and the real credentials from ``.env``. + +Nothing here touches the developer's running stack, the clone database or any +file in the repository. +""" + +from __future__ import annotations + +import os +import shutil +import socket +import subprocess +import sys +import time +import uuid +from pathlib import Path +from types import SimpleNamespace +from urllib import error, request +from urllib.parse import urlsplit + +import pytest +from dotenv import dotenv_values +from support import SERVER_TIMEOUT + +REPO = Path(__file__).resolve().parents[1] +DIST = REPO / "webclient" / "dist" +ENV_FILE = REPO / ".env" +CHROMIUM = os.environ.get("E2E_CHROMIUM", "/usr/bin/chromium") +SHOTS_DIR = Path(os.environ.get("E2E_SHOTS_DIR", "/tmp/navi-e2e-shots")) + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _database_url() -> str: + """The DSN the local dev stack uses, so the e2e DB sits on the same host.""" + dotenv = dotenv_values(ENV_FILE) + return ( + os.environ.get("E2E_DATABASE_URL") + or dotenv.get("DATABASE_URL") + or os.environ.get("DATABASE_URL") + or "" + ).strip() + + +def _psql(dsn: str, sql: str, database: str = "postgres") -> subprocess.CompletedProcess: + parts = urlsplit(dsn) + env = os.environ.copy() + if parts.password: + env["PGPASSWORD"] = parts.password + command = [ + "psql", + "-h", parts.hostname or "127.0.0.1", + "-p", str(parts.port or 5432), + "-U", parts.username or os.environ.get("USER", ""), + "-d", database, + "-v", "ON_ERROR_STOP=1", + "-c", sql, + ] + return subprocess.run( + command, env=env, capture_output=True, text=True, timeout=60, check=False + ) + + +def _tail(path: Path, lines: int = 80) -> str: + try: + return "\n".join(path.read_text(errors="replace").splitlines()[-lines:]) + except OSError: + return "(no log)" + + +def _assert_frontend_built() -> None: + """Fail loudly on a stale SPA build instead of quietly testing old assets.""" + index = DIST / "index.html" + if not index.exists(): + pytest.fail("webclient/dist is missing — run `npm run build` in webclient/") + newest = 0.0 + for path in (REPO / "webclient" / "src").rglob("*"): + if path.is_file(): + newest = max(newest, path.stat().st_mtime) + for name in ("package.json", "vite.config.js"): + candidate = REPO / "webclient" / name + if candidate.exists(): + newest = max(newest, candidate.stat().st_mtime) + if newest > index.stat().st_mtime + 1: + pytest.fail( + "webclient/dist is older than webclient/src — " + "run `npm run build` in webclient/ before the e2e run" + ) + + +@pytest.fixture(scope="session") +def e2e_database() -> str: + dsn = _database_url() + if not dsn: + pytest.skip("no DATABASE_URL (set E2E_DATABASE_URL or put it in .env)") + if not shutil.which("psql"): + pytest.skip("psql not found (install the postgresql client)") + + name = f"navidb_e2e_{os.getpid()}_{uuid.uuid4().hex[:8]}" + created = _psql(dsn, f'CREATE DATABASE "{name}"') + if created.returncode != 0: + pytest.fail(f"could not create the e2e database: {created.stderr.strip()}") + + parts = urlsplit(dsn) + e2e_dsn = parts._replace(path="/" + name).geturl() + try: + yield e2e_dsn + finally: + dropped = _psql(dsn, f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)') + if dropped.returncode != 0: + # PostgreSQL < 13 has no FORCE; disconnect the server first. + _psql( + dsn, + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " + f"WHERE datname = '{name}'", + ) + _psql(dsn, f'DROP DATABASE IF EXISTS "{name}"') + + +@pytest.fixture(scope="session") +def e2e_server(e2e_database: str, tmp_path_factory) -> SimpleNamespace: + _assert_frontend_built() + + run_root = tmp_path_factory.mktemp("navi-e2e") + cwd = run_root / "cwd" + workspace = run_root / "workspace" + session_files = run_root / "session_files" + for directory in (cwd, workspace, session_files): + directory.mkdir() + + port = _free_port() + base_url = f"http://127.0.0.1:{port}" + + env = os.environ.copy() + # Mirror the developer's .env first (that is where the Ollama key lives), + # then override everything the harness must isolate. + env.update({key: value for key, value in dotenv_values(ENV_FILE).items() if value is not None}) + env.update({ + "NAVI_AUTH_ENABLED": "false", + "NAVI_OVERLAY": "off", + "NAVI_UI_MCP_ENABLED": "false", + "NAVI_HOST": "127.0.0.1", + "NAVI_PORT": str(port), + "PUBLIC_URL": base_url, + "DATABASE_URL": e2e_database, + "FS_ALLOWED_PATHS": str(workspace), + "SESSION_FILES_DIR": str(session_files), + "PYTHONUNBUFFERED": "1", + "LOG_LEVEL": os.environ.get("E2E_LOG_LEVEL", "INFO"), + # Even if the developer's .env has an export consumer configured, the + # throwaway database must not produce export files. + "NAVI_EXPORT_AGE_RECIPIENT": "", + "NAVI_EXPORT_TOKEN": "", + }) + + log_path = run_root / "server.log" + log_file = log_path.open("wb") + proc = subprocess.Popen( + [sys.executable, "-m", "navi.server"], + cwd=str(cwd), + env=env, + stdout=log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + try: + deadline = time.monotonic() + SERVER_TIMEOUT + while time.monotonic() < deadline: + if proc.poll() is not None: + pytest.fail( + f"navi-server exited during startup (code {proc.returncode})\n" + f"{_tail(log_path)}" + ) + try: + with request.urlopen(f"{base_url}/health", timeout=2) as resp: + if resp.status == 200: + break + except (error.URLError, OSError): + pass + time.sleep(0.5) + else: + pytest.fail( + f"navi-server did not become healthy in {SERVER_TIMEOUT:.0f}s\n" + f"{_tail(log_path)}" + ) + + yield SimpleNamespace( + base_url=base_url, + workspace=workspace, + session_files=session_files, + log_path=log_path, + proc=proc, + cwd=cwd, + ) + finally: + proc.terminate() + try: + proc.wait(timeout=15) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=10) + log_file.close() + + +@pytest.fixture(scope="session") +def browser(): + if not Path(CHROMIUM).exists(): + pytest.skip(f"chromium not found at {CHROMIUM} (set E2E_CHROMIUM)") + playwright = pytest.importorskip("playwright.sync_api") + with playwright.sync_playwright() as p: + instance = p.chromium.launch( + headless=True, + executable_path=CHROMIUM, + args=["--no-sandbox"], + ) + try: + yield instance + finally: + instance.close() + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """Save a screenshot, the DOM and the browser log when a test fails. + + pytest does not throw the test's exception into fixture finalizers, so the + artefacts have to be captured from the report hook. The ``page`` fixture + leaves itself and its console log on the item for this hook to find. + """ + outcome = yield + report = outcome.get_result() + if report.when != "call" or not report.failed: + return + page = getattr(item, "_e2e_page", None) + if page is None: + return + SHOTS_DIR.mkdir(parents=True, exist_ok=True) + stem = f"{item.name}-{time.strftime('%H%M%S')}" + screenshot = SHOTS_DIR / f"{stem}.png" + try: + page.screenshot(path=str(screenshot), full_page=True) + except Exception: # noqa: BLE001 — best-effort artefact capture + screenshot = None + dom = SHOTS_DIR / f"{stem}.html" + try: + dom.write_text(page.content()) + except Exception: # noqa: BLE001 — best-effort artefact capture + dom = None + events = getattr(item, "_e2e_events", []) + diagnostics = SHOTS_DIR / f"{stem}.txt" + diagnostics.write_text("\n".join(events) or "(no browser errors)") + print( + f"\n[e2e] screenshot: {screenshot}\n[e2e] DOM: {dom}\n" + f"[e2e] browser log: {diagnostics}" + ) + + +@pytest.fixture +def page(browser, request): + """A fresh browser context per test. + + Service workers are blocked so a previous build's cache never answers a + request during the run. + """ + context = browser.new_context( + viewport={"width": 1440, "height": 900}, + locale="en-US", + service_workers="block", + ) + page = context.new_page() + page.set_default_timeout(30_000) + request.node._e2e_page = page + + events: list[str] = [] + request.node._e2e_events = events + page.on( + "console", + lambda message: events.append(f"console.{message.type}: {message.text}") + if message.type == "error" else None, + ) + page.on("pageerror", lambda exc: events.append(f"pageerror: {exc}")) + page.on( + "requestfailed", + lambda req: events.append(f"requestfailed: {req.method} {req.url}: {req.failure}"), + ) + + try: + yield page + finally: + context.close() + + +@pytest.fixture +def workdir(e2e_server: SimpleNamespace) -> Path: + """A private directory inside the agent's allowed workspace.""" + path = e2e_server.workspace / uuid.uuid4().hex[:10] + path.mkdir() + return path diff --git a/e2e/support.py b/e2e/support.py new file mode 100644 index 0000000..3a3d9f1 --- /dev/null +++ b/e2e/support.py @@ -0,0 +1,72 @@ +"""Helpers shared by the end-to-end tests: HTTP, sessions, UI waiters. + +The suite drives the real server and the built webclient, so the helpers stay +deliberately small: create a session over REST, open it in the browser, type a +prompt, and wait for the tool cards the agent is supposed to produce. +""" + +from __future__ import annotations + +import json +import os +from urllib import request + +# How long one agent turn may take. A real cloud model plans, thinks and calls +# tools, so this is minutes rather than seconds by design. +TURN_TIMEOUT_MS = int(os.environ.get("E2E_TURN_TIMEOUT_MS", "300000")) +# How long the fresh backend may take to answer /health. +SERVER_TIMEOUT = float(os.environ.get("E2E_SERVER_TIMEOUT", "120")) + + +def post_json(base_url: str, path: str, payload: dict) -> dict: + data = json.dumps(payload).encode() + req = request.Request( + base_url + path, + data=data, + method="POST", + headers={"Content-Type": "application/json"}, + ) + with request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode()) + + +def create_session(base_url: str, profile: str = "assistant") -> str: + """Create a session as the anonymous admin (auth is off in the harness).""" + return post_json(base_url, "/sessions", {"profile_id": profile})["session_id"] + + +def open_session(page, base_url: str, session_id: str) -> None: + page.goto(f"{base_url}/#{session_id}", wait_until="domcontentloaded") + page.wait_for_selector(".input-textarea", timeout=30_000) + + +def send_message(page, text: str) -> None: + page.fill(".input-textarea", text) + page.press(".input-textarea", "Enter") + + +def tool_card(page, name: str): + return page.locator(f'.tool-card:has(.tool-name:text-is("{name}"))').first + + +def wait_for_tools(page, names: list[str], timeout_ms: int) -> None: + """Wait until every named tool card exists and the turn has finished. + + ``.msg-meta-row`` is rendered only for a completed assistant message (the + stats/timestamp row), so it is the end-of-turn marker the UI already owns. + """ + for name in names: + page.wait_for_selector( + f'.tool-card:has(.tool-name:text-is("{name}"))', + timeout=timeout_ms, + ) + page.wait_for_selector(".msg-meta-row", timeout=timeout_ms) + + +def assert_ok(card) -> None: + classes = card.get_attribute("class") or "" + assert "is-success" in classes, f"tool card did not succeed: {classes}" + + +def result_text(card) -> str: + return card.locator(".tool-result").text_content() or "" diff --git a/e2e/test_agent_file_tools.py b/e2e/test_agent_file_tools.py new file mode 100644 index 0000000..48af456 --- /dev/null +++ b/e2e/test_agent_file_tools.py @@ -0,0 +1,125 @@ +"""End-to-end: browser → real agent → real file tools → cards → disk. + +Every test opens the built SPA against a throwaway backend, sends an explicit +prompt to a real model, and then checks both the rendered tool cards and the +file the agent wrote. The model is the production route from ``llm.json``, so +these tests are slow and, like production, not perfectly deterministic — the +prompts say exactly which tools to call, and a deviation fails with a +screenshot and the browser log. +""" + +from __future__ import annotations + +import uuid +from pathlib import Path + +from support import ( + TURN_TIMEOUT_MS, + assert_ok, + create_session, + open_session, + result_text, + send_message, + tool_card, + wait_for_tools, +) + +_PREAMBLE = ( + "Run this end-to-end check. Use the tools yourself — do not ask questions, " + "do not spawn a subagent, do not use terminal or code_exec.\n" +) + + +def test_agent_writes_then_reads_file(page, e2e_server, workdir: Path) -> None: + target = workdir / "probe.txt" + sentinel = f"navi-e2e-{uuid.uuid4().hex[:10]}" + + session = create_session(e2e_server.base_url) + open_session(page, e2e_server.base_url, session) + send_message( + page, + _PREAMBLE + + f'1. Call filesystem_write with action="write", path="{target}", ' + f'content="{sentinel}\\n".\n' + + f'2. Call filesystem_read with action="read", path="{target}".\n' + + "3. Reply with exactly: DONE", + ) + + wait_for_tools(page, ["filesystem_write", "filesystem_read"], TURN_TIMEOUT_MS) + + assert target.is_file(), f"the agent did not create {target}" + assert target.read_text(encoding="utf-8").strip() == sentinel + + write_card = tool_card(page, "filesystem_write") + assert_ok(write_card) + + read_card = tool_card(page, "filesystem_read") + assert_ok(read_card) + read_output = result_text(read_card) + assert sentinel in read_output, read_output[:400] + assert target.name in read_output, "the read card should name the file" + # The renderer paints numbered lines for a read result. + assert read_card.locator(".read-num").count() >= 1 + assert page.locator(".tool-card.is-error").count() == 0 + + +def test_agent_edits_file_and_reads_diff(page, e2e_server, workdir: Path) -> None: + target = workdir / "edit-me.txt" + before = workdir / "before.txt" + original = "alpha beta gamma\n" + target.write_text(original, encoding="utf-8") + before.write_text(original, encoding="utf-8") + + session = create_session(e2e_server.base_url) + open_session(page, e2e_server.base_url, session) + send_message( + page, + _PREAMBLE + + f'1. Call filesystem_write with action="edit", path="{target}", ' + 'old="beta", new="BETA".\n' + + f'2. Call filesystem_read with action="diff", path="{target}", other="{before}".\n' + + "3. Reply with exactly: DONE", + ) + + wait_for_tools(page, ["filesystem_write", "filesystem_read"], TURN_TIMEOUT_MS) + + edited = target.read_text(encoding="utf-8") + assert "BETA" in edited, edited + assert "beta" not in edited, edited + + assert_ok(tool_card(page, "filesystem_write")) + diff_card = tool_card(page, "filesystem_read") + assert_ok(diff_card) + diff_output = result_text(diff_card) + assert "BETA" in diff_output and "beta" in diff_output, diff_output[:400] + # A unified diff, rendered with the add/remove colouring. + assert diff_card.locator(".diff-add").count() >= 1 + assert page.locator(".tool-card.is-error").count() == 0 + + +def test_agent_uses_file_ai_to_edit(page, e2e_server, workdir: Path) -> None: + target = workdir / "ai-me.txt" + target.write_text("alpha beta gamma\n", encoding="utf-8") + + session = create_session(e2e_server.base_url) + open_session(page, e2e_server.base_url, session) + send_message( + page, + _PREAMBLE + + f'1. Call file_ai with action="edit", path="{target}", ' + 'instruction="Replace the word alpha with OMEGA."\n' + + "2. Reply with exactly: DONE", + ) + + wait_for_tools(page, ["file_ai"], TURN_TIMEOUT_MS) + + edited = target.read_text(encoding="utf-8") + assert "OMEGA" in edited, edited + assert "alpha" not in edited, edited + + card = tool_card(page, "file_ai") + assert_ok(card) + assert "OMEGA" in result_text(card) + # file_ai edit returns a diff; the webclient renders it as one. + assert card.locator(".diff-add").count() >= 1 + assert page.locator(".tool-card.is-error").count() == 0