Newer
Older
navi-1 / e2e / test_agent_file_tools.py
"""End-to-end: browser → real agent → real file tools → cards → disk.

Every test opens the built SPA against a throwaway backend, sends an explicit
prompt to a real model, and then checks both the rendered tool cards and the
file the agent wrote. The model is the production route from ``llm.json``, so
these tests are slow and, like production, not perfectly deterministic — the
prompts say exactly which tools to call, and a deviation fails with a
screenshot and the browser log.
"""

from __future__ import annotations

import uuid
from pathlib import Path

from support import (
    TURN_TIMEOUT_MS,
    assert_ok,
    create_session,
    open_session,
    result_text,
    send_message,
    tool_card,
    wait_for_tools,
)

_PREAMBLE = (
    "Run this end-to-end check. Use the tools yourself — do not ask questions, "
    "do not spawn a subagent, do not use terminal or code_exec.\n"
)


def test_agent_writes_then_reads_file(page, e2e_server, workdir: Path) -> None:
    target = workdir / "probe.txt"
    sentinel = f"navi-e2e-{uuid.uuid4().hex[:10]}"

    session = create_session(e2e_server.base_url)
    open_session(page, e2e_server.base_url, session)
    send_message(
        page,
        _PREAMBLE
        + f'1. Call filesystem_write with action="write", path="{target}", '
          f'content="{sentinel}\\n".\n'
        + f'2. Call filesystem_read with action="read", path="{target}".\n'
        + "3. Reply with exactly: DONE",
    )

    wait_for_tools(page, ["filesystem_write", "filesystem_read"], TURN_TIMEOUT_MS)

    assert target.is_file(), f"the agent did not create {target}"
    assert target.read_text(encoding="utf-8").strip() == sentinel

    write_card = tool_card(page, "filesystem_write")
    assert_ok(write_card)

    read_card = tool_card(page, "filesystem_read")
    assert_ok(read_card)
    read_output = result_text(read_card)
    assert sentinel in read_output, read_output[:400]
    assert target.name in read_output, "the read card should name the file"
    # The renderer paints numbered lines for a read result.
    assert read_card.locator(".read-num").count() >= 1
    assert page.locator(".tool-card.is-error").count() == 0


def test_agent_edits_file_and_reads_diff(page, e2e_server, workdir: Path) -> None:
    target = workdir / "edit-me.txt"
    before = workdir / "before.txt"
    original = "alpha beta gamma\n"
    target.write_text(original, encoding="utf-8")
    before.write_text(original, encoding="utf-8")

    session = create_session(e2e_server.base_url)
    open_session(page, e2e_server.base_url, session)
    send_message(
        page,
        _PREAMBLE
        + f'1. Call filesystem_write with action="edit", path="{target}", '
          'old="beta", new="BETA".\n'
        + f'2. Call filesystem_read with action="diff", path="{target}", other="{before}".\n'
        + "3. Reply with exactly: DONE",
    )

    wait_for_tools(page, ["filesystem_write", "filesystem_read"], TURN_TIMEOUT_MS)

    edited = target.read_text(encoding="utf-8")
    assert "BETA" in edited, edited
    assert "beta" not in edited, edited

    assert_ok(tool_card(page, "filesystem_write"))
    diff_card = tool_card(page, "filesystem_read")
    assert_ok(diff_card)
    diff_output = result_text(diff_card)
    assert "BETA" in diff_output and "beta" in diff_output, diff_output[:400]
    # A unified diff, rendered with the add/remove colouring.
    assert diff_card.locator(".diff-add").count() >= 1
    assert page.locator(".tool-card.is-error").count() == 0


def test_agent_uses_file_ai_to_edit(page, e2e_server, workdir: Path) -> None:
    target = workdir / "ai-me.txt"
    target.write_text("alpha beta gamma\n", encoding="utf-8")

    session = create_session(e2e_server.base_url)
    open_session(page, e2e_server.base_url, session)
    send_message(
        page,
        _PREAMBLE
        + f'1. Call file_ai with action="edit", path="{target}", '
          'instruction="Replace the word alpha with OMEGA."\n'
        + "2. Reply with exactly: DONE",
    )

    wait_for_tools(page, ["file_ai"], TURN_TIMEOUT_MS)

    edited = target.read_text(encoding="utf-8")
    assert "OMEGA" in edited, edited
    assert "alpha" not in edited, edited

    card = tool_card(page, "file_ai")
    assert_ok(card)
    assert "OMEGA" in result_text(card)
    # file_ai edit returns a diff; the webclient renders it as one.
    assert card.locator(".diff-add").count() >= 1
    assert page.locator(".tool-card.is-error").count() == 0