Newer
Older
navi-1 / mcp-servers / navi-web / app / browse.py
"""web_view — headless browser page extraction."""

from __future__ import annotations

import asyncio
import base64
from contextlib import suppress
from typing import Any

from playwright.async_api import async_playwright

from .extract import clean_visible_text, extract_page, truncate

_TIMEOUT = 20_000
_MAX_TEXT = 20_000

# A call may open several pages at once: pages take 1-8s each, so a batch costs
# the slowest instead of the sum (three URLs here: 17.3s one by one, 1.9s
# together). The shared text budget is the counterweight — three pages at the
# single-page cap would be 60k chars of context, so a batch stops at 40k and
# says where it stopped.
_MAX_BATCH = 3
_BATCH_TEXT = 40_000
_BATCH_DEADLINE = 30.0

# One Chromium for the life of the server. Launching a browser per call cost ~5s
# even for ``example.com``; this process is long-lived, so the browser can be
# too. Each call still gets a fresh context, which is what isolates cookies,
# storage and the page from its neighbours.
_playwright = None
_browser = None
_browser_lock = asyncio.Lock()

# Contexts are cheap but not free; the cap keeps a burst of concurrent calls
# (a batch, or two tool calls Navi ran in parallel) from launching a crowd.
_view_slots = asyncio.Semaphore(_MAX_BATCH)


def _normalize(raw: str | list[str]) -> tuple[list[str], list[str]]:
    """Turn the argument into the URLs to open, plus notes worth telling the model."""
    urls: list[str] = []
    for item in [raw] if isinstance(raw, str) else raw:
        url = str(item).strip()
        if url and url not in urls:
            urls.append(url)
    notes: list[str] = []
    if len(urls) > _MAX_BATCH:
        notes.append(
            f"Only the first {_MAX_BATCH} of {len(urls)} URLs were opened — "
            "ask the rest in another call."
        )
        urls = urls[:_MAX_BATCH]
    return urls, notes



async def _get_browser():
    """Return the shared browser, launching (or relaunching) it if needed."""
    global _playwright, _browser
    async with _browser_lock:
        if _browser is not None and _browser.is_connected():
            return _browser
        # A browser that crashed leaves a Playwright driver behind; stop it
        # before starting another, or the driver processes accumulate.
        if _playwright is not None:
            with suppress(Exception):
                await _playwright.stop()
            _playwright = None
            _browser = None
        _playwright = await async_playwright().start()
        _browser = await _playwright.chromium.launch(headless=True)
        return _browser


async def close_browser() -> None:
    """Stop Chromium and its Playwright driver. Called from the server lifespan."""
    global _playwright, _browser
    async with _browser_lock:
        if _browser is not None:
            with suppress(Exception):
                await _browser.close()
            _browser = None
        if _playwright is not None:
            with suppress(Exception):
                await _playwright.stop()
            _playwright = None


# The page's own visible text, without the chrome that never had a reader.
_VISIBLE_TEXT = """() => {
    const kill = ['script','style','noscript','iframe',
                  'nav','header','footer','aside',
                  '[role="navigation"]','[role="banner"]',
                  '[role="contentinfo"]'];
    const clone = document.body.cloneNode(true);
    kill.forEach(sel => {
        clone.querySelectorAll(sel).forEach(el => el.remove());
    });
    return clone.innerText || clone.textContent || '';
}"""


async def web_view(
    url: str | list[str],
    screenshot: bool = False,
    wait_until: str = "networkidle",
) -> dict[str, Any]:
    """Open a URL — or up to three — in a headless browser and return readable text.

    Returns dict with success, output, error, metadata.
    """
    urls, notes = _normalize(url)
    if not urls:
        return {
            "success": False,
            "output": "No URL given.",
            "error": "empty_url",
            "metadata": {},
        }
    if screenshot and len(urls) > 1:
        return {
            "success": False,
            "output": (
                "screenshot works on one URL at a time — ask for a single URL "
                "with screenshot, or several without."
            ),
            "error": "screenshot_batch",
            "metadata": {},
        }

    if len(urls) == 1 and not notes:
        return await _view_bounded(urls[0], screenshot=screenshot, wait_until=wait_until)

    outcomes = await asyncio.gather(
        *(_view_bounded(u, wait_until=wait_until) for u in urls),
        return_exceptions=True,
    )
    return _combine(urls, outcomes, notes)


async def _view_bounded(
    url: str,
    screenshot: bool = False,
    wait_until: str = "networkidle",
) -> dict[str, Any]:
    """One page, bounded on its own.

    The deadline is per URL: a batch waits for its slowest page, and a single
    ceiling around the whole batch would let one hung page erase the others
    that already loaded.
    """
    try:
        return await asyncio.wait_for(
            _view_one(url, screenshot=screenshot, wait_until=wait_until),
            timeout=_BATCH_DEADLINE,
        )
    except TimeoutError:
        return {
            "success": False,
            "output": f"No answer within {_BATCH_DEADLINE:.0f}s: {url}",
            "error": "timeout",
            "metadata": {},
        }


def _combine(
    urls: list[str],
    outcomes: list[Any],
    notes: list[str],
) -> dict[str, Any]:
    """One block per URL, sharing one text budget.

    A URL that failed is named rather than dropped — the model can retry that
    one page instead of repeating the batch — and a batch that opened something
    is a success even when the rest did not.
    """
    blocks: list[str] = []
    per_url: list[dict[str, Any]] = []
    budget = _BATCH_TEXT
    opened = 0

    for url, outcome in zip(urls, outcomes, strict=True):
        if isinstance(outcome, BaseException):
            blocks.append(f"### {url} -> failed: {outcome}")
            per_url.append({"url": url, "success": False, "error": str(outcome)})
            continue
        if not outcome["success"]:
            blocks.append(f"### {url} -> failed: {outcome['output']}")
            per_url.append({"url": url, "success": False, "error": outcome["error"]})
            continue

        opened += 1
        block = f"### {url}\n{outcome['output']}"
        if budget <= 0:
            block = (
                f"### {url}\n[… not shown: the batch's "
                f"{_BATCH_TEXT // 1000}k-char text budget is used up]"
            )
            per_url.append({"url": url, "success": True, "truncated": True})
            blocks.append(block)
            continue
        if len(block) > budget:
            block = (
                block[:budget]
                + f"\n\n[… truncated: the batch's {_BATCH_TEXT // 1000}k-char "
                "text budget is used up]"
            )
            per_url.append({"url": url, "success": True, "truncated": True})
            budget = 0
        else:
            per_url.append({"url": url, "success": True})
            budget -= len(block)
        blocks.append(block)

    body = "\n\n".join(notes + blocks)
    if not opened:
        return {
            "success": False,
            "output": body or "No URL could be opened.",
            "error": "all_urls_failed",
            "metadata": {"urls": per_url},
        }
    return {
        "success": True,
        "output": body,
        "error": None,
        "metadata": {"urls": per_url},
    }


async def _view_one(
    url: str,
    screenshot: bool = False,
    wait_until: str = "networkidle",
) -> dict[str, Any]:
    """Open one page in its own context. Returns dict with success, output, error, metadata."""
    if not url.startswith(("http://", "https://")):
        return {
            "success": False,
            "output": "URL must start with http:// or https://",
            "error": "invalid_url",
            "metadata": {},
        }

    try:
        async with _view_slots:
            browser = await _get_browser()
            context = await browser.new_context(
                viewport={"width": 1280, "height": 800},
                user_agent=(
                    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
                    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
                ),
            )
            try:
                return await _read_page(context, url, screenshot, wait_until)
            finally:
                with suppress(Exception):
                    await context.close()

    except Exception as e:
        return {
            "success": False,
            "output": f"Browser error: {e}",
            "error": str(e),
            "metadata": {},
        }


async def _read_page(
    context: Any,
    url: str,
    screenshot: bool,
    wait_until: str,
) -> dict[str, Any]:
    page = await context.new_page()

    try:
        await page.goto(url, wait_until=wait_until, timeout=_TIMEOUT)
    except Exception:
        pass

    title = await page.title()
    final_url = page.url

    # The visible text is both the fallback and the yardstick: a page whose
    # extractor kept a fraction of it is a page the extractor did not understand.
    visible = clean_visible_text(await page.evaluate(_VISIBLE_TEXT))
    extracted = extract_page(await page.content(), final_url, visible_chars=len(visible))

    if extracted is not None:
        text, links = extracted
        text = truncate(text, _MAX_TEXT)
        if links:
            text = f"{text}\n\nLinks on this page:\n" + "\n".join(links)
        extractor = "trafilatura"
    else:
        text, links, extractor = truncate(visible, _MAX_TEXT), [], "innerText"

    output_parts = []
    if title:
        output_parts.append(f"Title: {title}")
    if final_url != url:
        output_parts.append(f"Final URL: {final_url}")
    output_parts.append("")
    output_parts.append(text)
    output = "\n".join(output_parts)

    metadata: dict[str, Any] = {"extractor": extractor, "links": len(links)}
    if screenshot:
        png = await page.screenshot(full_page=False)
        metadata |= {
            "base64": base64.b64encode(png).decode(),
            "mime": "image/png",
            "is_image": True,
        }

    return {
        "success": True,
        "output": output,
        "error": None,
        "metadata": metadata,
    }