"""web_view must send the page, not the page's furniture.
The old path was `innerText` of the body after a tag kill-list: navigation
crumbs survived, tables collapsed into one line per thousand characters
(news.ycombinator.com came back as four lines), headings were unmarked and
every link was gone. These tests pin the replacement: main content only,
structure kept, links lifted into an appendix.
"""
PAGE = """<html><head><title>Guide</title></head><body>
<nav><a href="/">Home</a><a href="/about">About us</a></nav>
<article>
<h1>Guide</h1>
<p>First paragraph of the guide: real prose that the reader came here for, long
enough to be kept by the extractor without any doubt at all about it.</p>
<pre>def f():
if True:
return 1
</pre>
<h2>Second section</h2>
<p>Second paragraph, also prose, also worth keeping: it explains the second
section in a sentence or two, which is what the reader wanted.</p>
<p><a href="/docs/page">Read the docs</a> and
<a href="https://other.example/x">another page</a>.</p>
<p>Third paragraph is here only so the page clears the extractor's minimum
length, and it says nothing except that it exists.</p>
</article>
<footer>© 2026 Boilerplate Inc.</footer>
</body></html>"""
class TestExtraction:
def test_the_content_survives_and_the_chrome_does_not(self, navi_web_extract):
body, _ = navi_web_extract.extract_page(PAGE, "https://example.com/a/b")
assert "First paragraph of the guide" in body
assert "Second section" in body
assert "About us" not in body
assert "Boilerplate" not in body
def test_headings_and_code_keep_their_shape(self, navi_web_extract):
"""The fence matters: markdown output from trafilatura flattens
`` return 1`` to column 0, which is not Python any more."""
body, _ = navi_web_extract.extract_page(PAGE, "https://example.com/a/b")
assert "# Guide" in body
assert "## Second section" in body
assert "```\ndef f():\n if True:\n return 1\n```" in body
def test_links_move_to_an_appendix_with_absolute_urls(self, navi_web_extract):
body, links = navi_web_extract.extract_page(PAGE, "https://example.com/a/b")
assert "Read the docs" in body
assert "https://example.com/docs/page" not in body
assert links == [
"- Read the docs -> https://example.com/docs/page",
"- another page -> https://other.example/x",
]
def test_a_page_without_text_is_not_a_page(self, navi_web_extract):
html = "<html><body><script>var a = 1</script></body></html>"
assert navi_web_extract.extract_page(html, "https://example.com/") is None
def test_a_thread_is_read_with_its_comments(self, navi_web_extract, monkeypatch):
"""Forum pages are their comments: a header-only extract is a miss, and
the signal is how much of the visible page the extractor threw away."""
calls: list[bool] = []
def fake_extract(html, url, comments):
calls.append(comments)
if comments:
return "x" * 5_000, ["- reply -> https://forum.example/1"]
return "x" * 100, []
monkeypatch.setattr(navi_web_extract, "_extract", fake_extract)
body, links = navi_web_extract.extract_page(
"<html></html>", "https://forum.example/1", visible_chars=100_000
)
assert calls == [False, True]
assert len(body) == 5_000
assert links == ["- reply -> https://forum.example/1"]
def test_a_page_that_was_understood_is_not_read_twice(
self, navi_web_extract, monkeypatch
):
calls: list[bool] = []
def fake_extract(html, url, comments):
calls.append(comments)
return "x" * 1_000, []
monkeypatch.setattr(navi_web_extract, "_extract", fake_extract)
navi_web_extract.extract_page(
"<html></html>", "https://blog.example/1", visible_chars=2_000
)
assert calls == [False]
class TestLinkAppendix:
def test_noise_links_are_dropped(self, navi_web_extract):
page = """<html><body><article><p>Prose that is long enough to be kept, and it goes on for a while: it says
a few things about the subject at hand, which is what the reader came for, and
it keeps saying them until the extractor is satisfied.</p>
<p>Then it links onwards: <a href="/vote?id=1">upvote</a>,
<a href="/user?id=2">alice</a>, <a href="https://bare.example/x">https://bare.example/x</a>,
and <a href="/real">The real target</a>, which is the only one worth following.</p>
</article></body></html>"""
_, links = navi_web_extract.extract_page(page, "https://example.com/post")
assert links == ["- The real target -> https://example.com/real"]
def test_the_appendix_is_capped(self, navi_web_extract):
anchors = "".join(f'<a href="/p{i}">page {i}</a> ' for i in range(80))
page = f"""<html><body><article><p>Prose that is long enough to be kept, and it goes on for a while: it says
a few things about the subject at hand, which is what the reader came for, and
it keeps saying them until the extractor is satisfied, one page at a time.</p>
<p>{anchors}</p></article></body></html>"""
_, links = navi_web_extract.extract_page(page, "https://example.com/")
assert len(links) == navi_web_extract.MAX_LINKS
def test_the_same_link_is_listed_once(self, navi_web_extract):
page = """<html><body><article><p>Prose that is long enough to be kept, and it goes on for a while: it says
a few things about the subject at hand, which is what the reader came for, and
it keeps saying them until the extractor is satisfied.</p>
<p>Both link to <a href="/real">The real target</a> and again to
<a href="/real">The real target</a>, which should still be one line.</p>
</article></body></html>"""
_, links = navi_web_extract.extract_page(page, "https://example.com/")
assert links == ["- The real target -> https://example.com/real"]
class TestTruncate:
def test_short_text_is_untouched(self, navi_web_extract):
assert navi_web_extract.truncate("# T\n\nbody", 100) == "# T\n\nbody"
def test_a_cut_names_the_sections_that_did_not_fit(self, navi_web_extract):
text = "# A\n\n" + "a" * 400 + "\n\n## Kept\n\n" + "b" * 400 + "\n\n## Lost\n\n" + "c" * 400
cut = navi_web_extract.truncate(text, 500)
assert cut.startswith("# A")
assert "## Kept" in cut
assert "## Lost" not in cut.split("[")[0]
assert "Sections not shown: ## Lost" in cut
def test_a_cut_inside_a_fence_closes_it(self, navi_web_extract):
text = "# T\n\n```\n" + "code line\n" * 100
cut = navi_web_extract.truncate(text, 300)
assert cut.count("```") % 2 == 0
def test_the_cut_lands_on_a_paragraph_boundary(self, navi_web_extract):
text = ("paragraph words " * 10 + "\n\n") * 10
cut = navi_web_extract.truncate(text, 300)
assert cut.split("[")[0].rstrip().endswith("words")
class _FakePage:
def __init__(self, html: str, visible: str, title: str = "Guide"):
self._html = html
self._visible = visible
self._title = title
self.url = "https://example.com/final"
self.closed = False
async def goto(self, *args, **kwargs):
return None
async def title(self):
return self._title
async def evaluate(self, _script):
return self._visible
async def content(self):
return self._html
async def screenshot(self, **kwargs):
return b"\x89PNG"
async def close(self):
self.closed = True
class _FakeContext:
def __init__(self, page: _FakePage):
self._page = page
async def new_page(self):
return self._page
class TestReadPage:
async def test_the_output_carries_the_body_and_the_links(self, navi_web_browse):
page = _FakePage(PAGE, "visible text")
result = await navi_web_browse._read_page(
_FakeContext(page), "https://example.com/a", False, "networkidle"
)
assert result["output"].startswith("Title: Guide\nFinal URL: https://example.com/final")
assert "## Second section" in result["output"]
assert "Links on this page:\n- Read the docs -> https://example.com/docs/page" in result["output"]
assert result["metadata"] == {"extractor": "trafilatura", "links": 2}
async def test_a_page_the_extractor_cannot_read_falls_back_to_its_text(
self, navi_web_browse, monkeypatch
):
monkeypatch.setattr(navi_web_browse, "extract_page", lambda *a, **k: None)
page = _FakePage("<html></html>", "plain visible text\n\n\n\nafter blanks")
result = await navi_web_browse._read_page(
_FakeContext(page), "https://example.com/a", False, "networkidle"
)
assert "plain visible text" in result["output"]
assert result["metadata"]["extractor"] == "innerText"
assert result["metadata"]["links"] == 0
async def test_a_long_page_is_cut_with_a_note(self, navi_web_browse, monkeypatch):
page = _FakePage(PAGE, "visible text")
monkeypatch.setattr(navi_web_browse, "_MAX_TEXT", 120)
result = await navi_web_browse._read_page(
_FakeContext(page), "https://example.com/a", False, "networkidle"
)
assert "truncated at 120 chars" in result["output"]
async def test_a_screenshot_keeps_the_text_metadata(self, navi_web_browse):
page = _FakePage(PAGE, "visible text")
result = await navi_web_browse._read_page(
_FakeContext(page), "https://example.com/a", True, "networkidle"
)
assert result["metadata"]["is_image"] is True
assert result["metadata"]["extractor"] == "trafilatura"
