"""Readable text out of a rendered page.

What this replaced: `innerText` of the body after a tag kill-list. That gave a
wall of navigation crumbs, and on table-heavy pages a single line per thousand
characters — news.ycombinator.com came back as four lines, none of them a
sentence. Nothing was marked as a heading, and no link survived, so the model
could neither see the page's structure nor follow anything on it.

trafilatura picks the main content out by text density instead of by tag name,
and its ``html`` output keeps the whitespace inside ``<pre>`` that its markdown
output throws away (``def f():`` and ``    return 1`` came back at the same
indent, i.e. as broken code). So the HTML fragment is rendered here: headings
get their ``#``, code keeps its fences, and links are lifted into an appendix
instead of turning every other sentence into ``[label](url)``.
"""

from __future__ import annotations

import re
from typing import Any
from urllib.parse import urljoin

import trafilatura
from lxml import html as lxml_html

# Enough links to follow the page's own argument, not so many that the appendix
# outweighs the text it belongs to.
MAX_LINKS = 40

# trafilatura can return a bare menu for a page whose content the browser never
# rendered; that is not a reason to hand back a heading and nothing else.
MIN_TEXT = 200

# Links a site offers about itself rather than about its content: vote buttons,
# "hide", login, edit-history, citation footnotes.
_SKIP_HREF = ("hide?", "from?site", "user?id", "vote?", "login?", "action=edit",
              "cite_note")

_SPACES = re.compile(r"[ \t\r\f\v]+")
_HEADING = re.compile(r"^(#{1,6}) \S")
_HEADINGS = {f"h{level}": level for level in range(1, 7)}

# Containers with no markdown meaning of their own: their children are blocks.
_CONTAINERS = {"html", "body", "main", "article", "div", "section", "header",
               "footer", "figure", "figcaption", "details", "summary", "center",
               "form", "fieldset"}

_INLINE = {"a", "span", "small", "sub", "sup", "time", "abbr", "del", "ins",
           "mark", "u", "font", "big", "s", "strike", "cite", "q", "bdi", "bdo",
           "ruby", "var", "samp", "kbd", "wbr", "nobr"}

# trafilatura's own row dialect, and the html one it falls back to.
_ROWS = {"row", "tr"}


def _tag(el: Any) -> str:
    tag = getattr(el, "tag", None)
    return tag.lower() if isinstance(tag, str) else ""


def _collect_links(
    label: str, href: str | None, base: str, links: list[str]
) -> None:
    """Keep a link worth following: not an anchor into this page, not site chrome."""
    label = _SPACES.sub(" ", label).strip()
    if not label or not href:
        return
    target = urljoin(base, href.strip())
    if not target.startswith(("http://", "https://")):
        return
    if target.rstrip("/") == base.rstrip("/"):
        return
    # A link whose text is just the URL adds nothing the URL does not already say.
    if label.lower().startswith(("http://", "https://")):
        return
    if any(marker in target for marker in _SKIP_HREF):
        return
    if len(label) > 100:
        label = label[:97].rstrip() + "…"
    entry = f"- {label} -> {target}"
    if entry in links or len(links) >= MAX_LINKS:
        return
    links.append(entry)


def _join(parts: list[str]) -> str:
    """Glue text pieces back together without gluing their words.

    trafilatura's html output drops the spaces around loose inline elements —
    a wiki sentence comes back as ``<a>C++</a>and<a>Qt</a>`` — so two pieces
    that both start and end on a letter get a space between them. Punctuation
    keeps its neighbours: ``(<a>hackerbooks.org</a>)`` stays in parentheses.
    """
    out = ""
    for part in parts:
        if not part:
            continue
        if out and not out[-1].isspace() and not part[0].isspace() \
                and out[-1].isalnum() and part[0].isalnum():
            out += " "
        out += part
    return _SPACES.sub(" ", out).strip()


def _inline(el: Any, base: str, links: list[str]) -> str:
    """Text of an element that sits inside a paragraph, with links harvested."""
    parts: list[str] = [el.text or ""]
    for child in el:
        tag = _tag(child)
        if tag == "a":
            label = _inline(child, base, links)
            _collect_links(label, child.get("href"), base, links)
            parts.append(label)
        elif tag in ("strong", "b"):
            inner = _inline(child, base, links).strip()
            parts.append(f"**{inner}**" if inner else "")
        elif tag == "img":
            # The image itself is unavailable as text; its alt text usually is not.
            parts.append((child.get("alt") or "").strip())
        elif tag == "br":
            parts.append(" ")
        elif tag == "pre":
            parts.append(_code(child))
        elif tag in _INLINE or tag in ("i", "em", "code", "tt"):
            parts.append(_inline(child, base, links))
        else:
            parts.append(_inline(child, base, links))
        parts.append(child.tail or "")
    return _join(parts)


def _code(el: Any) -> str:
    """A <pre> as a fence that keeps its own indentation.

    trafilatura emits its own one-liners as <pre> too — ``<pre>asyncio.run()</pre>``
    inside a sentence — so a short single line becomes inline code.
    """
    code = "".join(el.itertext()).replace(" ", " ").strip("\n")
    if not code.strip():
        return ""
    if "\n" not in code.strip() and len(code.strip()) <= 60:
        return f"`{code.strip()}`"
    return f"```\n{code.rstrip()}\n```"


def _list(el: Any, base: str, links: list[str]) -> str:
    ordered = _tag(el) == "ol"
    items: list[str] = []
    number = 0
    for child in el:
        if _tag(child) != "li":
            continue
        number += 1
        marker = f"{number}. " if ordered else "- "
        body = _block(child, base, links).strip()
        if not body:
            continue
        lines = body.splitlines()
        pad = " " * len(marker)
        items.append(marker + lines[0] + "".join("\n" + pad + ln for ln in lines[1:]))
    return "\n".join(items)


def _row(el: Any, base: str, links: list[str]) -> str:
    """A table row — HN, wiki infoboxes — as one line of columns.

    trafilatura names its own rows ``<row>/<cell>`` and keeps real ones as
    ``<tr>/<td>``, with no column model either way and a colspan here or there,
    so a plain ``|``-joined line keeps the shape without inventing a header.
    """
    cells: list[str] = []
    nested: list[str] = []
    for child in el:
        tag = _tag(child)
        text = _block(child, base, links).replace("\n", " ").strip()
        if not text:
            continue
        (nested if tag in _ROWS else cells).append(text)
    line = " | ".join(cells)
    return "\n".join(part for part in [line, *nested] if part)


def _block(el: Any, base: str, links: list[str]) -> str:
    tag = _tag(el)
    if tag in _HEADINGS:
        title = _inline(el, base, links)
        return f"{'#' * _HEADINGS[tag]} {title}" if title else ""
    if tag == "pre":
        return _code(el)
    if tag in ("ul", "ol"):
        return _list(el, base, links)
    if tag == "blockquote":
        inner = _blocks(el, base, links)
        return "\n".join(f"> {line}".rstrip() if line else ">" for line in inner.splitlines())
    if tag in _ROWS:
        return _row(el, base, links)
    if tag == "hr":
        return "---"
    return _inline(el, base, links)


def _render_child(el: Any, base: str, links: list[str]) -> str:
    """A container's child as markdown: blocks that contain blocks, or a leaf."""
    tag = _tag(el)
    if tag in _CONTAINERS:
        return _blocks(el, base, links)
    if tag in ("table", "tbody", "thead", "tfoot", "dl"):
        return _join_pieces([
            ("row" if _tag(grandchild) in _ROWS else _tag(grandchild),
             _block(grandchild, base, links))
            for grandchild in el
        ])
    return _block(el, base, links)


def _join_pieces(pieces: list[tuple[str, str]]) -> str:
    """Blank line between blocks — except between table rows, which stay adjacent.

    news.ycombinator.com is a table, and a blank line after every row is what
    turned it into a page-tall wall before.
    """
    out = ""
    previous = ""
    for kind, text in pieces:
        if not text.strip():
            continue
        if not out:
            out = text
        elif kind == previous == "row":
            out += "\n" + text
        else:
            out += "\n\n" + text
        previous = kind
    return out


def _blocks(el: Any, base: str, links: list[str]) -> str:
    """Children of a container, one block each."""
    pieces: list[tuple[str, str]] = []
    if (el.text or "").strip():
        pieces.append(("text", _SPACES.sub(" ", el.text or "").strip()))
    for child in el:
        tag = _tag(child)
        pieces.append((tag, _render_child(child, base, links)))
        if (child.tail or "").strip():
            pieces.append(("text", _SPACES.sub(" ", child.tail or "").strip()))
    return _join_pieces(pieces)


def _tidy(text: str) -> str:
    """Collapse the blank runs the renderer leaves behind; never touch a fence."""
    out: list[str] = []
    fence = False
    blank = 0
    for line in text.splitlines():
        if line.lstrip().startswith("```"):
            fence = not fence
            out.append(line.rstrip())
            blank = 0
            continue
        if fence:
            out.append(line.rstrip())
            continue
        line = line.rstrip()
        if not line.strip():
            blank += 1
            if blank > 1:
                continue
            out.append("")
            continue
        blank = 0
        out.append(line.strip())
    # A pilcrow is how documentation sites mark a heading's own anchor.
    return "\n".join(out).replace("¶", "").strip()


def _extract(html: str, url: str, comments: bool) -> tuple[str, list[str]]:
    """One trafilatura pass, rendered. ``("", [])`` when the page has no content."""
    try:
        fragment = trafilatura.extract(
            html,
            url=url,
            output_format="html",
            include_links=True,
            include_tables=True,
            include_comments=comments,
            include_formatting=True,
        )
    except Exception:  # noqa: BLE001 — a broken page must not break the tool
        return "", []
    if not fragment:
        return "", []

    try:
        tree = lxml_html.fromstring(fragment)
    except Exception:  # noqa: BLE001 — same
        return "", []

    links: list[str] = []
    return _tidy(_blocks(tree, url, links)), links


def extract_page(
    html: str, url: str, visible_chars: int = 0
) -> tuple[str, list[str]] | None:
    """Main content of a page as markdown, plus the links worth following.

    ``None`` means the extractor found no text worth sending — the caller falls
    back to reading the page's own visible text rather than returning a menu.
    """
    body, links = _extract(html, url, comments=False)

    # A page whose content *is* its comment thread — a forum, a Q&A, a code
    # review — comes back as little more than a header when comments are off.
    # Keeping less than a quarter of what the reader can see is the signal that
    # the extractor threw the point of the page away, so ask again for all of it.
    if visible_chars and len(body) < visible_chars // 4:
        wide_body, wide_links = _extract(html, url, comments=True)
        if len(wide_body) > len(body):
            body, links = wide_body, wide_links

    if len(body) < MIN_TEXT:
        return None
    return body, links


def truncate(text: str, limit: int) -> str:
    """Cut at a paragraph boundary and name the sections that did not fit.

    Cutting mid-sentence left the model with a half-page and no sign that there
    was more; naming the headings that were dropped tells it whether the rest of
    the page is worth another request.
    """
    if len(text) <= limit:
        return text
    cut = text.rfind("\n\n", 0, limit)
    if cut < limit // 2:
        cut = limit
    head, tail = text[:cut].rstrip(), text[cut:]
    missing = [line.strip() for line in tail.splitlines() if _HEADING.match(line)]

    note = f"[… truncated at {limit} chars]"
    if missing:
        shown = ", ".join(missing[:8])
        more = f" and {len(missing) - 8} more" if len(missing) > 8 else ""
        note = f"[… truncated at {limit} chars. Sections not shown: {shown}{more}]"
    if head.count("```") % 2:
        head += "\n```"
    return f"{head}\n\n{note}"


def clean_visible_text(text: str) -> str:
    """Fallback path: the page's own visible text, with blank runs collapsed."""
    lines = [line.rstrip() for line in text.splitlines()]
    result: list[str] = []
    blank_run = 0
    for line in lines:
        if line == "":
            blank_run += 1
            if blank_run <= 2:
                result.append("")
            continue
        blank_run = 0
        result.append(line)
    return "\n".join(result).strip()