"""Breadcrumb keys — the names a later search will ask by.
When compression drops a run of messages, the summary is the only thing left in
the model's context. A summary is good at the shape of what happened and bad at
the exact things a person later says out loud: a file name, a command, a port, an
amount, a ticket number. Those are exactly the strings a search query is made of.
So every compression also stores a breadcrumb list — the entities that were
dropped — and the breadcrumb index is searched alongside the chunk text. The list
comes from two places: the summarizer writes a `KEYS:` line (the prompt asks for
it), and when that line is missing — the hard-truncate fallback never sees a
summarizer at all — a deterministic extractor reads the dropped messages directly.
The extractor errs toward including a name: a breadcrumb that nobody searches
for costs a few bytes, one that is missing costs the detail.
"""
from __future__ import annotations
import re
# Enough names to find a chunk by, few enough that the index stays a list and not
# a copy of the conversation.
MAX_BREADCRUMBS = 40
_MAX_ITEM_CHARS = 120
_KEYS_LINE = re.compile(r"^\s*(?:\*\*)?KEYS(?:\*\*)?\s*[::]\s*(.*)$", re.IGNORECASE)
_KEYS_NOISE = frozenset({"", "none", "n/a", "—", "-", "ничего", "нет"})
# Inline code first: commands, flags, paths and identifiers the agent wrote
# deliberately are the strongest breadcrumbs in a transcript.
_BACKTICK = re.compile(r"`([^`\n]{2,80})`")
_URL = re.compile(r"https?://[^\s<>\"')\]]{4,200}")
# A path has at least one slash and at least one dotted segment or a leading
# slash/dot — `foo/bar` alone is too often prose ("and/or").
_PATH = re.compile(
r"(?<![\w/])(?:~|\.{1,2})?/?(?:[\w.-]+/)+[\w.-]*\.[A-Za-z0-9]{1,8}\b"
r"|(?<![\w/])/(?:[\w.-]+/)+[\w.-]+"
)
_SNAKE = re.compile(r"\b[a-z][a-z0-9]*(?:_[a-z0-9]+)+\b")
_CAMEL = re.compile(r"\b(?:[A-Z][a-z0-9]+){2,}\b")
_CONST = re.compile(r"\b[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\b")
_AMOUNT = re.compile(
r"(?<![\w.])\d[\d\s.,]*\s?"
r"(?:%|GB|MB|TB|KB|ms|kg|km|°C|°F|USD|EUR|UAH|RUB|руб\.?|грн\.?|грн)\b",
re.IGNORECASE,
)
_TICKET = re.compile(r"(?<![\w&])#\d{1,6}\b")
_QUOTED = re.compile(r"\"([^\"\n]{3,60})\"")
# Ordered by signal strength; the cap trims the tail, so weaker patterns go last.
_PATTERNS = (
_URL,
_BACKTICK,
_PATH,
_CONST,
_SNAKE,
_CAMEL,
_AMOUNT,
_TICKET,
_QUOTED,
)
def split_keys_line(text: str) -> tuple[str, list[str]]:
"""Split the summarizer's trailing `KEYS:` line off the summary.
Returns the summary without that line and the parsed keys. A summary with no
`KEYS:` line comes back unchanged with an empty list — the caller then falls
back to :func:`extract_breadcrumbs` on the dropped messages.
"""
if not text:
return text, []
lines = text.splitlines()
for index in range(len(lines) - 1, -1, -1):
match = _KEYS_LINE.match(lines[index])
if match is None:
continue
keys = _parse_keys(match.group(1))
remaining = lines[:index] + lines[index + 1:]
return "\n".join(remaining).rstrip() + ("\n" if text.endswith("\n") else ""), keys
return text, []
def _parse_keys(raw: str) -> list[str]:
keys: list[str] = []
seen: set[str] = set()
for part in re.split(r"[,;]", raw):
item = re.sub(r"^[-*•]\s*", "", part.strip())
item = item.strip("`\"'*").strip()
if item.lower() in _KEYS_NOISE:
continue
if not item or len(item) > _MAX_ITEM_CHARS:
continue
if item.lower() in seen:
continue
seen.add(item.lower())
keys.append(item)
if len(keys) >= MAX_BREADCRUMBS:
break
return keys
def extract_breadcrumbs(texts: list[str], limit: int = MAX_BREADCRUMBS) -> list[str]:
"""Pull the entities worth indexing out of dropped message text.
Deterministic on purpose: the same conversation must produce the same index
keys on every replay, otherwise a chunk written twice gets two different
breadcrumb sets.
"""
found: list[str] = []
seen: set[str] = set()
if limit <= 0:
return found
for text in texts:
if not text:
continue
for pattern in _PATTERNS:
for match in pattern.finditer(text):
item = match.group(1) if match.groups() else match.group(0)
item = item.strip().strip("`\"'").strip()
if not _keep(item):
continue
key = " ".join(item.lower().split())
if key in seen:
continue
seen.add(key)
found.append(item)
if len(found) >= limit:
return found
return found
_GENERIC = frozenset({
"true", "false", "none", "null", "and/or", "etc", "e.g", "i.e",
})
def _keep(item: str) -> bool:
if len(item) < 2 or len(item) > _MAX_ITEM_CHARS:
return False
if item.lower() in _GENERIC:
return False
# A breadcrumb must carry at least one letter or digit; pure punctuation
# shows up in backticks and quoted strings often enough to matter.
return any(ch.isalnum() for ch in item)