"""web_search — SearXNG primary, DDG fallback."""
from __future__ import annotations
import asyncio
from typing import Any
import httpx
from ddgs import DDGS
from .config import Settings
# Every wait on this path is bounded, including the chain as a whole. The caller
# is sitting on this call, and the backends used to add up: 15s SearXNG + two 5s
# DDG attempts + a 2s pause — half a minute in which a slow search and a broken
# one looked the same. The budgets below make the worst case ~20s. The DDG leg is
# the one that needs the room: when SearXNG is down it becomes the only backend,
# and a real multi-engine scrape behind it has been measured at 1-13s — a 4s cap
# cut healthy searches off as "timeouts".
_SEARXNG_TIMEOUT = 6.0
_DDG_TIMEOUT = 10.0
_DDG_RETRY_DELAY = 1.0
_TOTAL_TIMEOUT = 20.0
# Which ddgs engines the fallback may use. ``backend`` is a comma-separated list
# of engine names (ddgs v9: duckduckgo, mojeek, yahoo, yandex, wikipedia,
# grokipedia, brave) whose results ddgs merges. It used to read "html", which is
# not an engine name: ddgs warns "backends do not exist or are disabled" and
# silently runs *auto* — every engine, yandex and grokipedia included, shuffled.
# That is where the slow, flaky, mixed-provenance results came from. The set is
# pinned explicitly so it is known. Two names are left out on purpose: yandex by
# the owner's choice, and brave because brave.com refuses this host (its HTML
# engine answers nothing) and the Brave Search API it would stand in for refused
# to issue a key. An engine that cannot answer only costs time in the merge.
_DDG_BACKENDS = "duckduckgo,mojeek,yahoo,wikipedia,grokipedia"
# One call may carry several independent queries: they run concurrently, so the
# call costs the slowest instead of the sum. The cap is what keeps a batch from
# turning into a scrape of the search engine — three concurrent queries on one
# instance still answered in under a second here.
_MAX_BATCH = 3
def _normalize(raw: str | list[str]) -> tuple[list[str], list[str]]:
"""Turn the argument into the queries to run, plus notes worth telling the model."""
queries: list[str] = []
for item in [raw] if isinstance(raw, str) else raw:
query = str(item).strip()
if query and query not in queries:
queries.append(query)
notes: list[str] = []
if len(queries) > _MAX_BATCH:
notes.append(
f"Only the first {_MAX_BATCH} of {len(queries)} queries ran — "
"ask the rest in another call."
)
queries = queries[:_MAX_BATCH]
return queries, notes
async def web_search(
settings: Settings,
query: str | list[str],
max_results: int = 5,
) -> dict[str, Any]:
"""Search the web. Returns dict with success, output, error, metadata."""
queries, notes = _normalize(query)
if not queries:
return {
"success": False,
"output": "No query given.",
"error": "empty_query",
"metadata": {},
}
# ``return_exceptions`` rather than a try/except: ``_search_one`` already
# turns every backend failure into a result, so an exception here is one
# query's own, and the batch keeps the rest.
outcomes = await asyncio.gather(
*(_search_one(settings, q, max_results) for q in queries),
return_exceptions=True,
)
if len(queries) == 1 and not notes:
outcome = outcomes[0]
if isinstance(outcome, BaseException):
return {
"success": False,
"output": f"Search failed: {outcome}",
"error": str(outcome),
"metadata": {},
}
return outcome
return _combine(queries, outcomes, notes)
def _combine(
queries: list[str],
outcomes: list[Any],
notes: list[str],
) -> dict[str, Any]:
"""One block per query. A query that failed is named, not dropped: the model
can retry that one instead of repeating the batch, and a batch that answered
something is a success even when the rest did not."""
blocks: list[str] = []
per_query: list[dict[str, Any]] = []
answered = 0
for query, outcome in zip(queries, outcomes, strict=True):
if isinstance(outcome, BaseException):
blocks.append(f'### "{query}" -> failed: {outcome}')
per_query.append({"query": query, "success": False, "error": str(outcome)})
elif outcome["success"]:
answered += 1
blocks.append(f'### "{query}"\n{outcome["output"]}')
per_query.append(
{
"query": query,
"success": True,
"source": outcome["metadata"].get("source"),
}
)
else:
blocks.append(f'### "{query}" -> failed: {outcome["error"]}')
per_query.append(
{"query": query, "success": False, "error": outcome["error"]}
)
body = "\n\n".join(notes + blocks)
if not answered:
reasons = "; ".join(
f"{item['query']}: {item['error']}" for item in per_query
)
return {
"success": False,
"output": f"No query produced results. {reasons}",
"error": "all_queries_failed",
"metadata": {"queries": per_query},
}
return {
"success": True,
"output": body,
"error": None,
"metadata": {"queries": per_query},
}
async def _search_one(
settings: Settings,
query: str,
max_results: int,
) -> dict[str, Any]:
"""One query, bounded by the chain's own budget.
The deadline is per query on purpose: a batch waits for the slowest one, so
putting a single ceiling around the whole batch would let one hung backend
erase the answers the other queries already had.
"""
try:
return await asyncio.wait_for(
_search_chain(settings, query, max_results), timeout=_TOTAL_TIMEOUT
)
except TimeoutError:
return {
"success": False,
"output": (
f"Search did not finish within {_TOTAL_TIMEOUT:.0f}s "
f"(SearXNG -> DDG) for query: {query!r}"
),
"error": "timeout",
"metadata": {},
}
async def _search_chain(
settings: Settings,
query: str,
max_results: int,
) -> dict[str, Any]:
last_error = ""
# 1. SearXNG (primary). Its engine list is pinned on purpose: the instance's
# own defaults are mostly suspended, and without ``engines`` it returns
# nothing at all. Bing is left out — from this host it answers with trending
# and homepage blocks for queries it cannot match (Block Blast for a
# motherboard query), and a non-empty list of noise is indistinguishable from
# an answer here, so it used to win the chain outright. Yandex is left out by
# the owner's choice; the direct DDGS leg does not ask for it either.
if settings.searxng_url:
try:
results = await asyncio.wait_for(
_searxng(settings, query, max_results), timeout=_SEARXNG_TIMEOUT
)
if results:
return _format(results, source="SearXNG")
last_error = "SearXNG: empty response"
except TimeoutError:
last_error = f"SearXNG: no answer in {_SEARXNG_TIMEOUT:.0f}s"
except Exception as e:
last_error = f"SearXNG: {e}"
# 2. DuckDuckGo html fallback. One attempt is cheap; a second is worth it
# only when the first *failed* (a transient block or reset). An empty result
# is an answer — retrying it only buys another timeout.
for attempt in range(2):
try:
results = await asyncio.to_thread(_ddg, settings, query, max_results)
if results:
return _format(results, source="DDG")
last_error = "DDG: empty response"
break
except Exception as e:
last_error = f"DDG: {e}"
if attempt == 0:
await asyncio.sleep(_DDG_RETRY_DELAY)
return {
"success": False,
"output": f"All search backends failed for query: {query!r}\nLast error: {last_error}",
"error": last_error,
"metadata": {},
}
def _ddg(settings: Settings, query: str, max_results: int) -> list[dict[str, Any]]:
"""One ddgs attempt, off the event loop.
``timeout`` is a ``DDGS`` client argument, not a ``text()`` one — passing it
to ``text()`` would land in ``**kwargs`` and do nothing. ``backend`` names the
engines to merge (see ``_DDG_BACKENDS``); an unknown name silently widens the
set to all of them.
"""
ddgs = DDGS(timeout=int(_DDG_TIMEOUT))
return list(
ddgs.text(
query,
backend=_DDG_BACKENDS,
max_results=max_results,
region=settings.ddg_region,
)
)
async def _searxng(settings: Settings, query: str, max_results: int) -> list[dict]:
async with httpx.AsyncClient(timeout=_SEARXNG_TIMEOUT) as client:
resp = await client.get(
f"{settings.searxng_url}/search",
params={
"q": query,
"format": "json",
"engines": "google,duckduckgo",
# Not a fixed language: a pinned "uk-UA" made the instance's
# DuckDuckGo engine answer with CAPTCHA (kl=ua-uk) on every
# request and skewed results for English queries. "all" lets each
# engine read the query's own language.
"language": "all",
},
)
resp.raise_for_status()
results = resp.json().get("results", [])[:max_results]
return [
{"title": r["title"], "href": r["url"], "body": r.get("content", "")}
for r in results
]
def _format(results: list[dict], source: str) -> dict[str, Any]:
lines = [
f"[{i+1}] {r.get('title', '')} — {r.get('href') or r.get('url', '')}"
for i, r in enumerate(results)
]
return {
"success": True,
"output": f"[{source}]\n" + "\n".join(lines),
"error": None,
"metadata": {"results": results, "source": source},
}