diff --git a/deploy/env.template b/deploy/env.template index a95fb16..69ea6cc 100644 --- a/deploy/env.template +++ b/deploy/env.template @@ -77,6 +77,10 @@ # rate-limited). The config maps this through ${SEARXNG_URL:-}, so a missing # value costs you the backend, not the server. See docs/mcp.md#navi-web-settings. #SEARXNG_URL=http://127.0.0.1:8888 +# Optional Brave Search API key (third backend, used when SearXNG and the +# ddgs fallback both fail). Empty means off — the config maps it with a ":-" +# fallback, so the server still connects without it. +#NAVI_MCP_NAVI_WEB_BRAVE_KEY= # ─── MCP server credentials ────────────────────────────────────────── # mcp_servers.d/*.json is tracked in git and must never hold a live token: it diff --git a/docs/config.md b/docs/config.md index 7e136ad..5dd821d 100644 --- a/docs/config.md +++ b/docs/config.md @@ -125,10 +125,11 @@ Web search is not a `Settings` field: it is the `navi-web` MCP server. Its settings reach it through `mcp_servers.d/navi-web.json`, which maps `SEARXNG_URL` (and, -optionally, `BRAVE_SEARCH_API_KEY`) from `.env` into the server's own environment. -`SEARXNG_URL` is therefore a real `.env` setting — a self-hosted SearXNG, e.g. -`http://localhost:8888` — just not one Navi itself reads. Without it the server -falls back to DuckDuckGo scraping. See [`mcp.md`](mcp.md#navi-web-settings). +optionally, `NAVI_MCP_NAVI_WEB_BRAVE_KEY`) from `.env` into the server's own +environment. `SEARXNG_URL` is therefore a real `.env` setting — a self-hosted +SearXNG, e.g. `http://localhost:8888` — just not one Navi itself reads. Without +it the server falls back to the pinned `ddgs` engines. See +[`mcp.md`](mcp.md#navi-web-settings). ## LLM Timeouts diff --git a/docs/mcp.md b/docs/mcp.md index 0faf4b6..a41a843 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -87,15 +87,26 @@ |---|---|---| | `NAVI_WEB_MCP_TRANSPORT` | — | `stdio`, declared inline | | `NAVI_WEB_SEARXNG_URL` | `SEARXNG_URL` (via `${SEARXNG_URL:-}`) | Self-hosted SearXNG base URL — the primary backend | -| `NAVI_WEB_BRAVE_API_KEY` | `BRAVE_SEARCH_API_KEY` | Optional third backend. Add the placeholder only once the key exists: an unset placeholder without a fallback fails the connection (see above) | +| `NAVI_WEB_BRAVE_API_KEY` | `NAVI_MCP_NAVI_WEB_BRAVE_KEY` (via `${NAVI_MCP_NAVI_WEB_BRAVE_KEY:-}`) | Optional third backend. Empty means off; the key is optional, so the placeholder carries a fallback (see above) | | `NAVI_WEB_DDG_REGION` | — | DuckDuckGo region, default `ua-uk` | -Without `NAVI_WEB_SEARXNG_URL` the server starts straight from DuckDuckGo HTML -scraping — roughly an order of magnitude slower, and the first backend to be -rate-limited. The `${SEARXNG_URL:-}` fallback is what keeps a host with no -SearXNG working: an unset value costs the backend, not the server's three tools. -`navi.config.Settings` has no search fields: `SEARXNG_URL` is read here, not by -Navi itself. +Without `NAVI_WEB_SEARXNG_URL` the server starts straight at the `ddgs` merge — +the engines pinned in `search._DDG_BACKENDS` (DuckDuckGo, Brave, Mojeek, Yahoo, +Wikipedia, Grokipedia; Yandex is excluded on purpose). It is slower and the +first thing to be rate-limited. The `${SEARXNG_URL:-}` fallback is what keeps a +host with no SearXNG working: an unset value costs the backend, not the server's +three tools. `navi.config.Settings` has no search fields: `SEARXNG_URL` is read +here, not by Navi itself. + +Two details in the SearXNG request are deliberate. The `engines` list is pinned +(`google,duckduckgo`) because the instance's own defaults are mostly suspended, +and **bing is excluded**: from a host it cannot match, Bing answers with +trending/homepage blocks, and a non-empty list of noise otherwise wins the chain +over a working fallback. The `language` is `all`, not a fixed tag — a pinned +`uk-UA` made the instance's DuckDuckGo engine return CAPTCHA on every request +(`kl=ua-uk`) and skewed English queries. The same trap exists in `ddgs` v9: +`backend` takes engine names, and an unknown name does not fail — it silently +means "all engines". #### Batching is the latency lever diff --git a/mcp-servers/navi-web/app/search.py b/mcp-servers/navi-web/app/search.py index 48faf2a..3dafd14 100644 --- a/mcp-servers/navi-web/app/search.py +++ b/mcp-servers/navi-web/app/search.py @@ -13,13 +13,24 @@ # Every wait on this path is bounded, including the chain as a whole. The caller # is sitting on this call, and the backends used to add up: 15s SearXNG + two 5s # DDG attempts + a 2s pause + 15s Brave — half a minute in which a slow search -# and a broken one looked the same. The budgets below make the worst case -# ~12s, long enough for a healthy backend and short enough to still answer. +# and a broken one looked the same. The budgets below make the worst case ~20s. +# The DDG leg is the one that needs the room: when SearXNG is down it becomes the +# only backend, and a real multi-engine scrape behind it has been measured at +# 1-13s — a 4s cap cut healthy searches off as "timeouts". _SEARXNG_TIMEOUT = 6.0 -_DDG_TIMEOUT = 4.0 +_DDG_TIMEOUT = 10.0 _BRAVE_TIMEOUT = 6.0 _DDG_RETRY_DELAY = 1.0 -_TOTAL_TIMEOUT = 12.0 +_TOTAL_TIMEOUT = 20.0 + +# Which ddgs engines the fallback may use. ``backend`` is a comma-separated list +# of engine names (ddgs v9: duckduckgo, brave, mojeek, yahoo, yandex, wikipedia, +# grokipedia) whose results ddgs merges. It used to read "html", which is not an +# engine name: ddgs warns "backends do not exist or are disabled" and silently +# runs *auto* — every engine, yandex and grokipedia included, shuffled. That is +# where the slow, flaky, mixed-provenance results came from. The set is pinned +# explicitly so it is known, and yandex is excluded on purpose. +_DDG_BACKENDS = "duckduckgo,brave,mojeek,yahoo,wikipedia,grokipedia" # One call may carry several independent queries: they run concurrently, so the # call costs the slowest instead of the sum. The cap is what keeps a batch from @@ -169,7 +180,11 @@ # 1. SearXNG (primary). Its engine list is pinned on purpose: the instance's # own defaults are mostly suspended, and without ``engines`` it returns - # nothing at all. + # nothing at all. Bing is left out — from this host it answers with trending + # and homepage blocks for queries it cannot match (Block Blast for a + # motherboard query), and a non-empty list of noise is indistinguishable from + # an answer here, so it used to win the chain outright. Yandex is left out by + # the owner's choice; the direct DDGS leg does not ask for it either. if settings.searxng_url: try: results = await asyncio.wait_for( @@ -221,16 +236,18 @@ def _ddg(settings: Settings, query: str, max_results: int) -> list[dict[str, Any]]: - """One DuckDuckGo attempt, off the event loop. + """One ddgs attempt, off the event loop. ``timeout`` is a ``DDGS`` client argument, not a ``text()`` one — passing it - to ``text()`` would land in ``**kwargs`` and do nothing. + to ``text()`` would land in ``**kwargs`` and do nothing. ``backend`` names the + engines to merge (see ``_DDG_BACKENDS``); an unknown name silently widens the + set to all of them. """ ddgs = DDGS(timeout=int(_DDG_TIMEOUT)) return list( ddgs.text( query, - backend="html", + backend=_DDG_BACKENDS, max_results=max_results, region=settings.ddg_region, ) @@ -244,8 +261,12 @@ params={ "q": query, "format": "json", - "engines": "google,bing,duckduckgo", - "language": "uk-UA", + "engines": "google,duckduckgo", + # Not a fixed language: a pinned "uk-UA" made the instance's + # DuckDuckGo engine answer with CAPTCHA (kl=ua-uk) on every + # request and skewed results for English queries. "all" lets each + # engine read the query's own language. + "language": "all", }, ) resp.raise_for_status() diff --git a/mcp_servers.d/navi-web.json b/mcp_servers.d/navi-web.json index 1274aaf..c4ebb3a 100644 --- a/mcp_servers.d/navi-web.json +++ b/mcp_servers.d/navi-web.json @@ -8,7 +8,8 @@ "cwd": "./mcp-servers/navi-web", "env": { "NAVI_WEB_MCP_TRANSPORT": "stdio", - "NAVI_WEB_SEARXNG_URL": "${SEARXNG_URL:-}" + "NAVI_WEB_SEARXNG_URL": "${SEARXNG_URL:-}", + "NAVI_WEB_BRAVE_API_KEY": "${NAVI_MCP_NAVI_WEB_BRAVE_KEY:-}" }, "groups": { "search": [ diff --git a/tests/unit/mcp/test_navi_web_search.py b/tests/unit/mcp/test_navi_web_search.py index 20d692c..ad07271 100644 --- a/tests/unit/mcp/test_navi_web_search.py +++ b/tests/unit/mcp/test_navi_web_search.py @@ -202,10 +202,70 @@ assert navi_web_search._ddg(navi_web_settings, "q", 3)[0]["title"] == "t" assert seen["timeout"] == int(navi_web_search._DDG_TIMEOUT) assert seen["query"] == "q" - assert seen["backend"] == "html" + assert seen["backend"] == navi_web_search._DDG_BACKENDS assert seen["max_results"] == 3 assert seen["region"] == "ru-ru" + def test_the_pinned_engines_are_real_ddgs_names(self, navi_web_search): + """An unknown name is not an error in ddgs — it silently means "all". + + ``backend="html"`` did exactly that, quietly adding yandex and every other + engine to the merge. Pin the names against the library's own engine map. + """ + from ddgs.engines import ENGINES + + known = set(ENGINES["text"].keys()) + pinned = set(navi_web_search._DDG_BACKENDS.split(",")) + assert pinned <= known, f"unknown ddgs backends: {sorted(pinned - known)}" + assert "yandex" not in pinned + + +class TestSearxngRequest: + async def test_engines_and_language_are_pinned( + self, navi_web_search, monkeypatch, navi_web_settings + ): + """Bing is deliberately absent and the language is not fixed. + + Bing answered with trending/homepage noise on this host and, being + non-empty, won the chain; a pinned language made the instance's DDG engine + return CAPTCHA on every request. + """ + seen: dict = {} + + class FakeResponse: + def raise_for_status(self): + pass + + def json(self): + return {"results": [{"title": "t", "url": "https://e", "content": "c"}]} + + class FakeClient: + def __init__(self, timeout=None): + seen["timeout"] = timeout + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def get(self, url, params=None): + seen["url"], seen["params"] = url, params + return FakeResponse() + + monkeypatch.setattr(navi_web_search.httpx, "AsyncClient", FakeClient) + navi_web_settings.searxng_url = "http://searx" + + results = await navi_web_search._searxng(navi_web_settings, "q", 5) + + assert results == [{"title": "t", "href": "https://e", "body": "c"}] + assert seen["url"] == "http://searx/search" + engines = seen["params"]["engines"].split(",") + assert "bing" not in engines + assert "yandex" not in engines + assert seen["params"]["language"] == "all" + assert seen["params"]["format"] == "json" + class TestBatch: """One call may carry up to three independent queries.