"""tool_manual — reaching the right manual, and failing usefully.
The bug this covers: the tool looked up `manuals/<name>.md` by the exact string the
model sent, then fell back to `registry.get(name)`. The agent calls MCP tools by
their full name (`mcp__navi-3d__compile_scad`), but the file is named after the tool
(`compile_scad.md`) — so a hand-written manual that existed was unreachable, and the
agent silently got a thin schema dump instead. The executor resolves such names fine;
the tool that *documents* them did not.
"""
import pytest
from navi.core.registry import ProfileRegistry, ToolRegistry
from navi.profiles.base import ToolConfig, ToolScopeConfig
from navi.tools import tool_manual
from navi.tools._internal.base import ToolContext
from navi.tools.tool_manual import ToolManualTool
from tests.conftest_factory import FakeTool, make_profile
@pytest.fixture(autouse=True)
def no_user_enabled_tools(monkeypatch):
"""Every profile also picks up tools/enabled.json — a real file on the machine
running the tests. Tests must see exactly what they declare.
"""
monkeypatch.setattr("navi.core.tool_utils.load_user_enabled_tools", list)
@pytest.fixture
def manuals(tmp_path, monkeypatch):
"""A manuals/ tree this test owns, with the guides/ subdir beside it."""
root = tmp_path / "manuals"
(root / "guides").mkdir(parents=True)
monkeypatch.setattr(tool_manual, "MANUALS_DIR", root)
monkeypatch.setattr(tool_manual, "GUIDES_DIR", root / "guides")
return root
class FakeMcpManager:
"""Stand-in for McpManager — group name to tool names, as a server config holds them."""
def __init__(self, groups: dict[str, dict[str, list[str]]] | None = None) -> None:
self._groups = groups or {}
def resolve_group(self, server_name: str, group_name: str) -> list[str]:
return list(self._groups.get(server_name, {}).get(group_name, []))
def build(registry=None, profiles=None, mcp_manager=None) -> ToolManualTool:
return ToolManualTool(
registry=registry, profile_registry=profiles, mcp_manager=mcp_manager
)
def registry_with(*tools: FakeTool) -> ToolRegistry:
registry = ToolRegistry()
for tool in tools:
registry.register(tool, builtin=True)
return registry
def profiles_with(profile) -> ProfileRegistry:
registry = ProfileRegistry()
registry.register(profile)
return registry
def profile_with(profile_id="developer", native=(), mcp=None, subagent_native=None):
"""A profile holding exactly the declared tools — no enabled_tools migration on top."""
subagent = (
ToolScopeConfig(native=list(subagent_native))
if subagent_native is not None
else ToolScopeConfig()
)
return make_profile(
profile_id,
enabled_tools=[],
tools=ToolConfig(
agent=ToolScopeConfig(native=list(native), mcp=dict(mcp or {})),
subagent=subagent,
),
)
class TestManualFiles:
async def test_a_bare_name_finds_the_manual(self, manuals):
(manuals / "compile_scad.md").write_text("# compile_scad\n\nhand-written\n")
result = await build().execute({"tool_name": "compile_scad"})
assert result.success is True
assert "hand-written" in result.output
async def test_the_full_mcp_name_reaches_the_hand_written_manual(self, manuals):
"""The regression: the agent calls the MCP spelling, the file is the bare one."""
(manuals / "compile_scad.md").write_text("# compile_scad\n\nhand-written\n")
result = await build().execute({"tool_name": "mcp__navi-3d__compile_scad"})
assert result.success is True
assert "hand-written" in result.output
async def test_an_underscored_server_name_reaches_it_too(self, manuals):
(manuals / "lint_scad.md").write_text("# lint_scad\n\nhand-written\n")
result = await build().execute({"tool_name": "mcp__navi_3d__lint_scad"})
assert "hand-written" in result.output
async def test_the_name_is_matched_case_insensitively(self, manuals):
(manuals / "compile_scad.md").write_text("# compile_scad\n\nhand-written\n")
result = await build().execute({"tool_name": "Compile_SCAD"})
assert "hand-written" in result.output
async def test_a_dash_spelling_finds_the_underscored_file(self, manuals):
(manuals / "compile_scad.md").write_text("# compile_scad\n\nhand-written\n")
result = await build().execute({"tool_name": "compile-scad"})
assert "hand-written" in result.output
async def test_a_guide_is_reachable_by_name(self, manuals):
"""Guides live in guides/ and are not tools, but naming one must still read it."""
(manuals / "guides" / "write_context_provider.md").write_text("# guide\n\nbody\n")
result = await build().execute({"tool_name": "write_context_provider"})
assert result.success is True
assert "body" in result.output
async def test_the_hand_written_manual_beats_the_generated_one(self, manuals):
(manuals / "filesystem.md").write_text("# filesystem\n\nhand-written\n")
registry = registry_with(FakeTool("filesystem", description="schema description"))
result = await build(registry=registry).execute({"tool_name": "filesystem"})
assert "hand-written" in result.output
assert "schema description" not in result.output
class TestGeneratedManual:
async def test_an_unwritten_manual_falls_back_to_the_schema(self, manuals):
registry = registry_with(FakeTool("todo", description="Manage the todo list"))
result = await build(registry=registry).execute({"tool_name": "todo"})
assert result.success is True
assert "# todo" in result.output
assert "Manage the todo list" in result.output
async def test_a_bare_mcp_name_resolves_against_the_profiles_toolset(self, manuals):
"""The same name the executor would resolve must resolve here — mcp group '*'."""
registry = registry_with(FakeTool("mcp__navi-web__web_search", description="Search"))
profiles = profiles_with(profile_with("developer", mcp={"navi-web": ["*"]}))
result = await build(registry=registry, profiles=profiles).execute(
{"tool_name": "web_search"}, ctx=ToolContext(profile_id="developer")
)
assert result.success is True
assert "# mcp__navi-web__web_search" in result.output
class TestGeneratedRendering:
"""The generated manual is what most tools get, so it has to render the whole
schema: a model that cannot see the shape of an array of objects guesses it."""
def tool(self, parameters, description="A tool"):
return FakeTool("shape_probe", description=description, parameters=parameters)
async def render(self, manuals, parameters, **kwargs):
registry = registry_with(self.tool({"type": "object", **parameters}))
result = await build(registry=registry, **kwargs).execute({"tool_name": "shape_probe"})
return result.output
async def test_the_header_names_the_source_and_the_schema(self, manuals):
out = await self.render(manuals, {"properties": {}})
assert "Generated from the tool's JSON schema (source: native)" in out
assert "not a hand-written manual" in out
async def test_an_mcp_tool_reports_its_server_as_the_source(self, manuals):
registry = registry_with(FakeTool("mcp__navi-3d__compile_scad", description="Scad"))
profiles = profiles_with(profile_with("developer", mcp={"navi-3d": ["*"]}))
result = await build(registry=registry, profiles=profiles).execute(
{"tool_name": "compile_scad"}, ctx=ToolContext(profile_id="developer")
)
assert "source: mcp__navi-3d__" in result.output
async def test_a_tool_with_no_parameters_says_so(self, manuals):
out = await self.render(manuals, {})
assert "takes no parameters" in out
async def test_a_nested_object_is_spelled_out(self, manuals):
out = await self.render(
manuals,
{
"properties": {
"filter": {
"type": "object",
"description": "Restrict the search",
"properties": {
"limit": {"type": "integer", "description": "How many"},
"tag": {"type": "string"},
},
"required": ["limit"],
}
}
},
)
assert "- `filter` (object, optional): Restrict the search" in out
assert "`limit` (integer, required): How many" in out
assert "`tag` (string, optional)" in out
async def test_an_array_of_objects_shows_the_item_shape(self, manuals):
out = await self.render(
manuals,
{
"properties": {
"entries": {
"type": "array",
"description": "Rows to write",
"items": {
"type": "object",
"properties": {
"name": {"type": "string", "description": "Key"},
"value": {"type": "number"},
},
"required": ["name"],
},
}
}
},
)
assert "(array of object, optional): Rows to write" in out
assert "each item:" in out
assert "`name` (string, required): Key" in out
assert "`value` (number, optional)" in out
async def test_an_array_of_scalars_names_the_item_type(self, manuals):
out = await self.render(manuals, {"properties": {"tags": {"type": "array", "items": {"type": "string"}}}})
assert "(array of string, optional)" in out
async def test_enum_and_default_are_shown(self, manuals):
out = await self.render(
manuals,
{
"properties": {
"action": {
"type": "string",
"description": "What to do",
"enum": ["add", "remove"],
"default": "add",
}
}
},
)
assert "one of: 'add', 'remove'" in out
assert "default: 'add'" in out
async def test_a_union_is_rendered_with_its_branches(self, manuals):
out = await self.render(
manuals,
{
"properties": {
"target": {
"description": "Where to write",
"oneOf": [
{"type": "string", "description": "A path"},
{"type": "array", "items": {"type": "string"}},
],
}
}
},
)
assert "(string | array of string, optional): Where to write" in out
assert "oneOf — one of:" in out
assert "• string — A path" in out
async def test_deep_nesting_is_capped_not_unbounded(self, manuals):
"""A pathological schema must not turn the manual into an infinite tree."""
spec = {"type": "string", "description": "bottom"}
for level in range(12):
spec = {
"type": "object",
"properties": {"next": spec},
"description": f"level {level}",
}
out = await self.render(manuals, {"properties": {"root": spec}})
assert "nested deeper" in out
assert out.count("- `next`") <= 6
class TestProfileScope:
async def test_a_registered_tool_warns_it_is_not_enabled_for_the_profile(self, manuals):
registry = registry_with(FakeTool("ssh_exec", description="Run a remote command"))
profiles = profiles_with(profile_with("secretary", native=["filesystem"]))
result = await build(registry=registry, profiles=profiles).execute(
{"tool_name": "ssh_exec"}, ctx=ToolContext(profile_id="secretary")
)
# Still documents it — the hint must not turn into a refusal — but says why
# calling it would fail.
assert result.success is True
assert "not enabled for profile 'secretary'" in result.output
assert "Run a remote command" in result.output
async def test_a_tool_only_in_the_subagent_scope_is_not_called_disabled(self, manuals):
"""A sub-agent asking about its own tool must not be told it lacks it."""
registry = registry_with(FakeTool("ssh_exec", description="Run a remote command"))
profiles = profiles_with(profile_with("developer", subagent_native=["ssh_exec"]))
result = await build(registry=registry, profiles=profiles).execute(
{"tool_name": "ssh_exec"}, ctx=ToolContext(profile_id="developer")
)
assert "not enabled" not in result.output
assert "Run a remote command" in result.output
async def test_without_an_active_profile_a_registered_tool_is_still_documented(self, manuals):
registry = registry_with(FakeTool("ssh_exec", description="Run a remote command"))
result = await build(registry=registry).execute({"tool_name": "ssh_exec"})
assert result.success is True
assert "not enabled for this run" in result.output
assert "Run a remote command" in result.output
class TestNoArgument:
"""The old code did params["tool_name"].strip() — KeyError when the model omits
the key, AttributeError when it sends null or a number. Neither is an answer, so
a missing name now returns the catalogue of what is documented."""
async def test_no_name_gives_the_catalogue_of_manuals(self, manuals):
(manuals / "filesystem.md").write_text("# filesystem\n")
(manuals / "todo.md").write_text("# todo\n")
result = await build().execute({})
assert result.success is True
assert "filesystem" in result.output
assert "todo" in result.output
assert "tool_name" in result.output
@pytest.mark.parametrize("bad", [None, 42, [], {"a": 1}])
async def test_a_non_string_name_is_not_a_crash(self, manuals, bad):
(manuals / "filesystem.md").write_text("# filesystem\n")
result = await build().execute({"tool_name": bad})
assert result.success is True
assert "filesystem" in result.output
async def test_a_whitespace_name_is_not_a_crash(self, manuals):
(manuals / "filesystem.md").write_text("# filesystem\n")
result = await build().execute({"tool_name": " "})
assert result.success is True
assert "filesystem" in result.output
async def test_no_manuals_at_all_is_not_a_crash(self, manuals):
result = await build().execute({})
assert result.success is False
assert result.error == "no_manuals"
class TestIndex:
async def test_manuals_are_grouped_by_source(self, manuals):
"""A manual is filed under the tool's name — which source it belongs to comes
from the registry, not from how the file happens to be spelled."""
(manuals / "filesystem.md").write_text("# filesystem\n")
(manuals / "web_search.md").write_text("# web_search\n")
registry = registry_with(
FakeTool("filesystem"), FakeTool("mcp__navi-web__web_search")
)
result = await build(registry=registry).execute({})
assert "native (1): filesystem" in result.output
assert "mcp__navi-web__ (1): web_search" in result.output
async def test_guides_are_listed_apart_from_tools(self, manuals):
(manuals / "filesystem.md").write_text("# filesystem\n")
(manuals / "guides" / "write_context_provider.md").write_text("# guide\n")
result = await build().execute({})
assert "guides (not tools; readable by name): write_context_provider" in result.output
assert "write_context_provider" not in result.output.split("guides")[0]
async def test_the_index_says_other_tools_still_have_a_manual(self, manuals):
(manuals / "filesystem.md").write_text("# filesystem\n")
result = await build().execute({})
assert "generated from its schema" in result.output
class TestMiss:
async def test_a_typo_gets_a_suggestion(self, manuals):
registry = registry_with(FakeTool("filesystem", description="Files"))
result = await build(registry=registry).execute({"tool_name": "filesistem"})
assert result.success is False
assert result.error == "not_found"
assert "filesystem" in result.output
async def test_a_typo_in_a_bare_mcp_name_gets_a_suggestion(self, manuals):
registry = registry_with(FakeTool("mcp__navi-web__web_search", description="Search"))
result = await build(registry=registry).execute({"tool_name": "web_serch"})
assert "web_search" in result.output
async def test_a_hopeless_name_still_points_at_list_tools(self, manuals):
result = await build().execute({"tool_name": "zzzz_qqqq_nonsense"})
assert result.success is False
assert "list_tools" in result.output
