Newer
Older
navi-1 / navi / tools / _filesystem / helpers.py
"""Pure filesystem helpers: path checks, formatting, diffs, AI text plumbing.

Nothing here touches a `Tool` instance — `tool` composes these.
"""

import difflib
import os
import re
from datetime import datetime
from pathlib import Path

from navi.config import settings

from .._internal.areas import allowed_areas, resolve_in_areas
from .._internal.base import (
    current_session_id,
    current_user_id,
    current_user_role,
    current_working_directory,
)

_READ_WARN_BYTES = 100_000  # 100 KB — add size warning in output
_READ_HARD_BYTES = 1_000_000  # 1 MB  — legacy constant, kept for compatibility
_READ_PAGE_LINES = 500  # default page size when a file is larger than this
_READ_PAGE_BYTES = 50_000  # …or when the text is larger than this
_LIST_MAX_ENTRIES = 500
_FIND_MAX_RESULTS = 200
_GREP_MAX_RESULTS = 100
# Directories a content search normally does not want to walk.  `include_ignored`
# turns this off for the unusual case.
_SKIP_DIRS = frozenset({
    ".git",
    ".hg",
    ".svn",
    ".venv",
    "venv",
    "env",
    "node_modules",
    "__pycache__",
    ".mypy_cache",
    ".pytest_cache",
    ".ruff_cache",
    "target",
    "dist",
    "build",
})

# AI actions: ~20k tokens of file content per chunk (4 chars ≈ 1 token)
_AI_CHUNK_CHARS = 80_000
_AI_OVERLAP_LINES = 30
# smart_edit: refuse files larger than ~50k tokens (full file must fit in one call)
_AI_EDIT_MAX_CHARS = 200_000

# ── System prompts ────────────────────────────────────────────────────────────

_QUERY_SINGLE_SYSTEM = (
    "You are a precise file analysis assistant. "
    "Answer the question based strictly on the file content shown. "
    "Be specific and concise. Include line numbers when relevant."
)

_QUERY_CHUNK_SYSTEM = (
    "You are analyzing one section of a larger file. "
    "Answer the question using only the lines shown. "
    "If the answer is not present in this section, respond with exactly: NOT_FOUND\n"
    "Otherwise be specific and include line numbers."
)

_QUERY_SYNTHESIS_SYSTEM = (
    "Combine these partial findings from different sections of a file into one clear answer. "
    "Remove duplicates. Be direct and concise."
)

_EDIT_SYSTEM = (
    "You are a precise file editor. "
    "Given file content with line numbers and an instruction, output ONLY a JSON array of edit operations. "
    "No explanation, no markdown — just the JSON.\n\n"
    "Format:\n"
    "[\n"
    '  {"op": "replace", "start": LINE, "end": LINE, "content": "new text\\nmore lines"},\n'
    '  {"op": "delete",  "start": LINE, "end": LINE},\n'
    '  {"op": "insert",  "after": LINE, "content": "text to insert"}\n'
    "]\n\n"
    "Rules:\n"
    "- Line numbers are 1-based and inclusive\n"
    "- Use \\n in content strings for embedded newlines\n"
    "- Make MINIMAL changes to accomplish the instruction\n"
    "- 'insert' after=0 inserts before the first line\n"
    "- If no changes are needed, return []"
)


# ── Path helpers ──────────────────────────────────────────────────────────────


def _resolve_relative_path(
    p: Path, user_id: str | None, role: str | None, cwd: str | None = None
) -> Path:
    """Resolve a relative path against the session cwd when available.

    In single-user/legacy/admin mode, uses the client-provided working directory
    so that `filesystem read src/main.py` works from the project directory the
    user launched Navi Code in, regardless of the server's cwd.
    """
    if user_id and role != "admin":
        # Sandbox mode keeps its own resolution rules.
        return p

    cwd = cwd or current_working_directory.get(None)
    if cwd:
        return (Path(cwd) / p).resolve()
    return p.resolve()


def _check_path(
    path_str: str,
    user_id: str | None = None,
    role: str | None = None,
    cwd: str | None = None,
) -> Path | None:
    """Return resolved Path if access is allowed, else None.

    When a user_id is active (multi-user mode), all paths are resolved
    inside user_data/<user_id>/ or inside the current session directory.
    This prevents users from accessing each other's files or random OS
    directories, while still letting a user read what an upload put in the
    session and what share_file hands back.
    """
    if not path_str or path_str.strip() == "":
        return None
    try:
        p = Path(path_str)
    except Exception:
        return None

    user_id = user_id or current_user_id.get(None)
    role = role or current_user_role.get()

    # Admins bypass sandbox and use FS_ALLOWED_PATHS directly
    if user_id and role != "admin":
        # Sandboxed mode: the user's own sandbox, plus the current session
        # directory.  Relative paths keep resolving inside the sandbox.
        areas = allowed_areas(user_id, current_session_id.get(None))
        return resolve_in_areas(p, areas)

    # Fallback to FS_ALLOWED_PATHS for single-user / legacy mode / admin
    try:
        if not p.is_absolute():
            p = _resolve_relative_path(p, user_id, role, cwd=cwd)
        else:
            p = p.expanduser().resolve()
    except Exception:
        return None

    if settings.fs_allowed_paths.strip() == "*":
        return p

    allowed = [Path(r).expanduser().resolve() for r in settings.fs_allowed_paths_list]
    for root in allowed:
        try:
            p.relative_to(root)
            return p
        except ValueError:
            continue
    return None


def _fmt_size(n: int) -> str:
    if n < 1024:
        return f"{n} B"
    if n < 1024**2:
        return f"{n / 1024:.1f} KB"
    if n < 1024**3:
        return f"{n / 1024**2:.1f} MB"
    return f"{n / 1024**3:.1f} GB"


def _fmt_time(ts: float) -> str:
    return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M")


# ── AI helpers (module-level, no self) ────────────────────────────────────────


def _number_lines(lines: list[str], start: int = 1) -> str:
    """Return file lines with 1-based line numbers, right-aligned."""
    width = len(str(start + len(lines) - 1))
    return "\n".join(f"{start + i:>{width}}: {line}" for i, line in enumerate(lines))


def _make_chunks(lines: list[str], target_chars: int, overlap: int) -> list[tuple[int, int]]:
    """
    Split lines into (start_idx, end_idx) chunks of at most target_chars characters.
    Consecutive chunks overlap by `overlap` lines to preserve boundary context.
    """
    if not lines:
        return [(0, 0)]
    total = len(lines)
    total_chars = sum(len(line) + 1 for line in lines)
    if total_chars <= target_chars:
        return [(0, total)]

    chunks: list[tuple[int, int]] = []
    start = 0
    while start < total:
        chars = 0
        end = start
        while end < total and chars < target_chars:
            chars += len(lines[end]) + 1
            end += 1
        chunks.append((start, end))
        if end >= total:
            break
        start = max(start + 1, end - overlap)  # always make progress

    return chunks or [(0, total)]


def _validate_ops(ops: list, total_lines: int) -> list[str]:
    errors: list[str] = []
    ranges: list[tuple[int, int, int]] = []
    for i, op in enumerate(ops):
        if not isinstance(op, dict):
            errors.append(f"op[{i}] is not a dict")
            continue
        kind = op.get("op")
        if kind not in ("replace", "delete", "insert"):
            errors.append(f"op[{i}] unknown type {kind!r}")
            continue
        if kind in ("replace", "delete"):
            s, e = op.get("start"), op.get("end")
            if not isinstance(s, int) or not isinstance(e, int):
                errors.append(f"op[{i}] start/end must be integers")
            elif s < 1 or e > total_lines or s > e:
                errors.append(f"op[{i}] range {s}-{e} out of bounds (file has {total_lines} lines)")
            else:
                ranges.append((s, e, i))
        elif kind == "insert":
            after = op.get("after")
            if not isinstance(after, int):
                errors.append(f"op[{i}] 'after' must be integer")
            elif after < 0 or after > total_lines:
                errors.append(f"op[{i}] 'after'={after} out of bounds (0–{total_lines})")

    # Overlapping replace/delete ranges make the result depend on sort order and
    # are almost always a model mistake.  Report them instead of guessing.
    ranges.sort()
    for (s1, e1, i1), (s2, _e2, i2) in zip(ranges, ranges[1:]):
        if s2 <= e1:
            errors.append(
                f"op[{i1}] range {s1}-{e1} overlaps op[{i2}] range {s2}-...; "
                "merge them into one operation"
            )
    return errors


def _apply_ops(lines: list[str], ops: list[dict]) -> list[str]:
    """Apply edit operations bottom-up (highest line first) to preserve line numbers."""
    sorted_ops = sorted(
        ops,
        key=lambda o: o.get("start", o.get("after", 0)),
        reverse=True,
    )
    result = list(lines)
    for op in sorted_ops:
        kind = op["op"]
        if kind == "replace":
            s = op["start"] - 1  # 0-based
            e = op["end"]  # exclusive (1-based end = exclusive 0-based end)
            new = op.get("content", "").split("\n")
            result[s:e] = new
        elif kind == "delete":
            s = op["start"] - 1
            e = op["end"]
            del result[s:e]
        elif kind == "insert":
            after = op["after"]  # insert after this 1-based line (0 = before line 1)
            new = op.get("content", "").split("\n")
            result[after:after] = new
    return result


_DIFF_HUNK_RE = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@")


def _number_diff(diff_lines: list[str]) -> list[str]:
    """Prefix each unified-diff content line with its real file line number.

    Numbers come from the enclosing ``@@ -old_start,n +new_start,n @@`` hunk:
    removed and context lines use the old (from) line number, added lines use the
    new (to) line number. The format ``{marker} {num:>width}│ {content}`` keeps the
    ``+``/``-``/`` `` marker first so existing ``startswith``-based highlighting
    stays valid; a fixed space separates the marker from the right-aligned number
    column. Hunk headers, file headers (``---``/``+++``) and the
    ``\\ No newline`` marker are left unchanged.
    """
    # First pass: assign numbers, track the max for column width.
    numbered: list[tuple[str, int | None, str]] = []  # (marker, num, content)
    max_num = 0
    old_line = 0
    new_line = 0
    for raw_line in diff_lines:
        # difflib.unified_diff keeps a trailing "\n" on content lines when the
        # input was fed with explicit newlines (see _unified_diff, which adds
        # "\n" so difflib does not emit a "\ No newline at end of file" marker).
        # Strip it here so the later "\n".join does not produce a blank line
        # between every diff line. No-op for the _diff action, whose lines come
        # from splitlines() without trailing newlines.
        line = raw_line.rstrip("\n")
        if line.startswith("@@"):
            m = _DIFF_HUNK_RE.match(line)
            if m:
                old_line = int(m.group(1))
                new_line = int(m.group(3))
            numbered.append(("", None, line))
            continue
        if line.startswith("---") or line.startswith("+++") or line.startswith("\\"):
            numbered.append(("", None, line))
            continue
        marker = line[0] if line else " "
        content = line[1:]
        if marker == "+":
            num = new_line
            new_line += 1
        elif marker == "-":
            num = old_line
            old_line += 1
        else:  # " " context — present in both old and new
            num = old_line
            old_line += 1
            new_line += 1
        if num > max_num:
            max_num = num
        numbered.append((marker, num, content))

    width = len(str(max_num)) if max_num else 1
    out: list[str] = []
    for marker, num, content in numbered:
        if num is None:
            out.append(content)
        else:
            out.append(f"{marker} {num:>{width}}│ {content}")
    return out


def _unified_diff(original: list[str], modified: list[str], path: Path) -> str:
    diff = list(
        difflib.unified_diff(
            [line + "\n" for line in original],
            [line + "\n" for line in modified],
            fromfile=f"a/{path.name}",
            tofile=f"b/{path.name}",
            lineterm="",
        )
    )
    return "\n".join(_number_diff(diff))


# ── Small-model recovery helpers ─────────────────────────────────────────────

_NUMBERED_LINE_RE = re.compile(r"^\s*\d+\s*[:│]\s")


def _atomic_write(path: Path, text: str) -> None:
    """Write through a sibling temp file so a failure never leaves a half file."""
    tmp = path.with_suffix(path.suffix + ".tmp")
    try:
        tmp.write_text(text, encoding="utf-8")
        os.replace(tmp, path)
    finally:
        tmp.unlink(missing_ok=True)


def _is_binary(path: Path) -> bool:
    """A NUL byte in the first 8 KB is the same cheap heuristic ripgrep uses."""
    try:
        with path.open("rb") as f:
            return b"\x00" in f.read(8192)
    except OSError:
        return False


def _looks_numbered(text: str) -> bool:
    """True when most lines look like the ``12: text`` column from a numbered read."""
    lines = [line for line in text.splitlines() if line.strip()]
    if not lines:
        return False
    hits = sum(1 for line in lines if _NUMBERED_LINE_RE.match(line))
    return hits >= max(1, len(lines) // 2)


def _nearest_lines(text: str, old: str, limit: int = 3) -> list[tuple[int, str]]:
    """Best-effort candidates for a failed exact edit.

    The longest non-blank line of ``old`` is the most distinctive needle; lines
    scoring at least 0.5 similarity are reported with their 1-based number.
    """
    needles = [line.strip() for line in old.splitlines() if line.strip()]
    if not needles:
        return []
    needle = max(needles, key=len)
    scored: list[tuple[float, int, str]] = []
    for number, line in enumerate(text.splitlines(), 1):
        ratio = difflib.SequenceMatcher(None, needle, line.strip()).ratio()
        if ratio >= 0.5:
            scored.append((ratio, number, line))
    scored.sort(key=lambda item: (-item[0], item[1]))
    return [(number, line) for _ratio, number, line in scored[:limit]]


def _context_around(
    lines: list[str], changed: list[int] | range, radius: int = 2
) -> str:
    """Numbered context lines around changed 1-based line numbers, ranges merged."""
    wanted = sorted({line for line in changed if 1 <= line <= len(lines)})
    if not wanted:
        return ""
    ranges: list[tuple[int, int]] = []
    for number in wanted:
        start = max(1, number - radius)
        end = min(len(lines), number + radius)
        if ranges and start <= ranges[-1][1] + 1:
            ranges[-1] = (ranges[-1][0], max(ranges[-1][1], end))
        else:
            ranges.append((start, end))
    width = len(str(len(lines)))
    out: list[str] = []
    for index, (start, end) in enumerate(ranges):
        if index:
            out.append("...")
        for number in range(start, end + 1):
            out.append(f"{number:>{width}}: {lines[number - 1]}")
    return "\n".join(out)


def _ignored_by_default(path: Path, root: Path) -> bool:
    try:
        parts = path.relative_to(root).parts
    except ValueError:
        return False
    return any(part in _SKIP_DIRS for part in parts[:-1])


def _iter_search_files(root: Path, glob_filter: str, include_ignored: bool):
    """Yield files for grep.  A file path yields itself; a directory walks it."""
    if root.is_file():
        yield root
        return
    pattern = glob_filter or "*"
    for candidate in root.rglob(pattern):
        if not candidate.is_file():
            continue
        if not include_ignored and _ignored_by_default(candidate, root):
            continue
        yield candidate