Newer
Older
navi-1 / navi / overlay / apply.py
@Eugene Sukhodolskiy Eugene Sukhodolskiy 6 hours ago 14 KB overlay: ack the config again after a restart
"""Put a validated overlay into effect — at startup and on a live refresh.

Startup is simple: settings are constructed once, from base values plus the
overlay, and the other sections are picked up by the registry/container while
they build.

A live refresh is where the distinction between hot and staged matters:

  - **profiles** are rebuilt from the local files and swapped in the registry.
    The agent reads a profile per run, so the next LLM call sees it — the same
    guarantee ``switch_profile`` and the admin profile editor already rely on;
  - **hot settings** (secrets, timeouts, guards — see ``schema.py``) are
    written into the live singleton. Every module holds the same object and
    reads attributes at call time, so rotating a key takes effect without a
    restart;
  - **everything else** is reported as ``restart_required``. Claiming an
    effect that only happens on restart would be worse than asking for one.
"""

from __future__ import annotations

import os
import time
from typing import Callable

import structlog

from navi.overlay.fetch import Fetched, fetch_layers, send_ack, store_applied
from navi.overlay.merge import merge_patch
from navi.overlay.schema import HOT_SETTINGS
from navi.overlay.state import get_state, lock, status_summary
from navi.overlay.validate import validate_layers

log = structlog.get_logger()

_apply_hooks: list[Callable[[], None]] = []


def register_apply_hook(fn: Callable[[], None]) -> None:
    """Run *fn* after hot settings changed (caches built from old values)."""
    if fn not in _apply_hooks:
        _apply_hooks.append(fn)


def _run_apply_hooks() -> None:
    # Built-in: the default OAuth client caches client_id/secret at creation.
    try:
        from navi.auth.client import reset_default_client

        reset_default_client()
    except Exception:
        log.warning("hive.config.oauth_client_reset_failed", exc_info=True)
    for hook in list(_apply_hooks):
        try:
            hook()
        except Exception:
            log.warning("hive.config.apply_hook_failed", hook=repr(hook), exc_info=True)


# ── settings ────────────────────────────────────────────────────────────────


def build_settings(base, patch: dict):
    """Construct the process settings from local base + overlay settings."""
    if not patch:
        return base
    return type(base)(**{**base.model_dump(), **patch})


def _set_in_place(settings_obj, name: str, value: object) -> None:
    """Write one field into the frozen settings singleton.

    ``frozen=True`` forbids ``__setattr__``; ``object.__setattr__`` is the one
    supported escape hatch and is used deliberately, in one place, so a hot
    refresh is visible to every module that imported the object.
    """
    object.__setattr__(settings_obj, name, value)


def apply_settings_live(merged_settings: dict) -> tuple[list[str], list[str]]:
    """Apply hot fields in place; return (hot_applied, restart_required).

    Values removed from the overlay fall back to the local base, so a field
    can be handed back without editing any file.
    """
    state = get_state()
    base = state.base_settings
    previous = dict((state.merged or {}).get("settings") or {})
    if base is None:
        return [], []

    from navi.config import settings as live

    hot: list[str] = []
    restart: list[str] = []
    for name in set(previous) | set(merged_settings):
        if not hasattr(base, name):
            continue
        target = merged_settings.get(name, getattr(base, name))
        if getattr(live, name) == target:
            continue
        if name in HOT_SETTINGS:
            _set_in_place(live, name, target)
            hot.append(name)
        else:
            restart.append(name)

    if hot:
        _run_apply_hooks()
        log.info("hive.config.hot_settings_applied", fields=sorted(hot))
    return sorted(hot), sorted(restart)


# ── profiles ────────────────────────────────────────────────────────────────


def apply_profiles_list(profiles: list):
    """Return profiles with the overlay applied (used while building the registry)."""
    patch = (get_state().merged or {}).get("profiles") or {}
    if not patch:
        return profiles
    from navi.profiles.base import AgentProfile

    out = []
    for profile in profiles:
        if profile.id in patch:
            merged = merge_patch(profile.model_dump(), patch[profile.id])
            out.append(AgentProfile.model_validate(merged))
        else:
            out.append(profile)
    return out


def apply_profiles_registry(registry) -> None:
    """Rebuild every known profile from its local base and swap it in place.

    Rebuilding from the base (not from the current object) is what makes a
    refresh idempotent: a field dropped from the overlay is dropped from the
    running profile too, instead of lingering from the previous overlay.
    """
    patch = (get_state().merged or {}).get("profiles") or {}
    from navi.profiles import ALL_PROFILES
    from navi.profiles.base import AgentProfile

    base_by_id = {p.id: p for p in ALL_PROFILES}
    changed = False
    for current in registry.all():
        base = base_by_id.get(current.id)
        if base is None:
            continue  # created at runtime through the admin API — not ours
        merged = merge_patch(base.model_dump(), patch.get(current.id, {}))
        registry.update(AgentProfile.model_validate(merged))
        changed = True
    if changed:
        try:
            from navi.api.deps import get_agent

            get_agent().invalidate_prompt_cache()
        except Exception:
            log.warning("hive.config.prompt_cache_invalidate_failed", exc_info=True)
        log.info("hive.config.profiles_applied", profiles=sorted(patch))


# ── llm / mcp ───────────────────────────────────────────────────────────────


def overlay_llm_raw(raw: dict | None) -> dict | None:
    patch = (get_state().merged or {}).get("llm")
    if not patch:
        return raw
    return merge_patch(raw or {}, patch)  # type: ignore[return-value]


def overlay_mcp_configs(configs: dict) -> dict:
    patch = (get_state().merged or {}).get("mcp_servers")
    if patch is None:
        return configs
    from navi.mcp.config import McpServerConfig

    dumped = {name: cfg.model_dump() for name, cfg in configs.items()}
    merged = merge_patch(dumped, patch)
    return {
        name: McpServerConfig.model_validate(value)
        for name, value in merged.items()
        if isinstance(value, dict)
    }


# ── live refresh ────────────────────────────────────────────────────────────


def _staged_sections(previous: dict, merged: dict) -> list[str]:
    """Sections that changed but only take effect at the next start.

    ``llm`` and ``mcp_servers`` are consumed while the container is built —
    backends and the MCP pool do not survive a swap — so a refresh that
    changes them must say "restart", not quietly report the document applied.
    ``profiles`` are rebuilt live and settings are diffed separately.
    """
    staged = []
    for section in ("llm", "mcp_servers"):
        if (previous.get(section) or {}) != (merged.get(section) or {}):
            staged.append(section)
    return staged


def refresh(trigger: str = "notify") -> dict:
    """Fetch, validate and apply the current hive config. Never raises."""
    state = get_state()
    base = state.base_settings
    if base is None or not state.enabled:
        return {"status": "disabled", "trigger": trigger}
    if not lock.acquire(blocking=False):
        return {"status": "busy", "trigger": trigger}

    try:
        try:
            fetched = fetch_layers(base)
        except Exception as exc:
            state.status = "unreachable"
            state.last_error = f"{type(exc).__name__}: {exc}"
            log.warning("hive.config.fetch_failed", trigger=trigger, error=state.last_error)
            return {"status": "unreachable", "error": state.last_error, "trigger": trigger}

        if fetched is None:
            # No config channel (older hive) or no key — keep what we run.
            state.last_error = "hive has no config endpoint"
            return {"status": state.status, "trigger": trigger}

        validation = validate_layers(base, fetched.layers)
        state.last_fetch = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
        state.warnings = validation.warnings
        state.unknown_fields = validation.unknown_fields
        state.forbidden_fields = validation.forbidden_fields

        if not validation.ok:
            state.status = "rejected"
            state.errors = validation.errors
            state.last_error = validation.errors[0] if validation.errors else None
            log.error(
                "hive.config.rejected",
                trigger=trigger,
                version=fetched.version,
                hash=fetched.hash,
                errors=validation.errors,
            )
            # Tell the hive, so a rejected document is visible in its status
            # view instead of looking like a machine that never answered.
            send_ack(base, fetched, "rejected", [], validation.errors)
            return {
                "status": "rejected",
                "version": fetched.version,
                "hash": fetched.hash,
                "error": state.last_error,
                "trigger": trigger,
            }

        # Settings first (they may be read by the other sections). The apply
        # compares against the *previous* merged document to know which fields
        # were handed back to local values, so it must run before we replace
        # ``state.merged``.
        hot, restart = apply_settings_live(validation.merged.get("settings") or {})
        restart += _staged_sections(state.merged or {}, validation.merged)
        state.merged = validation.merged
        apply_profiles_registry_from_state()

        state.version = fetched.version
        state.hash = fetched.hash
        state.source = "hive"
        state.layers = [layer["name"] for layer in fetched.layers]
        state.errors = []
        state.last_error = None
        state.restart_required = restart
        state.hot_applied = hot
        state.status = "staged" if restart else "applied"
        store_applied(base, fetched)
        send_ack(base, fetched, state.status, restart, [])

        log.info(
            "hive.config.applied",
            trigger=trigger,
            version=fetched.version,
            hash=fetched.hash,
            status=state.status,
            hot=hot,
            restart_required=restart,
            unknown_fields=validation.unknown_fields,
            forbidden_fields=validation.forbidden_fields,
        )
        return {"status": state.status, **status_summary(), "trigger": trigger}
    finally:
        lock.release()


def apply_profiles_registry_from_state() -> None:
    """Refresh the live profile registry; safe to call before it exists."""
    try:
        from navi.api.deps import get_profile_registry

        apply_profiles_registry(get_profile_registry())
    except Exception:
        # Startup imports the settings long before the container exists; there
        # the registry is built from the overlay directly (apply_profiles_list).
        log.debug("hive.config.profile_registry_unavailable")


# ── startup ─────────────────────────────────────────────────────────────────


def overlay_disabled_by_env() -> bool:
    return os.environ.get("NAVI_OVERLAY", "").strip().lower() in {"off", "0", "false", "no"}


def bootstrap_settings(settings_cls):
    """Build the process settings: local base, then the hive overlay."""
    from navi.overlay.fetch import resolve_startup

    base = settings_cls()
    state = get_state()
    state.base_settings = base
    state.configured = bool(base.hive_url)

    if overlay_disabled_by_env() or not base.hive_config_enabled:
        state.enabled = False
        state.status = "disabled"
        return base
    if not base.hive_url:
        state.enabled = False
        state.status = "disabled"
        return base

    try:
        result = resolve_startup(base)
    except Exception as exc:
        state.status = "unreachable"
        state.last_error = f"{type(exc).__name__}: {exc}"
        log.exception("hive.config.startup_failed")
        return base

    state.boot_guard = result.boot_guard
    if result.validation is not None:
        state.warnings = result.validation.warnings
        state.unknown_fields = result.validation.unknown_fields
        state.forbidden_fields = result.validation.forbidden_fields
        state.errors = result.validation.errors
    if result.error:
        state.last_error = result.error

    if not result.layers:
        # Nothing to apply. Two different situations share this branch: the
        # hive answered "no layers for you" (source="hive" — a real answer,
        # the local config is final) and the hive was unreachable or the
        # document was rejected (source="local" — try again next boot).
        state.source = result.source
        if result.source == "hive":
            state.last_error = None
            state.status = "applied"
            state.version = result.version
            state.hash = result.hash
            state.layers = []
            state.last_fetch = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
        else:
            state.status = "unreachable" if result.error else "not_targeted"
        return base

    try:
        overlaid = build_settings(base, result.merged.get("settings") or {})
    except Exception as exc:  # validation already passed; belt and braces
        state.status = "rejected"
        state.errors = [f"settings: {exc}"]
        state.last_error = state.errors[0]
        log.exception("hive.config.settings_build_failed")
        return base

    state.merged = result.merged
    state.source = result.source
    state.last_error = None
    state.version = result.version
    state.hash = result.hash
    state.layers = [layer["name"] for layer in result.layers]
    state.status = "applied"
    state.last_fetch = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
    log.info(
        "hive.config.loaded",
        source=result.source,
        version=result.version,
        hash=result.hash,
        layers=state.layers,
        unknown_fields=state.unknown_fields,
        forbidden_fields=state.forbidden_fields,
    )
    return overlaid