"""Getting the overlay from the hive, and surviving the cases where it fails.
Two entry points:
``resolve_startup`` runs while ``navi.config`` is being imported. It fetches
with a short timeout, falls back to the last validated overlay on disk, and
runs the boot-loop guard: a config that was written as ``pending`` and never
confirmed means the previous start died with it, so it is demoted after a
second failure instead of crash-looping forever.
``refresh`` runs in a live process, triggered by a hive notification or by an
announce reply. It never raises into the caller: every failure becomes a
status line, because a broken config channel must not break the server that
is supposed to keep running.
The cache keeps the last **validated** overlay. A newly fetched document that
fails validation never overwrites it — that is what "last known good" means
here.
"""
from __future__ import annotations
import json
import os
import time
from dataclasses import dataclass, field
from pathlib import Path
import structlog
from navi.overlay.state import get_state
from navi.overlay.util import canonical_json, content_hash
from navi.overlay.validate import ValidationResult, validate_envelope, validate_layers
log = structlog.get_logger()
_CACHE_VERSION = 1
@dataclass
class Fetched:
version: int
layers: list[dict]
hash: str
note: str = ""
@dataclass
class CachedState:
good: dict | None = None
pending: dict | None = None
blocked_hash: str = ""
@dataclass
class StartupResult:
layers: list[dict] = field(default_factory=list)
source: str = "local"
version: int = 0
hash: str = ""
merged: dict = field(default_factory=dict)
validation: ValidationResult | None = None
error: str | None = None
boot_guard: bool = False
# ── local helpers ───────────────────────────────────────────────────────────
def _cache_path(base_settings) -> Path:
return Path(base_settings.hive_config_cache_file).expanduser()
def _read_key(path: str | Path) -> str | None:
try:
value = Path(path).read_text(encoding="utf-8").strip()
except OSError:
return None
return value or None
def _identity_params(path: str | Path) -> dict[str, str]:
"""Name, id and tags from instance.json.
Not ``ensure_identity``: at import time the file may legitimately not
exist yet (first start), and a config fetch must not create it as a side
effect. Without identity the hive still returns the global layers.
"""
try:
data = json.loads(Path(path).read_text(encoding="utf-8"))
except (OSError, ValueError):
return {}
params: dict[str, str] = {}
for key in ("name", "instance_id"):
value = data.get(key)
if value:
params[key] = str(value)
tags = data.get("tags")
if isinstance(tags, list) and tags:
params["tags"] = ",".join(str(t) for t in tags)
return params
def load_cached(path: Path) -> CachedState:
try:
raw = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return CachedState()
if not isinstance(raw, dict):
return CachedState()
def _slot(name: str) -> dict | None:
value = raw.get(name)
if not isinstance(value, dict) or not isinstance(value.get("layers"), list):
return None
return value
return CachedState(
good=_slot("good"),
pending=_slot("pending"),
blocked_hash=str(raw.get("blocked_hash") or ""),
)
def save_cached(path: Path, state: CachedState) -> None:
"""Atomic replace; a cache that cannot be written only costs a fetch."""
payload = {
"cache_version": _CACHE_VERSION,
"good": state.good,
"pending": state.pending,
"blocked_hash": state.blocked_hash,
}
tmp = path.with_name(path.name + ".tmp")
try:
tmp.write_text(
canonical_json(payload) + "\n",
encoding="utf-8",
)
try:
os.chmod(tmp, 0o600)
except OSError:
pass
os.replace(tmp, path)
except OSError:
log.warning("hive.config.cache_write_failed", path=str(path))
# ── HTTP ────────────────────────────────────────────────────────────────────
def fetch_layers(base_settings) -> Fetched | None:
"""GET the overlay for this instance.
``None`` means the hive has no config endpoint (an older hive) or answered
with something unusable; an empty layer list is a real answer ("publish
nothing"), which reverts an instance to its local config.
A missing swarm key raises: ``HIVE_URL`` without a key is a misconfigured
channel, not the hive saying "no overlay for you", and the status view
should send the operator to the key file rather than call the machine
untargeted.
"""
if not base_settings.hive_url:
return None
key = _read_key(base_settings.swarm_key_file)
if not key:
log.warning(
"hive.config.key_missing",
key_file=base_settings.swarm_key_file,
detail="HIVE_URL is set but there is no swarm key; central config disabled",
)
raise RuntimeError(f"swarm key missing: {base_settings.swarm_key_file}")
import httpx
url = base_settings.hive_url.rstrip("/") + "/config"
params = _identity_params(base_settings.instance_file)
try:
with httpx.Client(timeout=base_settings.hive_config_timeout_sec) as client:
resp = client.get(url, params=params, headers={"X-Swarm-Key": key})
except Exception as exc:
raise RuntimeError(f"{type(exc).__name__}: {exc}") from exc
if resp.status_code == 404:
return None
if resp.status_code != 200:
raise RuntimeError(f"HTTP {resp.status_code}: {resp.text[:200]}")
try:
data = resp.json()
except ValueError as exc:
raise RuntimeError(f"invalid JSON from hive: {exc}") from exc
if not isinstance(data, dict):
raise RuntimeError("hive answered with a non-object")
raw_layers = data.get("layers", [])
layers, _ = validate_envelope(raw_layers)
version = data.get("version", 0)
if not isinstance(version, int) or isinstance(version, bool):
version = 0
# Trust the bytes, not the header: the hash is what acks are compared by.
return Fetched(version=version, layers=layers, hash=content_hash(layers))
# ── startup ─────────────────────────────────────────────────────────────────
def resolve_startup(base_settings) -> StartupResult:
"""Pick the overlay this process starts with.
Candidates in order: an unconfirmed ``pending`` from the previous start
(one retry), the freshly fetched overlay, the last known good from disk.
The first one that validates wins; if none does, the local config stands.
"""
path = _cache_path(base_settings)
cached = load_cached(path)
fetched: Fetched | None = None
fetch_error: str | None = None
try:
fetched = fetch_layers(base_settings)
except Exception as exc:
fetch_error = str(exc)
log.warning("hive.config.fetch_failed", error=fetch_error)
if fetched is not None and cached.blocked_hash and fetched.hash != cached.blocked_hash:
# A new document was published — the guard was about one bad config,
# not about the channel.
cached.blocked_hash = ""
boot_guard = False
pending = cached.pending
if pending is not None:
attempts = int(pending.get("attempts") or 0)
if attempts >= 2:
boot_guard = True
cached.blocked_hash = str(pending.get("hash") or "")
cached.pending = None
save_cached(path, cached)
log.error(
"hive.config.boot_guard",
hash=cached.blocked_hash,
detail="the previous config prevented startup twice; keeping the last good one",
)
pending = None
candidates: list[tuple[str, Fetched]] = []
if pending is not None:
candidates.append(
(
"cache",
Fetched(
version=int(pending.get("version") or 0),
layers=list(pending.get("layers") or []),
hash=str(pending.get("hash") or ""),
),
)
)
if fetched is not None:
candidates.append(("hive", fetched))
if cached.good is not None:
candidates.append(
(
"cache",
Fetched(
version=int(cached.good.get("version") or 0),
layers=list(cached.good.get("layers") or []),
hash=str(cached.good.get("hash") or ""),
),
)
)
seen: set[str] = set()
last_validation: ValidationResult | None = None
for source, item in candidates:
if not item.hash:
item.hash = content_hash(item.layers)
if item.hash in seen or item.hash == cached.blocked_hash:
continue
seen.add(item.hash)
validation = validate_layers(base_settings, item.layers)
if not validation.ok:
last_validation = validation
log.error(
"hive.config.rejected",
source=source,
version=item.version,
hash=item.hash,
errors=validation.errors,
)
continue
# A config that differs from the last confirmed one is "pending"
# until the lifespan reaches its end; that is the boot-loop guard.
if cached.good is None or item.hash != cached.good.get("hash"):
previous = cached.pending or {}
attempts = (
int(previous.get("attempts") or 0) + 1
if previous.get("hash") == item.hash
else 1
)
cached.pending = {
"version": item.version,
"hash": item.hash,
"layers": item.layers,
"attempts": attempts,
}
else:
cached.pending = None
save_cached(path, cached)
return StartupResult(
layers=item.layers,
source=source,
version=item.version,
hash=item.hash,
merged=validation.merged,
validation=validation,
error=fetch_error,
boot_guard=boot_guard,
)
# Nothing usable. Keep whatever the cache says is good out of the way —
# the local config is the base, and the next boot will try again.
return StartupResult(
source="local",
validation=last_validation,
error=fetch_error,
boot_guard=boot_guard,
)
def confirm_startup(base_settings) -> None:
"""The process reached the end of startup: the pending config works."""
path = _cache_path(base_settings)
cached = load_cached(path)
if cached.pending is None:
return
cached.good = cached.pending
cached.pending = None
save_cached(path, cached)
log.info("hive.config.confirmed", version=cached.good.get("version"))
def store_applied(base_settings, fetched: Fetched) -> None:
"""A live refresh succeeded — this is now the config to boot from."""
path = _cache_path(base_settings)
cached = load_cached(path)
cached.good = {
"version": fetched.version,
"hash": fetched.hash,
"layers": fetched.layers,
"fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
}
cached.pending = None
save_cached(path, cached)