Newer
Older
navi-1 / navi / export / builder.py
"""Build the encrypted history snapshot served at /export/v1/messages.

Shape of a build
----------------
1. One read-only REPEATABLE READ transaction over both message tables, so the
   rolling window and the rows that follow come from the same instant.
2. The window: whole sessions, newest activity first, until the raw text
   reaches `navi_export_max_bytes`. Cutting on a session boundary keeps a
   conversation from arriving half a story.
3. Rows are streamed through a cursor and serialised one JSON object per line.
   The plaintext gzip is built deterministically (fixed mtime, sorted keys) so
   the same input always yields the same `content_id` — that is what lets the
   consumer answer "nothing changed" without downloading anything.
4. The plaintext is encrypted to `navi_export_age_recipient` and the plaintext is
   deleted. Only ciphertext stays on disk.

Nothing here reads `users`, `mcp_user_keys`, `api_tokens` or attachments.
Adding a field is a schema change: bump EXPORT_VERSION and say so in the
manifest, because the consumer validates `fields`.
"""

from __future__ import annotations

import gzip
import hashlib
import json
import os
from datetime import datetime, timezone
from pathlib import Path

import structlog

from navi.config import settings

log = structlog.get_logger()

EXPORT_VERSION = 1
FIELDS = ["v", "session", "seq", "role", "content", "ts"]
MANIFEST_NAME = "manifest.json"
FILE_NAME = "messages-v1.jsonl.gz.age"

# Whole sessions, newest first, while the cumulative text stays under budget.
_WINDOW_SQL = """
WITH msgs AS (
    SELECT session_id, role, content, created_at, is_summary, is_compression
    FROM session_messages
    UNION ALL
    SELECT session_id, role, content, created_at, is_summary, is_compression
    FROM session_messages_archive
),
buckets AS (
    SELECT session_id,
           max(created_at) AS last_ts,
           sum(octet_length(content))::bigint AS text_bytes
    FROM msgs
    WHERE role = ANY($1::text[])
      AND content IS NOT NULL AND content <> ''
      AND NOT coalesce(is_summary, false)
      AND NOT coalesce(is_compression, false)
    GROUP BY session_id
),
ordered AS (
    SELECT session_id, last_ts, text_bytes,
           sum(text_bytes) OVER (ORDER BY last_ts DESC NULLS LAST, session_id) AS cum
    FROM buckets
)
SELECT session_id
FROM ordered
WHERE cum - text_bytes < $2
ORDER BY last_ts DESC NULLS LAST, session_id
"""

_SESSIONS_SQL = """
SELECT count(*) FROM (
    SELECT session_id
    FROM (
        SELECT session_id, role, content, created_at, is_summary, is_compression
        FROM session_messages
        UNION ALL
        SELECT session_id, role, content, created_at, is_summary, is_compression
        FROM session_messages_archive
    ) m
    WHERE role = ANY($1::text[])
      AND content IS NOT NULL AND content <> ''
      AND NOT coalesce(is_summary, false)
      AND NOT coalesce(is_compression, false)
    GROUP BY session_id
) t
"""

_ROWS_SQL = """
SELECT session_id, sequence_number, role, content, created_at
FROM (
    SELECT session_id, sequence_number, role, content, created_at, is_summary, is_compression
    FROM session_messages
    UNION
    SELECT session_id, sequence_number, role, content, created_at, is_summary, is_compression
    FROM session_messages_archive
) m
WHERE session_id = ANY($1::text[])
  AND role = ANY($2::text[])
  AND content IS NOT NULL AND content <> ''
  AND NOT coalesce(is_summary, false)
  AND NOT coalesce(is_compression, false)
ORDER BY created_at NULLS LAST, session_id, sequence_number
"""


def export_paths(out_dir: Path | None = None) -> tuple[Path, Path]:
    """(age file, manifest file) in the export directory."""
    base = Path(out_dir if out_dir is not None else settings.navi_export_dir)
    return base / FILE_NAME, base / MANIFEST_NAME


def read_manifest(out_dir: Path | None = None) -> dict | None:
    """The manifest on disk, or None when no export has been built yet."""
    _, manifest_path = export_paths(out_dir)
    try:
        return json.loads(manifest_path.read_text(encoding="utf-8"))
    except (OSError, ValueError):
        return None


def _record(row) -> dict:
    created = row["created_at"]
    ts = created.astimezone(timezone.utc).isoformat() if created else None
    return {
        "v": EXPORT_VERSION,
        "session": row["session_id"],
        "seq": row["sequence_number"],
        "role": row["role"],
        "content": row["content"],
        "ts": ts,
    }


async def build_export(pool, out_dir: Path | None = None) -> dict:
    """Build (or confirm) the snapshot and return the manifest.

    Raises when the export is not configured or the recipient key is malformed;
    the caller is a background loop that logs and retries on the next tick.
    """
    import pyrage

    if not settings.navi_export_configured:
        raise RuntimeError("export is not configured (NAVI_EXPORT_AGE_RECIPIENT / NAVI_EXPORT_TOKEN)")

    recipient = pyrage.x25519.Recipient.from_str(settings.navi_export_age_recipient.strip())
    roles = settings.navi_export_roles_list
    # 0 (or negative) disables the cap: the window then covers all history.
    budget = settings.navi_export_max_bytes if settings.navi_export_max_bytes > 0 else 2**62

    age_path, manifest_path = export_paths(out_dir)
    base = age_path.parent
    base.mkdir(parents=True, exist_ok=True)

    plain_tmp = base / (FILE_NAME + ".plain.tmp")
    age_tmp = base / (FILE_NAME + ".tmp")

    now = datetime.now(timezone.utc)
    rows = 0
    text_bytes = 0
    first_ts: datetime | None = None
    last_ts: datetime | None = None
    session_ids: list[str] = []
    total_sessions = 0

    digest = hashlib.sha256()
    try:
        async with pool.acquire() as conn, conn.transaction(
            isolation="repeatable_read", readonly=True
        ):
            session_ids = [
                r["session_id"] for r in await conn.fetch(_WINDOW_SQL, roles, budget)
            ]
            total_sessions = await conn.fetchval(_SESSIONS_SQL, roles) or 0

            # Blocking file I/O in a background job, once every few hours:
            # an executor hop here would buy nothing.
            with open(plain_tmp, "wb") as plain_fh, gzip.GzipFile(  # noqa: ASYNC230
                filename="",
                mode="wb",
                fileobj=plain_fh,
                compresslevel=9,
                mtime=0,
            ) as gz:
                if session_ids:
                    async for row in conn.cursor(_ROWS_SQL, session_ids, roles, prefetch=500):
                        line = json.dumps(
                            _record(row), ensure_ascii=False, sort_keys=True, separators=(",", ":")
                        ) + "\n"
                        chunk = line.encode("utf-8")
                        gz.write(chunk)
                        digest.update(chunk)
                        rows += 1
                        text_bytes += len(row["content"].encode("utf-8"))
                        created = row["created_at"]
                        if created is not None:
                            if first_ts is None or created < first_ts:
                                first_ts = created
                            if last_ts is None or created > last_ts:
                                last_ts = created
    except BaseException:
        plain_tmp.unlink(missing_ok=True)
        raise

    content_id = digest.hexdigest()
    previous = read_manifest(out_dir)

    # Same corpus as last time: keep the ciphertext (and its ETag) untouched.
    # Only the manifest's built_at moves, which is what the consumer ignores.
    if (
        previous
        and previous.get("content_id") == content_id
        and age_path.exists()
        and previous.get("file", {}).get("name") == FILE_NAME
    ):
        plain_tmp.unlink(missing_ok=True)
        manifest = {**previous, "built_at": now.isoformat()}
        _write_manifest(manifest_path, manifest)
        log.info("export.unchanged", content_id=content_id[:12], rows=rows)
        return manifest

    try:
        with open(plain_tmp, "rb") as src, open(age_tmp, "wb") as dst:  # noqa: ASYNC230
            pyrage.encrypt_io(src, dst, [recipient])
        file_bytes = age_tmp.stat().st_size
        file_sha = _sha256_file(age_tmp)
        os.replace(age_tmp, age_path)
    finally:
        plain_tmp.unlink(missing_ok=True)
        age_tmp.unlink(missing_ok=True)

    manifest = {
        "export_version": EXPORT_VERSION,
        "built_at": now.isoformat(),
        "content_id": content_id,
        "file": {"name": FILE_NAME, "bytes": file_bytes, "sha256": file_sha},
        "rows": rows,
        "sessions": len(session_ids),
        "sessions_total": total_sessions,
        "text_bytes": text_bytes,
        "fields": FIELDS,
        "roles": roles,
        "window": {
            "from": first_ts.astimezone(timezone.utc).isoformat() if first_ts else None,
            "to": last_ts.astimezone(timezone.utc).isoformat() if last_ts else None,
            "truncated": len(session_ids) < total_sessions,
            "max_bytes": budget,
        },
        "age_recipient": settings.navi_export_age_recipient.strip(),
    }
    _write_manifest(manifest_path, manifest)
    log.info(
        "export.built",
        rows=rows,
        sessions=len(session_ids),
        text_bytes=text_bytes,
        file_bytes=file_bytes,
        content_id=content_id[:12],
    )
    return manifest


def _write_manifest(path: Path, manifest: dict) -> None:
    tmp = path.with_suffix(".tmp")
    tmp.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    os.replace(tmp, path)


def _sha256_file(path: Path) -> str:
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for block in iter(lambda: f.read(1 << 20), b""):
            h.update(block)
    return h.hexdigest()