diff --git a/server/app/storage/__init__.py b/server/app/storage/__init__.py index 4c6f332..7639ad4 100644 --- a/server/app/storage/__init__.py +++ b/server/app/storage/__init__.py @@ -1,11 +1,16 @@ """PostgreSQL storage layer for Gnexus Book.""" from .db import Database, DatabaseError -from .factory import make_docs_repository, make_inventory_repository +from .factory import ( + make_docs_repository, + make_inventory_repository, + make_pending_change_repository, +) __all__ = [ "Database", "DatabaseError", "make_docs_repository", "make_inventory_repository", + "make_pending_change_repository", ] diff --git a/server/app/storage/db_repositories.py b/server/app/storage/db_repositories.py index 76d4997..deeeb63 100644 --- a/server/app/storage/db_repositories.py +++ b/server/app/storage/db_repositories.py @@ -23,7 +23,7 @@ ) -def _latest_doc_rows(session: Session): +def latest_doc_rows(session: Session): latest = _latest_doc_versions() return ( select(Document, DocVersion) @@ -46,7 +46,7 @@ def list_docs(self) -> list[dict[str, object]]: with self.database.session() as session: - rows = session.execute(_latest_doc_rows(session)).all() + rows = session.execute(latest_doc_rows(session)).all() return [ { "path": document.slug, @@ -59,7 +59,7 @@ def read_doc(self, path: str) -> dict[str, object]: with self.database.session() as session: row = session.execute( - _latest_doc_rows(session) + latest_doc_rows(session) .where(Document.slug == path) .limit(1) ).first() @@ -80,7 +80,7 @@ results: list[dict[str, object]] = [] with self.database.session() as session: - rows = session.execute(_latest_doc_rows(session)).all() + rows = session.execute(latest_doc_rows(session)).all() for document, version in rows: matches = [] for index, line in enumerate(version.raw.splitlines(), start=1): diff --git a/server/app/storage/factory.py b/server/app/storage/factory.py index cd7118d..413bd84 100644 --- a/server/app/storage/factory.py +++ b/server/app/storage/factory.py @@ -3,8 +3,10 @@ from ..config import Settings from ..docs_repository import DocsRepository from ..inventory import InventoryRepository +from ..pending_changes import PendingChangeRepository from .db import Database from .db_repositories import PostgresDocsRepository, PostgresInventoryRepository +from .pending_changes_db import PostgresPendingChangeRepository def make_docs_repository(settings: Settings): @@ -17,3 +19,11 @@ if settings.storage_backend == "db": return PostgresInventoryRepository(Database.from_settings(settings)) return InventoryRepository(settings) + + +def make_pending_change_repository(settings: Settings, via: str = "api"): + if settings.storage_backend == "db": + return PostgresPendingChangeRepository( + Database.from_settings(settings), settings, via=via + ) + return PendingChangeRepository(settings) diff --git a/server/app/storage/pending_changes_db.py b/server/app/storage/pending_changes_db.py new file mode 100644 index 0000000..5585294 --- /dev/null +++ b/server/app/storage/pending_changes_db.py @@ -0,0 +1,346 @@ +from __future__ import annotations + +import hashlib +from datetime import datetime +from pathlib import PurePosixPath +from typing import Any +from uuid import UUID + +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from ..config import Settings +from ..docs_repository import DocsRepository +from ..markdown import parse_frontmatter +from ..pending_changes import PendingChangeError, ProposedChangeRequest +from .db import Database +from .models import ( + AuditLog, + Blob, + ChangeItem, + Changeset, + DocVersion, + Document, + InventoryItem, + InventoryItemVersion, + InventoryType, + utcnow, +) +from .validation import validate_session + + +def _iso(value: datetime | None) -> str | None: + if value is None: + return None + return value.isoformat().replace("+00:00", "Z") + + +def _content_hash(raw: str) -> str: + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def _serialize(changeset: Changeset, items: list[ChangeItem]) -> dict[str, Any]: + primary = items[0] + return { + "id": str(changeset.id), + "kind": primary.kind, + "target": primary.target, + "summary": changeset.summary, + "reason": changeset.reason, + "payload": dict(primary.payload or {}), + "status": changeset.status, + "created_at": _iso(changeset.created_at), + "updated_at": _iso(changeset.applied_at or changeset.created_at), + "applied_at": _iso(changeset.applied_at), + "validation": changeset.validation, + "items": [ + { + "id": str(item.id), + "kind": item.kind, + "target": item.target, + "status": item.status, + "payload": dict(item.payload or {}), + } + for item in items + ], + } + + +class PostgresPendingChangeRepository: + def __init__(self, database: Database, settings: Settings, via: str = "api") -> None: + self.database = database + self.settings = settings + self.via = via + + def list(self) -> list[dict[str, Any]]: + with self.database.session() as session: + changesets = session.scalars( + select(Changeset).order_by(Changeset.created_at.desc()) + ).all() + result: list[dict[str, Any]] = [] + for changeset in changesets: + items = self._items(session, changeset.id) + if items: + result.append(_serialize(changeset, items)) + return result + + def get(self, change_id: str) -> dict[str, Any]: + with self.database.session() as session: + changeset = self._changeset(session, change_id) + items = self._items(session, changeset.id) + if not items: + raise PendingChangeError("Pending change not found") + return _serialize(changeset, items) + + def create(self, request: ProposedChangeRequest) -> dict[str, Any]: + with self.database.session() as session: + with session.begin(): + changeset = Changeset( + summary=request.summary, + reason=request.reason, + status="draft", + ) + session.add(changeset) + session.flush() + + item = ChangeItem( + changeset_id=changeset.id, + kind=request.kind, + target=request.target, + payload=request.payload, + status="draft", + ) + session.add(item) + session.flush() + + session.add( + AuditLog( + actor_kind="agent", + actor_name=None, + via=self.via, + action="propose", + entity_type="changeset", + entity_id=str(changeset.id), + after={ + "summary": request.summary, + "kind": request.kind, + "target": request.target, + }, + ) + ) + return _serialize(changeset, [item]) + + def apply(self, change_id: str) -> dict[str, Any]: + with self.database.session() as session: + with session.begin(): + changeset = self._changeset(session, change_id) + if changeset.status != "draft": + raise PendingChangeError("Only pending changes can be applied") + + items = self._items(session, changeset.id) + if not items: + raise PendingChangeError("Pending change has no items") + + for item in items: + if item.kind == "doc": + self._apply_doc(session, changeset, item) + elif item.kind == "inventory-item": + self._apply_inventory_item(session, changeset, item) + else: + raise PendingChangeError( + "Only doc and inventory-item changes can be applied in the current MVP" + ) + item.status = "applied" + + session.flush() + validation = validate_session(session) + if validation["status"] != "ok": + issues = validation.get("issues", []) + details = "\n".join( + f" - {issue['path']} ({issue['code']}): {issue['message']}" + for issue in issues[:10] + ) + raise PendingChangeError( + "Applied change failed validation and was rolled back. Issues:\n" + + details + ) + + changeset.status = "applied" + changeset.applied_at = utcnow() + changeset.validation = validation + session.add( + AuditLog( + actor_kind="agent", + actor_name=None, + via=self.via, + action="apply", + entity_type="changeset", + entity_id=str(changeset.id), + after={ + "summary": changeset.summary, + "targets": [item.target for item in items], + "validation": validation, + }, + ) + ) + return _serialize(changeset, items) + + def _apply_doc(self, session: Session, changeset: Changeset, item: ChangeItem) -> None: + target = item.target + if not target.endswith(".md") or target.startswith("/") or "\\" in target: + raise PendingChangeError("Doc change target must be a Markdown file") + parts = PurePosixPath(target).parts + if not parts or ".." in parts: + raise PendingChangeError("Doc change target escapes repository root") + + payload = item.payload or {} + content = payload.get("content") + if not isinstance(content, str) or not content.strip(): + raise PendingChangeError("Doc change payload.content must be a non-empty string") + if not content.endswith("\n"): + content += "\n" + + parsed = parse_frontmatter(target, content) + title = DocsRepository._extract_title(parsed.body) or PurePosixPath(target).stem + + document = session.scalars( + select(Document).where(Document.slug == target).with_for_update() + ).first() + if document is None: + document = Document(slug=target, title=title) + session.add(document) + session.flush() + version_no = 1 + parent_version_id = None + else: + if document.deleted_at is not None: + raise PendingChangeError("Document is deleted") + document.title = title + latest = session.scalars( + select(DocVersion) + .where(DocVersion.document_id == document.id) + .order_by(DocVersion.version_no.desc()) + .limit(1) + ).first() + version_no = (latest.version_no + 1) if latest else 1 + parent_version_id = latest.id if latest else None + + content_hash = _content_hash(content) + if session.get(Blob, content_hash) is None: + session.add(Blob(hash=content_hash, content=content, size=len(content))) + + session.add( + DocVersion( + document_id=document.id, + version_no=version_no, + parent_version_id=parent_version_id, + content_hash=content_hash, + raw=content, + body=parsed.body, + frontmatter=parsed.frontmatter, + actor_kind="agent", + actor_name=None, + via=self.via, + changeset_id=changeset.id, + ) + ) + + def _apply_inventory_item( + self, session: Session, changeset: Changeset, item: ChangeItem + ) -> None: + parts = item.target.split("/") + if len(parts) != 2 or not parts[0] or not parts[1]: + raise PendingChangeError("Inventory item target must use '/'") + inventory_type, item_id = parts + + inventory_type_row = session.get(InventoryType, inventory_type) + if inventory_type_row is None: + raise PendingChangeError("Inventory type not found") + + payload = item.payload or {} + mode = payload.get("mode", "update") + if mode not in {"update", "create"}: + raise PendingChangeError("Inventory change mode must be 'update' or 'create'") + + if "patch" in payload: + patch = payload["patch"] + elif "mode" in payload: + raise PendingChangeError("Inventory change payload.patch is required") + else: + patch = payload + + if not isinstance(patch, dict) or not patch: + raise PendingChangeError("Inventory change patch must be a non-empty object") + if "id" in patch: + raise PendingChangeError("Inventory item id cannot be changed") + + row = session.scalars( + select(InventoryItem) + .where( + InventoryItem.type == inventory_type, + InventoryItem.item_id == item_id, + InventoryItem.deleted_at.is_(None), + ) + .with_for_update() + ).first() + + if mode == "update": + if row is None: + raise PendingChangeError("Inventory item not found") + row.data = {**dict(row.data or {}), **patch} + latest = session.scalars( + select(InventoryItemVersion) + .where(InventoryItemVersion.inventory_item_id == row.id) + .order_by(InventoryItemVersion.version_no.desc()) + .limit(1) + ).first() + version_no = (latest.version_no + 1) if latest else 1 + version_item = row + else: + if row is not None: + raise PendingChangeError("Inventory item already exists") + max_position = session.scalar( + select(func.max(InventoryItem.position)).where( + InventoryItem.type == inventory_type + ) + ) + row = InventoryItem( + type=inventory_type, + item_id=item_id, + position=(max_position or 0) + 1, + data=dict(patch), + ) + session.add(row) + session.flush() + version_no = 1 + version_item = row + + session.add( + InventoryItemVersion( + inventory_item_id=version_item.id, + version_no=version_no, + data=dict(version_item.data or {}), + changeset_id=changeset.id, + ) + ) + + @staticmethod + def _changeset(session: Session, change_id: str) -> Changeset: + try: + identifier = UUID(str(change_id)) + except ValueError as exc: + raise PendingChangeError("Pending change not found") from exc + changeset = session.get(Changeset, identifier) + if changeset is None: + raise PendingChangeError("Pending change not found") + return changeset + + @staticmethod + def _items(session: Session, changeset_id: UUID) -> list[ChangeItem]: + return list( + session.scalars( + select(ChangeItem) + .where(ChangeItem.changeset_id == changeset_id) + .order_by(ChangeItem.created_at, ChangeItem.id) + ).all() + ) diff --git a/server/app/storage/protocols.py b/server/app/storage/protocols.py index 5ddd074..3c1bb3e 100644 --- a/server/app/storage/protocols.py +++ b/server/app/storage/protocols.py @@ -19,3 +19,13 @@ def read_parsed(self, inventory_type: str) -> Any: ... def read_item(self, inventory_type: str, item_id: str) -> dict[str, Any]: ... + + +class PendingChangeRepositoryProtocol(Protocol): + def list(self) -> list[dict[str, Any]]: ... + + def get(self, change_id: str) -> dict[str, Any]: ... + + def create(self, request: Any) -> dict[str, Any]: ... + + def apply(self, change_id: str) -> dict[str, Any]: ... diff --git a/server/app/storage/validation.py b/server/app/storage/validation.py new file mode 100644 index 0000000..3e5822f --- /dev/null +++ b/server/app/storage/validation.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +from typing import Any + +import yaml +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError +from sqlalchemy import select +from sqlalchemy.orm import Session + +from ..freshness import REQUIRED_FRONTMATTER +from ..validation import ( + PRIVATE_KEY_BLOCK_RE, + SAFE_SECRET_REFERENCE_RE, + SECRET_ASSIGNMENT_RE, +) +from .db import Database +from .db_repositories import latest_doc_rows +from .models import InventoryItem, InventoryType, JsonSchema + + +def _issue(path: str, code: str, message: str) -> dict[str, str]: + return { + "path": path, + "severity": "error", + "code": code, + "message": message, + } + + +def validate_database(database: Database) -> dict[str, object]: + with database.session() as session: + return validate_session(session) + + +def validate_session(session: Session) -> dict[str, object]: + issues: list[dict[str, str]] = [] + issues.extend(_validate_schemas(session)) + issues.extend(_validate_documents(session)) + issues.extend(_validate_inventory(session)) + return { + "status": "ok" if not issues else "issues", + "issue_count": len(issues), + "issues": issues, + } + + +def _validate_schemas(session: Session) -> list[dict[str, str]]: + issues: list[dict[str, str]] = [] + for schema in session.scalars(select(JsonSchema)).all(): + try: + Draft202012Validator.check_schema(schema.body) + except SchemaError as exc: + issues.append( + _issue(f"schemas/{schema.name}", "invalid-json-schema", exc.message) + ) + return issues + + +def _validate_documents(session: Session) -> list[dict[str, str]]: + issues: list[dict[str, str]] = [] + for document, version in session.execute(latest_doc_rows(session)).all(): + slug = document.slug + if slug == "README.md" or slug.startswith("server/"): + continue + + frontmatter = version.frontmatter or {} + if not frontmatter: + issues.append( + _issue(slug, "missing-frontmatter", "Markdown document is missing frontmatter") + ) + continue + + missing = sorted(REQUIRED_FRONTMATTER - set(frontmatter)) + if missing: + issues.append( + _issue( + slug, + "missing-frontmatter-fields", + "Missing frontmatter fields: " + ", ".join(missing), + ) + ) + + _scan_text(slug, version.raw, issues) + return issues + + +def _validate_inventory(session: Session) -> list[dict[str, str]]: + issues: list[dict[str, str]] = [] + schemas = {schema.id: schema for schema in session.scalars(select(JsonSchema)).all()} + + for inventory_type in session.scalars( + select(InventoryType).order_by(InventoryType.name) + ).all(): + path = f"40-inventory/{inventory_type.name}.yml" + items = session.scalars( + select(InventoryItem) + .where( + InventoryItem.type == inventory_type.name, + InventoryItem.deleted_at.is_(None), + ) + .order_by(InventoryItem.position, InventoryItem.item_id) + ).all() + data = [dict(item.data or {}) for item in items] + + schema = schemas.get(inventory_type.schema_id) if inventory_type.schema_id else None + if schema is not None: + validator = Draft202012Validator(schema.body) + for error in sorted(validator.iter_errors(data), key=lambda item: list(item.path)): + issues.append( + _issue( + path, + "json-schema-validation-error", + f"{_format_json_path(error.path)}: {error.message}", + ) + ) + + for item in items: + text = yaml.safe_dump(item.data or {}, sort_keys=False, allow_unicode=True) + _scan_text(path, text, issues) + + return issues + + +def _scan_text(path: str, text: str, issues: list[dict[str, str]]) -> None: + if PRIVATE_KEY_BLOCK_RE.search(text): + issues.append( + _issue(path, "possible-secret", "File contains a private key block marker") + ) + + for line_number, line in enumerate(text.splitlines(), start=1): + match = SECRET_ASSIGNMENT_RE.search(line) + if not match: + continue + value = match.group("value").strip().strip("\"'") + if not value or SAFE_SECRET_REFERENCE_RE.search(value): + continue + issues.append( + _issue( + path, + "possible-secret", + f"Possible raw secret assignment at line {line_number}", + ) + ) + + +def _format_json_path(path: Any) -> str: + parts = list(path) + if not parts: + return "$" + formatted = "$" + for part in parts: + if isinstance(part, int): + formatted += f"[{part}]" + else: + formatted += f".{part}" + return formatted