diff --git a/server/app/storage/cli.py b/server/app/storage/cli.py new file mode 100644 index 0000000..d5f5705 --- /dev/null +++ b/server/app/storage/cli.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import argparse +import json +from typing import Sequence + +from ..config import Settings +from .db import Database, DatabaseError +from .importer import database_counts, import_file_repository +from .parity import compare_repositories + + +def _database_settings(args: argparse.Namespace) -> Settings: + base = Settings() + database_url = args.database_url or base.database_url + if not database_url: + raise SystemExit( + "Set GNEXUS_BOOK_DATABASE_URL or pass --database-url " + "(for example postgresql+psycopg://gmikcon@127.0.0.1:5433/gnexus_book)" + ) + return Settings( + repo_root=base.repo_root, + database_url=database_url, + storage_backend="db", + ) + + +def _add_common(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--database-url", + default=None, + help="PostgreSQL URL; defaults to GNEXUS_BOOK_DATABASE_URL", + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m app.storage.cli", + description="Import and compare the file-backed knowledge base and PostgreSQL.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + import_parser = subparsers.add_parser("import", help="Import the file repository into PostgreSQL") + _add_common(import_parser) + import_parser.add_argument("--replace", action="store_true", help="Clear existing data first") + import_parser.add_argument("--dry-run", action="store_true", help="Count without writing") + + parity_parser = subparsers.add_parser("parity", help="Compare file and database read results") + _add_common(parity_parser) + parity_parser.add_argument( + "--query", + action="append", + default=None, + help="Search query to compare; repeatable", + ) + + counts_parser = subparsers.add_parser("counts", help="Print database row counts") + _add_common(counts_parser) + + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + settings = _database_settings(args) + database = Database.from_settings(settings) + + try: + if args.command == "import": + report = import_file_repository( + settings, + database, + replace=args.replace, + dry_run=args.dry_run, + ) + print(json.dumps(report.as_dict(), indent=2)) + return 0 + + if args.command == "counts": + print(json.dumps(database_counts(database), indent=2)) + return 0 + + if args.command == "parity": + result = compare_repositories(settings, database, queries=args.query) + print(json.dumps(result, indent=2, ensure_ascii=False)) + return 0 if result["ok"] else 1 + except (DatabaseError, ValueError) as exc: + print(f"error: {exc}") + return 2 + + parser.error(f"unknown command {args.command}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/app/storage/importer.py b/server/app/storage/importer.py new file mode 100644 index 0000000..6b13493 --- /dev/null +++ b/server/app/storage/importer.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass +from datetime import UTC, date, datetime +from pathlib import Path +from typing import Any + +from sqlalchemy import delete, func, select + +from ..config import Settings +from ..docs_repository import DocsRepository +from ..inventory import InventoryError, InventoryRepository +from ..validation import SCHEMA_BY_INVENTORY +from .db import Database, DatabaseError +from .models import ( + AuditLog, + Blob, + ChangeItem, + Changeset, + DocVersion, + Document, + InventoryItem, + InventoryItemVersion, + InventoryType, + JsonSchema, + ValidationRun, + utcnow, +) + + +@dataclass(frozen=True) +class ImportReport: + documents: int + doc_versions: int + inventory_types: int + inventory_items: int + schemas: int + dry_run: bool + + def as_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class _DocSource: + path: str + title: str + raw: str + body: str + frontmatter: dict[str, Any] + + +@dataclass(frozen=True) +class _InventorySource: + type_name: str + items: list[tuple[int, str, dict[str, Any]]] + + +def normalize_scalars(value: Any) -> Any: + if isinstance(value, (date, datetime)): + return value.isoformat() + if isinstance(value, list): + return [normalize_scalars(item) for item in value] + if isinstance(value, tuple): + return [normalize_scalars(item) for item in value] + if isinstance(value, dict): + return {key: normalize_scalars(item) for key, item in value.items()} + return value + + +def _collect_docs(settings: Settings) -> list[_DocSource]: + repo = DocsRepository(settings) + sources: list[_DocSource] = [] + for doc in repo.list_docs(): + path = str(doc["path"]) + record = repo.read_doc(path) + sources.append( + _DocSource( + path=path, + title=str(doc["title"]), + raw=str(record["raw"]), + body=str(record["body"]), + frontmatter=dict(record["frontmatter"]), + ) + ) + return sources + + +def _collect_inventory(settings: Settings) -> list[_InventorySource]: + repo = InventoryRepository(settings) + sources: list[_InventorySource] = [] + for type_name in repo.list_types(): + parsed = repo.read_parsed(type_name) + if not isinstance(parsed, list): + raise InventoryError(f"Inventory type {type_name} does not contain a list") + items: list[tuple[int, str, dict[str, Any]]] = [] + for position, item in enumerate(parsed): + if not isinstance(item, dict): + continue + item_id = item.get("id") + if not isinstance(item_id, str) or not item_id: + raise InventoryError( + f"Inventory type {type_name} has an item without a string id at index {position}" + ) + items.append((position, item_id, normalize_scalars(item))) + sources.append(_InventorySource(type_name=type_name, items=items)) + return sources + + +def _collect_schemas(settings: Settings) -> dict[str, dict[str, Any]]: + schemas: dict[str, dict[str, Any]] = {} + for path in sorted((settings.repo_root / "schemas").glob("*.json")): + schemas[path.name] = json.loads(path.read_text(encoding="utf-8")) + return schemas + + +def _content_hash(raw: str) -> str: + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def _clear_database(session) -> None: + for model in ( + AuditLog, + ValidationRun, + ChangeItem, + InventoryItemVersion, + DocVersion, + Changeset, + InventoryItem, + InventoryType, + JsonSchema, + Document, + Blob, + ): + session.execute(delete(model)) + + +def import_file_repository( + settings: Settings, + database: Database, + replace: bool = False, + dry_run: bool = False, +) -> ImportReport: + source_settings = Settings(repo_root=settings.repo_root, storage_backend="file") + docs = _collect_docs(source_settings) + inventory = _collect_inventory(source_settings) + schemas = _collect_schemas(source_settings) + + report = ImportReport( + documents=len(docs), + doc_versions=len(docs), + inventory_types=len(inventory), + inventory_items=sum(len(source.items) for source in inventory), + schemas=len(schemas), + dry_run=dry_run, + ) + if dry_run: + return report + + with database.session() as session: + with session.begin(): + existing = session.scalar(select(func.count()).select_from(Document)) or 0 + if existing and not replace: + raise DatabaseError( + "Database already contains documents; pass replace=True to re-import" + ) + if replace: + _clear_database(session) + + changeset = Changeset( + summary="Import knowledge base from file backend", + reason="Initial import of the Markdown and YAML knowledge base", + status="applied", + applied_at=utcnow(), + ) + session.add(changeset) + session.flush() + + schema_rows: dict[str, JsonSchema] = {} + for name, body in schemas.items(): + row = JsonSchema(name=name, version=1, body=body) + session.add(row) + schema_rows[name] = row + session.flush() + + seen_hashes: set[str] = set() + for source in docs: + document = Document(slug=source.path, title=source.title) + session.add(document) + session.flush() + + content_hash = _content_hash(source.raw) + if content_hash not in seen_hashes and session.get(Blob, content_hash) is None: + session.add( + Blob(hash=content_hash, content=source.raw, size=len(source.raw)) + ) + seen_hashes.add(content_hash) + + session.add( + DocVersion( + document_id=document.id, + version_no=1, + parent_version_id=None, + content_hash=content_hash, + raw=source.raw, + body=source.body, + frontmatter=source.frontmatter, + actor_kind="system", + actor_name="importer", + via="import", + changeset_id=changeset.id, + ) + ) + + for source in inventory: + schema_name = SCHEMA_BY_INVENTORY.get(source.type_name) + schema_row = schema_rows.get(schema_name) if schema_name else None + session.add( + InventoryType( + name=source.type_name, + schema_id=schema_row.id if schema_row else None, + ) + ) + session.flush() + + for source in inventory: + for position, item_id, data in source.items: + item = InventoryItem( + type=source.type_name, + item_id=item_id, + position=position, + data=data, + ) + session.add(item) + session.flush() + session.add( + InventoryItemVersion( + inventory_item_id=item.id, + version_no=1, + data=data, + changeset_id=changeset.id, + ) + ) + + session.add( + AuditLog( + actor_kind="system", + actor_name="importer", + via="import", + action="import", + entity_type="repository", + entity_id=str(changeset.id), + after=report.as_dict(), + ) + ) + + return report + + +def database_counts(database: Database) -> dict[str, int]: + with database.session() as session: + return { + "documents": session.scalar(select(func.count()).select_from(Document)) or 0, + "doc_versions": session.scalar(select(func.count()).select_from(DocVersion)) or 0, + "inventory_types": session.scalar(select(func.count()).select_from(InventoryType)) or 0, + "inventory_items": session.scalar(select(func.count()).select_from(InventoryItem)) or 0, + "schemas": session.scalar(select(func.count()).select_from(JsonSchema)) or 0, + "changesets": session.scalar(select(func.count()).select_from(Changeset)) or 0, + } diff --git a/server/app/storage/parity.py b/server/app/storage/parity.py new file mode 100644 index 0000000..b296385 --- /dev/null +++ b/server/app/storage/parity.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import json +from typing import Any + +from ..config import Settings +from ..docs_repository import DocsRepository +from ..inventory import InventoryRepository +from .db import Database +from .db_repositories import PostgresDocsRepository, PostgresInventoryRepository +from .importer import normalize_scalars + +DEFAULT_QUERIES = ("gnexus", "host", "GitBucket") + + +def _canonical(value: Any) -> str: + return json.dumps(normalize_scalars(value), sort_keys=True, ensure_ascii=False, default=str) + + +def _preview(value: Any, limit: int = 400) -> Any: + text = _canonical(value) + if len(text) <= limit: + return value + return text[:limit] + "..." + + +def _difference(kind: str, key: str, file_value: Any, db_value: Any) -> dict[str, Any]: + return { + "kind": kind, + "key": key, + "file": _preview(file_value), + "db": _preview(db_value), + } + + +def compare_repositories( + settings: Settings, + database: Database, + queries: list[str] | None = None, +) -> dict[str, Any]: + file_settings = Settings(repo_root=settings.repo_root, storage_backend="file") + file_docs = DocsRepository(file_settings) + db_docs = PostgresDocsRepository(database) + file_inventory = InventoryRepository(file_settings) + db_inventory = PostgresInventoryRepository(database) + + differences: list[dict[str, Any]] = [] + + file_doc_list = file_docs.list_docs() + db_doc_list = db_docs.list_docs() + if _canonical(file_doc_list) != _canonical(db_doc_list): + differences.append( + _difference("docs.list", "all", file_doc_list, db_doc_list) + ) + + file_paths = [str(doc["path"]) for doc in file_doc_list] + db_paths = [str(doc["path"]) for doc in db_doc_list] + if file_paths != db_paths: + differences.append( + _difference("docs.paths", "all", file_paths, db_paths) + ) + + for path in sorted(set(file_paths) & set(db_paths)): + file_doc = file_docs.read_doc(path) + db_doc = db_docs.read_doc(path) + for field in ("path", "frontmatter", "body", "raw"): + if _canonical(file_doc.get(field)) != _canonical(db_doc.get(field)): + differences.append( + _difference(f"docs.read.{field}", path, file_doc.get(field), db_doc.get(field)) + ) + + for query in queries or list(DEFAULT_QUERIES): + file_results = file_docs.search(query) + db_results = db_docs.search(query) + if _canonical(file_results) != _canonical(db_results): + differences.append( + _difference("docs.search", query, file_results, db_results) + ) + + file_types = file_inventory.list_types() + db_types = db_inventory.list_types() + if file_types != db_types: + differences.append( + _difference("inventory.types", "all", file_types, db_types) + ) + + for type_name in sorted(set(file_types) & set(db_types)): + file_parsed = normalize_scalars(file_inventory.read_parsed(type_name)) + db_parsed = normalize_scalars(db_inventory.read_parsed(type_name)) + if _canonical(file_parsed) != _canonical(db_parsed): + differences.append( + _difference("inventory.parsed", type_name, file_parsed, db_parsed) + ) + + for item in file_parsed if isinstance(file_parsed, list) else []: + if not isinstance(item, dict): + continue + item_id = item.get("id") + if not isinstance(item_id, str) or not item_id: + continue + file_item = file_inventory.read_item(type_name, item_id) + db_item = db_inventory.read_item(type_name, item_id) + if _canonical(file_item) != _canonical(db_item): + differences.append( + _difference( + "inventory.item", + f"{type_name}/{item_id}", + file_item, + db_item, + ) + ) + + return { + "ok": not differences, + "queries": list(queries or DEFAULT_QUERIES), + "difference_count": len(differences), + "differences": differences, + }