diff --git a/server/app/storage/cli.py b/server/app/storage/cli.py index d5f5705..fa1f337 100644 --- a/server/app/storage/cli.py +++ b/server/app/storage/cli.py @@ -2,11 +2,17 @@ import argparse import json +from pathlib import Path from typing import Sequence from ..config import Settings from .db import Database, DatabaseError -from .importer import database_counts, import_file_repository +from .importer import ( + database_counts, + database_manifest, + import_file_repository, + verify_import, +) from .parity import compare_repositories @@ -45,6 +51,26 @@ import_parser.add_argument("--replace", action="store_true", help="Clear existing data first") import_parser.add_argument("--dry-run", action="store_true", help="Count without writing") + migrate_parser = subparsers.add_parser( + "migrate", + help="Import the file repository, validate it and reconcile both backends", + ) + _add_common(migrate_parser) + migrate_parser.add_argument( + "--replace", action="store_true", help="Clear existing data first" + ) + migrate_parser.add_argument( + "--manifest", + default=None, + help="Write the import, verification and reconciliation report to this path", + ) + migrate_parser.add_argument( + "--query", + action="append", + default=None, + help="Search query to compare; repeatable", + ) + parity_parser = subparsers.add_parser("parity", help="Compare file and database read results") _add_common(parity_parser) parity_parser.add_argument( @@ -77,6 +103,39 @@ print(json.dumps(report.as_dict(), indent=2)) return 0 + if args.command == "migrate": + report = import_file_repository( + settings, + database, + replace=args.replace, + ) + verification = verify_import(settings, database) + reconciliation = compare_repositories( + settings, database, queries=args.query + ) + db_manifest, db_manifest_digest = database_manifest(database) + payload = { + "source_revision": verification.get("source_revision"), + "import": report.as_dict(), + "verification": verification, + "reconciliation": reconciliation, + "database_manifest_digest": db_manifest_digest, + } + if args.manifest: + manifest_path = Path(args.manifest) + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text( + json.dumps( + {**payload, "database_manifest": db_manifest}, + indent=2, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + print(json.dumps(payload, indent=2, ensure_ascii=False)) + return 0 if verification["ok"] and reconciliation["ok"] else 1 + if args.command == "counts": print(json.dumps(database_counts(database), indent=2)) return 0 diff --git a/server/app/storage/importer.py b/server/app/storage/importer.py index 6b13493..0b3ac68 100644 --- a/server/app/storage/importer.py +++ b/server/app/storage/importer.py @@ -2,6 +2,7 @@ import hashlib import json +import subprocess from dataclasses import asdict, dataclass from datetime import UTC, date, datetime from pathlib import Path @@ -14,6 +15,8 @@ from ..inventory import InventoryError, InventoryRepository from ..validation import SCHEMA_BY_INVENTORY from .db import Database, DatabaseError +from .db_repositories import latest_doc_rows +from .validation import validate_session from .models import ( AuditLog, Blob, @@ -38,6 +41,8 @@ inventory_items: int schemas: int dry_run: bool + source_revision: str | None = None + manifest_digest: str | None = None def as_dict(self) -> dict[str, Any]: return asdict(self) @@ -120,6 +125,53 @@ return hashlib.sha256(raw.encode("utf-8")).hexdigest() +def _canonical_json(value: Any) -> str: + return json.dumps( + normalize_scalars(value), + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + ) + + +def _source_revision(repo_root: Path) -> str | None: + try: + result = subprocess.run( + ["git", "-C", str(repo_root), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + except OSError: + return None + if result.returncode != 0: + return None + return result.stdout.strip() or None + + +def _manifest_from_sources( + docs: list[_DocSource], + inventory: list[_InventorySource], + schemas: dict[str, dict[str, Any]], +) -> tuple[dict[str, Any], str]: + manifest = { + "documents": {source.path: _content_hash(source.raw) for source in docs}, + "inventory": { + source.type_name: { + item_id: hashlib.sha256(_canonical_json(data).encode("utf-8")).hexdigest() + for _, item_id, data in source.items + } + for source in inventory + }, + "schemas": { + name: hashlib.sha256(_canonical_json(body).encode("utf-8")).hexdigest() + for name, body in schemas.items() + }, + } + digest = hashlib.sha256(_canonical_json(manifest).encode("utf-8")).hexdigest() + return manifest, digest + + def _clear_database(session) -> None: for model in ( AuditLog, @@ -147,6 +199,8 @@ docs = _collect_docs(source_settings) inventory = _collect_inventory(source_settings) schemas = _collect_schemas(source_settings) + _, manifest_digest = _manifest_from_sources(docs, inventory, schemas) + source_revision = _source_revision(settings.repo_root) report = ImportReport( documents=len(docs), @@ -155,6 +209,8 @@ inventory_items=sum(len(source.items) for source in inventory), schemas=len(schemas), dry_run=dry_run, + source_revision=source_revision, + manifest_digest=manifest_digest, ) if dry_run: return report @@ -169,9 +225,12 @@ if replace: _clear_database(session) + import_reason = "Initial import of the Markdown and YAML knowledge base" + if source_revision: + import_reason += f" from source revision {source_revision}" changeset = Changeset( summary="Import knowledge base from file backend", - reason="Initial import of the Markdown and YAML knowledge base", + reason=import_reason, status="applied", applied_at=utcnow(), ) @@ -244,6 +303,18 @@ ) ) + session.flush() + validation = validate_session(session) + if validation["status"] != "ok": + details = "\n".join( + f" - {issue['path']} ({issue['code']}): {issue['message']}" + for issue in validation.get("issues", [])[:10] + ) + raise ValueError( + "Imported repository failed validation:\n" + details + ) + changeset.validation = validation + session.add( AuditLog( actor_kind="system", @@ -259,6 +330,116 @@ return report +def database_manifest(database: Database) -> tuple[dict[str, Any], str]: + with database.session() as session: + documents = { + document.slug: version.content_hash + for document, version in session.execute(latest_doc_rows(session)).all() + } + inventory: dict[str, dict[str, str]] = {} + for item in session.scalars( + select(InventoryItem).where(InventoryItem.deleted_at.is_(None)) + ).all(): + inventory.setdefault(item.type, {})[item.item_id] = hashlib.sha256( + _canonical_json(item.data or {}).encode("utf-8") + ).hexdigest() + schemas: dict[str, str] = {} + for schema in session.scalars( + select(JsonSchema).order_by(JsonSchema.name, JsonSchema.version.desc()) + ).all(): + schemas.setdefault( + schema.name, + hashlib.sha256(_canonical_json(schema.body).encode("utf-8")).hexdigest(), + ) + manifest = { + "documents": documents, + "inventory": inventory, + "schemas": schemas, + } + digest = hashlib.sha256(_canonical_json(manifest).encode("utf-8")).hexdigest() + return manifest, digest + + +def verify_import(settings: Settings, database: Database) -> dict[str, Any]: + source_settings = Settings(repo_root=settings.repo_root, storage_backend="file") + docs = _collect_docs(source_settings) + inventory = _collect_inventory(source_settings) + schemas = _collect_schemas(source_settings) + file_manifest, file_digest = _manifest_from_sources(docs, inventory, schemas) + database_manifest_data, database_digest = database_manifest(database) + + differences: list[dict[str, Any]] = [] + for section in ("documents", "schemas"): + file_entries = file_manifest[section] + database_entries = database_manifest_data[section] + for key in sorted(set(file_entries) - set(database_entries)): + differences.append( + {"section": section, "key": key, "issue": "missing_in_database"} + ) + for key in sorted(set(database_entries) - set(file_entries)): + differences.append( + {"section": section, "key": key, "issue": "missing_in_files"} + ) + for key in sorted(set(file_entries) & set(database_entries)): + if file_entries[key] != database_entries[key]: + differences.append( + {"section": section, "key": key, "issue": "hash_mismatch"} + ) + + file_inventory = file_manifest["inventory"] + database_inventory = database_manifest_data["inventory"] + for type_name in sorted(set(file_inventory) | set(database_inventory)): + file_items = file_inventory.get(type_name, {}) + database_items = database_inventory.get(type_name, {}) + for item_id in sorted(set(file_items) - set(database_items)): + differences.append( + { + "section": "inventory", + "key": f"{type_name}/{item_id}", + "issue": "missing_in_database", + } + ) + for item_id in sorted(set(database_items) - set(file_items)): + differences.append( + { + "section": "inventory", + "key": f"{type_name}/{item_id}", + "issue": "missing_in_files", + } + ) + for item_id in sorted(set(file_items) & set(database_items)): + if file_items[item_id] != database_items[item_id]: + differences.append( + { + "section": "inventory", + "key": f"{type_name}/{item_id}", + "issue": "hash_mismatch", + } + ) + + db_counts = database_counts(database) + file_counts = { + "documents": len(docs), + "inventory_types": len(inventory), + "inventory_items": sum(len(source.items) for source in inventory), + "schemas": len(schemas), + } + + return { + "ok": not differences, + "source_revision": source_revision(settings.repo_root), + "file_manifest_digest": file_digest, + "database_manifest_digest": database_digest, + "file_counts": file_counts, + "database_counts": db_counts, + "differences": differences, + } + + +def source_revision(repo_root: Path) -> str | None: + return _source_revision(repo_root) + + def database_counts(database: Database) -> dict[str, int]: with database.session() as session: return { diff --git a/server/app/storage/parity.py b/server/app/storage/parity.py index b296385..bb77b29 100644 --- a/server/app/storage/parity.py +++ b/server/app/storage/parity.py @@ -5,10 +5,15 @@ from ..config import Settings from ..docs_repository import DocsRepository +from ..freshness import build_freshness_report from ..inventory import InventoryRepository +from ..relationships import build_relationships +from ..validation import validate_repository from .db import Database from .db_repositories import PostgresDocsRepository, PostgresInventoryRepository +from .freshness import build_freshness_report_db from .importer import normalize_scalars +from .validation import validate_database DEFAULT_QUERIES = ("gnexus", "host", "GitBucket") @@ -33,6 +38,23 @@ } +def _normalized_report(report: dict[str, Any]) -> dict[str, Any]: + return { + "status": report.get("status"), + "issue_count": report.get("issue_count"), + "document_count": report.get("document_count"), + "issues": sorted( + ( + issue.get("path"), + issue.get("severity"), + issue.get("code"), + issue.get("message"), + ) + for issue in report.get("issues", []) + ), + } + + def compare_repositories( settings: Settings, database: Database, @@ -110,9 +132,40 @@ ) ) + file_validation = _normalized_report(validate_repository(file_settings)) + db_validation = _normalized_report(validate_database(database)) + if _canonical(file_validation) != _canonical(db_validation): + differences.append( + _difference("validation", "all", file_validation, db_validation) + ) + + file_freshness = _normalized_report(build_freshness_report(file_settings)) + db_freshness = _normalized_report(build_freshness_report_db(database)) + if _canonical(file_freshness) != _canonical(db_freshness): + differences.append( + _difference("freshness", "all", file_freshness, db_freshness) + ) + + file_relationships = build_relationships(file_inventory) + db_relationships = build_relationships(db_inventory) + if _canonical(file_relationships) != _canonical(db_relationships): + differences.append( + _difference( + "relationships", "all", file_relationships, db_relationships + ) + ) + return { "ok": not differences, "queries": list(queries or DEFAULT_QUERIES), "difference_count": len(differences), + "validation": { + "file": file_validation["status"], + "database": db_validation["status"], + }, + "freshness": { + "file": file_freshness["status"], + "database": db_freshness["status"], + }, "differences": differences, }