from __future__ import annotations
import hashlib
import json
import subprocess
from dataclasses import asdict, dataclass
from datetime import UTC, date, datetime
from pathlib import Path
from typing import Any
from sqlalchemy import delete, func, select
from ..config import Settings
from ..docs_repository import DocsRepository
from ..inventory import InventoryError, InventoryRepository
from ..validation import SCHEMA_BY_INVENTORY
from .db import Database, DatabaseError
from .db_repositories import latest_doc_rows
from .validation import validate_session
from .models import (
AuditLog,
Blob,
ChangeItem,
Changeset,
DocVersion,
Document,
InventoryItem,
InventoryItemVersion,
InventoryType,
JsonSchema,
ValidationRun,
utcnow,
)
@dataclass(frozen=True)
class ImportReport:
documents: int
doc_versions: int
inventory_types: int
inventory_items: int
schemas: int
dry_run: bool
source_revision: str | None = None
manifest_digest: str | None = None
def as_dict(self) -> dict[str, Any]:
return asdict(self)
@dataclass(frozen=True)
class _DocSource:
path: str
title: str
raw: str
body: str
frontmatter: dict[str, Any]
@dataclass(frozen=True)
class _InventorySource:
type_name: str
items: list[tuple[int, str, dict[str, Any]]]
def normalize_scalars(value: Any) -> Any:
if isinstance(value, (date, datetime)):
return value.isoformat()
if isinstance(value, list):
return [normalize_scalars(item) for item in value]
if isinstance(value, tuple):
return [normalize_scalars(item) for item in value]
if isinstance(value, dict):
return {key: normalize_scalars(item) for key, item in value.items()}
return value
def _collect_docs(settings: Settings) -> list[_DocSource]:
repo = DocsRepository(settings)
sources: list[_DocSource] = []
for doc in repo.list_docs():
path = str(doc["path"])
record = repo.read_doc(path)
sources.append(
_DocSource(
path=path,
title=str(doc["title"]),
raw=str(record["raw"]),
body=str(record["body"]),
frontmatter=dict(record["frontmatter"]),
)
)
return sources
def _collect_inventory(settings: Settings) -> list[_InventorySource]:
repo = InventoryRepository(settings)
sources: list[_InventorySource] = []
for type_name in repo.list_types():
parsed = repo.read_parsed(type_name)
if not isinstance(parsed, list):
raise InventoryError(f"Inventory type {type_name} does not contain a list")
items: list[tuple[int, str, dict[str, Any]]] = []
for position, item in enumerate(parsed):
if not isinstance(item, dict):
continue
item_id = item.get("id")
if not isinstance(item_id, str) or not item_id:
raise InventoryError(
f"Inventory type {type_name} has an item without a string id at index {position}"
)
items.append((position, item_id, normalize_scalars(item)))
sources.append(_InventorySource(type_name=type_name, items=items))
return sources
def _collect_schemas(settings: Settings) -> dict[str, dict[str, Any]]:
schemas: dict[str, dict[str, Any]] = {}
for path in sorted((settings.repo_root / "schemas").glob("*.json")):
schemas[path.name] = json.loads(path.read_text(encoding="utf-8"))
return schemas
def _content_hash(raw: str) -> str:
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
def _canonical_json(value: Any) -> str:
return json.dumps(
normalize_scalars(value),
sort_keys=True,
ensure_ascii=False,
separators=(",", ":"),
)
def _source_revision(repo_root: Path) -> str | None:
try:
result = subprocess.run(
["git", "-C", str(repo_root), "rev-parse", "HEAD"],
check=False,
capture_output=True,
text=True,
)
except OSError:
return None
if result.returncode != 0:
return None
return result.stdout.strip() or None
def _manifest_from_sources(
docs: list[_DocSource],
inventory: list[_InventorySource],
schemas: dict[str, dict[str, Any]],
) -> tuple[dict[str, Any], str]:
manifest = {
"documents": {source.path: _content_hash(source.raw) for source in docs},
"inventory": {
source.type_name: {
item_id: hashlib.sha256(_canonical_json(data).encode("utf-8")).hexdigest()
for _, item_id, data in source.items
}
for source in inventory
},
"schemas": {
name: hashlib.sha256(_canonical_json(body).encode("utf-8")).hexdigest()
for name, body in schemas.items()
},
}
digest = hashlib.sha256(_canonical_json(manifest).encode("utf-8")).hexdigest()
return manifest, digest
def _clear_database(session) -> None:
for model in (
AuditLog,
ValidationRun,
ChangeItem,
InventoryItemVersion,
DocVersion,
Changeset,
InventoryItem,
InventoryType,
JsonSchema,
Document,
Blob,
):
session.execute(delete(model))
def import_file_repository(
settings: Settings,
database: Database,
replace: bool = False,
dry_run: bool = False,
) -> ImportReport:
source_settings = Settings(repo_root=settings.repo_root, storage_backend="file")
docs = _collect_docs(source_settings)
inventory = _collect_inventory(source_settings)
schemas = _collect_schemas(source_settings)
_, manifest_digest = _manifest_from_sources(docs, inventory, schemas)
source_revision = _source_revision(settings.repo_root)
report = ImportReport(
documents=len(docs),
doc_versions=len(docs),
inventory_types=len(inventory),
inventory_items=sum(len(source.items) for source in inventory),
schemas=len(schemas),
dry_run=dry_run,
source_revision=source_revision,
manifest_digest=manifest_digest,
)
if dry_run:
return report
with database.session() as session:
with session.begin():
existing = session.scalar(select(func.count()).select_from(Document)) or 0
if existing and not replace:
raise DatabaseError(
"Database already contains documents; pass replace=True to re-import"
)
if replace:
_clear_database(session)
import_reason = "Initial import of the Markdown and YAML knowledge base"
if source_revision:
import_reason += f" from source revision {source_revision}"
changeset = Changeset(
summary="Import knowledge base from file backend",
reason=import_reason,
status="applied",
applied_at=utcnow(),
)
session.add(changeset)
session.flush()
schema_rows: dict[str, JsonSchema] = {}
for name, body in schemas.items():
row = JsonSchema(name=name, version=1, body=body)
session.add(row)
schema_rows[name] = row
session.flush()
seen_hashes: set[str] = set()
for source in docs:
document = Document(slug=source.path, title=source.title)
session.add(document)
session.flush()
content_hash = _content_hash(source.raw)
if content_hash not in seen_hashes and session.get(Blob, content_hash) is None:
session.add(
Blob(hash=content_hash, content=source.raw, size=len(source.raw))
)
seen_hashes.add(content_hash)
session.add(
DocVersion(
document_id=document.id,
version_no=1,
parent_version_id=None,
content_hash=content_hash,
raw=source.raw,
body=source.body,
frontmatter=source.frontmatter,
actor_kind="system",
actor_name="importer",
via="import",
changeset_id=changeset.id,
)
)
for source in inventory:
schema_name = SCHEMA_BY_INVENTORY.get(source.type_name)
schema_row = schema_rows.get(schema_name) if schema_name else None
session.add(
InventoryType(
name=source.type_name,
schema_id=schema_row.id if schema_row else None,
)
)
session.flush()
for source in inventory:
for position, item_id, data in source.items:
item = InventoryItem(
type=source.type_name,
item_id=item_id,
position=position,
data=data,
)
session.add(item)
session.flush()
session.add(
InventoryItemVersion(
inventory_item_id=item.id,
version_no=1,
data=data,
changeset_id=changeset.id,
)
)
session.flush()
validation = validate_session(session)
if validation["status"] != "ok":
details = "\n".join(
f" - {issue['path']} ({issue['code']}): {issue['message']}"
for issue in validation.get("issues", [])[:10]
)
raise ValueError(
"Imported repository failed validation:\n" + details
)
changeset.validation = validation
session.add(
AuditLog(
actor_kind="system",
actor_name="importer",
via="import",
action="import",
entity_type="repository",
entity_id=str(changeset.id),
after=report.as_dict(),
)
)
return report
def database_manifest(database: Database) -> tuple[dict[str, Any], str]:
with database.session() as session:
documents = {
document.slug: version.content_hash
for document, version in session.execute(latest_doc_rows(session)).all()
}
inventory: dict[str, dict[str, str]] = {}
for item in session.scalars(
select(InventoryItem).where(InventoryItem.deleted_at.is_(None))
).all():
inventory.setdefault(item.type, {})[item.item_id] = hashlib.sha256(
_canonical_json(item.data or {}).encode("utf-8")
).hexdigest()
schemas: dict[str, str] = {}
for schema in session.scalars(
select(JsonSchema).order_by(JsonSchema.name, JsonSchema.version.desc())
).all():
schemas.setdefault(
schema.name,
hashlib.sha256(_canonical_json(schema.body).encode("utf-8")).hexdigest(),
)
manifest = {
"documents": documents,
"inventory": inventory,
"schemas": schemas,
}
digest = hashlib.sha256(_canonical_json(manifest).encode("utf-8")).hexdigest()
return manifest, digest
def verify_import(settings: Settings, database: Database) -> dict[str, Any]:
source_settings = Settings(repo_root=settings.repo_root, storage_backend="file")
docs = _collect_docs(source_settings)
inventory = _collect_inventory(source_settings)
schemas = _collect_schemas(source_settings)
file_manifest, file_digest = _manifest_from_sources(docs, inventory, schemas)
database_manifest_data, database_digest = database_manifest(database)
differences: list[dict[str, Any]] = []
for section in ("documents", "schemas"):
file_entries = file_manifest[section]
database_entries = database_manifest_data[section]
for key in sorted(set(file_entries) - set(database_entries)):
differences.append(
{"section": section, "key": key, "issue": "missing_in_database"}
)
for key in sorted(set(database_entries) - set(file_entries)):
differences.append(
{"section": section, "key": key, "issue": "missing_in_files"}
)
for key in sorted(set(file_entries) & set(database_entries)):
if file_entries[key] != database_entries[key]:
differences.append(
{"section": section, "key": key, "issue": "hash_mismatch"}
)
file_inventory = file_manifest["inventory"]
database_inventory = database_manifest_data["inventory"]
for type_name in sorted(set(file_inventory) | set(database_inventory)):
file_items = file_inventory.get(type_name, {})
database_items = database_inventory.get(type_name, {})
for item_id in sorted(set(file_items) - set(database_items)):
differences.append(
{
"section": "inventory",
"key": f"{type_name}/{item_id}",
"issue": "missing_in_database",
}
)
for item_id in sorted(set(database_items) - set(file_items)):
differences.append(
{
"section": "inventory",
"key": f"{type_name}/{item_id}",
"issue": "missing_in_files",
}
)
for item_id in sorted(set(file_items) & set(database_items)):
if file_items[item_id] != database_items[item_id]:
differences.append(
{
"section": "inventory",
"key": f"{type_name}/{item_id}",
"issue": "hash_mismatch",
}
)
db_counts = database_counts(database)
file_counts = {
"documents": len(docs),
"inventory_types": len(inventory),
"inventory_items": sum(len(source.items) for source in inventory),
"schemas": len(schemas),
}
return {
"ok": not differences,
"source_revision": source_revision(settings.repo_root),
"file_manifest_digest": file_digest,
"database_manifest_digest": database_digest,
"file_counts": file_counts,
"database_counts": db_counts,
"differences": differences,
}
def source_revision(repo_root: Path) -> str | None:
return _source_revision(repo_root)
def database_counts(database: Database) -> dict[str, int]:
with database.session() as session:
return {
"documents": session.scalar(select(func.count()).select_from(Document)) or 0,
"doc_versions": session.scalar(select(func.count()).select_from(DocVersion)) or 0,
"inventory_types": session.scalar(select(func.count()).select_from(InventoryType)) or 0,
"inventory_items": session.scalar(select(func.count()).select_from(InventoryItem)) or 0,
"schemas": session.scalar(select(func.count()).select_from(JsonSchema)) or 0,
"changesets": session.scalar(select(func.count()).select_from(Changeset)) or 0,
}