from __future__ import annotations
import hashlib
import json
from dataclasses import asdict, dataclass
from datetime import UTC, date, datetime
from pathlib import Path
from typing import Any
from sqlalchemy import delete, func, select
from ..config import Settings
from ..docs_repository import DocsRepository
from ..inventory import InventoryError, InventoryRepository
from ..validation import SCHEMA_BY_INVENTORY
from .db import Database, DatabaseError
from .models import (
AuditLog,
Blob,
ChangeItem,
Changeset,
DocVersion,
Document,
InventoryItem,
InventoryItemVersion,
InventoryType,
JsonSchema,
ValidationRun,
utcnow,
)
@dataclass(frozen=True)
class ImportReport:
documents: int
doc_versions: int
inventory_types: int
inventory_items: int
schemas: int
dry_run: bool
def as_dict(self) -> dict[str, Any]:
return asdict(self)
@dataclass(frozen=True)
class _DocSource:
path: str
title: str
raw: str
body: str
frontmatter: dict[str, Any]
@dataclass(frozen=True)
class _InventorySource:
type_name: str
items: list[tuple[int, str, dict[str, Any]]]
def normalize_scalars(value: Any) -> Any:
if isinstance(value, (date, datetime)):
return value.isoformat()
if isinstance(value, list):
return [normalize_scalars(item) for item in value]
if isinstance(value, tuple):
return [normalize_scalars(item) for item in value]
if isinstance(value, dict):
return {key: normalize_scalars(item) for key, item in value.items()}
return value
def _collect_docs(settings: Settings) -> list[_DocSource]:
repo = DocsRepository(settings)
sources: list[_DocSource] = []
for doc in repo.list_docs():
path = str(doc["path"])
record = repo.read_doc(path)
sources.append(
_DocSource(
path=path,
title=str(doc["title"]),
raw=str(record["raw"]),
body=str(record["body"]),
frontmatter=dict(record["frontmatter"]),
)
)
return sources
def _collect_inventory(settings: Settings) -> list[_InventorySource]:
repo = InventoryRepository(settings)
sources: list[_InventorySource] = []
for type_name in repo.list_types():
parsed = repo.read_parsed(type_name)
if not isinstance(parsed, list):
raise InventoryError(f"Inventory type {type_name} does not contain a list")
items: list[tuple[int, str, dict[str, Any]]] = []
for position, item in enumerate(parsed):
if not isinstance(item, dict):
continue
item_id = item.get("id")
if not isinstance(item_id, str) or not item_id:
raise InventoryError(
f"Inventory type {type_name} has an item without a string id at index {position}"
)
items.append((position, item_id, normalize_scalars(item)))
sources.append(_InventorySource(type_name=type_name, items=items))
return sources
def _collect_schemas(settings: Settings) -> dict[str, dict[str, Any]]:
schemas: dict[str, dict[str, Any]] = {}
for path in sorted((settings.repo_root / "schemas").glob("*.json")):
schemas[path.name] = json.loads(path.read_text(encoding="utf-8"))
return schemas
def _content_hash(raw: str) -> str:
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
def _clear_database(session) -> None:
for model in (
AuditLog,
ValidationRun,
ChangeItem,
InventoryItemVersion,
DocVersion,
Changeset,
InventoryItem,
InventoryType,
JsonSchema,
Document,
Blob,
):
session.execute(delete(model))
def import_file_repository(
settings: Settings,
database: Database,
replace: bool = False,
dry_run: bool = False,
) -> ImportReport:
source_settings = Settings(repo_root=settings.repo_root, storage_backend="file")
docs = _collect_docs(source_settings)
inventory = _collect_inventory(source_settings)
schemas = _collect_schemas(source_settings)
report = ImportReport(
documents=len(docs),
doc_versions=len(docs),
inventory_types=len(inventory),
inventory_items=sum(len(source.items) for source in inventory),
schemas=len(schemas),
dry_run=dry_run,
)
if dry_run:
return report
with database.session() as session:
with session.begin():
existing = session.scalar(select(func.count()).select_from(Document)) or 0
if existing and not replace:
raise DatabaseError(
"Database already contains documents; pass replace=True to re-import"
)
if replace:
_clear_database(session)
changeset = Changeset(
summary="Import knowledge base from file backend",
reason="Initial import of the Markdown and YAML knowledge base",
status="applied",
applied_at=utcnow(),
)
session.add(changeset)
session.flush()
schema_rows: dict[str, JsonSchema] = {}
for name, body in schemas.items():
row = JsonSchema(name=name, version=1, body=body)
session.add(row)
schema_rows[name] = row
session.flush()
seen_hashes: set[str] = set()
for source in docs:
document = Document(slug=source.path, title=source.title)
session.add(document)
session.flush()
content_hash = _content_hash(source.raw)
if content_hash not in seen_hashes and session.get(Blob, content_hash) is None:
session.add(
Blob(hash=content_hash, content=source.raw, size=len(source.raw))
)
seen_hashes.add(content_hash)
session.add(
DocVersion(
document_id=document.id,
version_no=1,
parent_version_id=None,
content_hash=content_hash,
raw=source.raw,
body=source.body,
frontmatter=source.frontmatter,
actor_kind="system",
actor_name="importer",
via="import",
changeset_id=changeset.id,
)
)
for source in inventory:
schema_name = SCHEMA_BY_INVENTORY.get(source.type_name)
schema_row = schema_rows.get(schema_name) if schema_name else None
session.add(
InventoryType(
name=source.type_name,
schema_id=schema_row.id if schema_row else None,
)
)
session.flush()
for source in inventory:
for position, item_id, data in source.items:
item = InventoryItem(
type=source.type_name,
item_id=item_id,
position=position,
data=data,
)
session.add(item)
session.flush()
session.add(
InventoryItemVersion(
inventory_item_id=item.id,
version_no=1,
data=data,
changeset_id=changeset.id,
)
)
session.add(
AuditLog(
actor_kind="system",
actor_name="importer",
via="import",
action="import",
entity_type="repository",
entity_id=str(changeset.id),
after=report.as_dict(),
)
)
return report
def database_counts(database: Database) -> dict[str, int]:
with database.session() as session:
return {
"documents": session.scalar(select(func.count()).select_from(Document)) or 0,
"doc_versions": session.scalar(select(func.count()).select_from(DocVersion)) or 0,
"inventory_types": session.scalar(select(func.count()).select_from(InventoryType)) or 0,
"inventory_items": session.scalar(select(func.count()).select_from(InventoryItem)) or 0,
"schemas": session.scalar(select(func.count()).select_from(JsonSchema)) or 0,
"changesets": session.scalar(select(func.count()).select_from(Changeset)) or 0,
}