diff --git a/server/app/main.py b/server/app/main.py index 7b4afe5..c27a293 100644 --- a/server/app/main.py +++ b/server/app/main.py @@ -145,7 +145,10 @@ request: ProposedChangeRequest, repo: PendingChangeRepository = Depends(get_pending_change_repo), ) -> dict[str, object]: - return repo.create(request) + try: + return repo.create(request) + except PendingChangeError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc @app.post("/changes/{change_id}/apply") @@ -159,6 +162,19 @@ raise HTTPException(status_code=400, detail=str(exc)) from exc +@app.delete("/changes/{change_id}") +def delete_change( + change_id: str, + repo: PendingChangeRepository = Depends(get_pending_change_repo), +) -> dict[str, object]: + try: + return repo.delete(change_id) + except PendingChangeError as exc: + message = str(exc) + status_code = 404 if "not found" in message.lower() else 409 + raise HTTPException(status_code=status_code, detail=message) from exc + + @app.get("/git/status") def git_status(git: GitAdapter = Depends(get_git_adapter)) -> dict[str, object]: try: diff --git a/server/app/mcp_server.py b/server/app/mcp_server.py index f0d227f..bb8c771 100644 --- a/server/app/mcp_server.py +++ b/server/app/mcp_server.py @@ -211,10 +211,15 @@ def list_pending_changes() -> list[dict[str, Any]]: - """List pending/applied/rejected knowledge-change records created by agents.""" + """List pending/applied/rejected/corrupt knowledge-change records created by agents.""" return _pending().list() +def delete_pending_change(change_id: str) -> dict[str, Any]: + """Delete one pending or corrupt knowledge-change record. Applied changes cannot be deleted.""" + return _pending().delete(change_id) + + def propose_doc_change( path: Annotated[str, Field(description="Repository-relative Markdown path to create or replace, for example '10-systems/applications/gitbucket.md'.")], content: Annotated[str, Field(description="Full Markdown document content, including frontmatter when the target document uses it.")], @@ -264,9 +269,21 @@ def update_doc(path: str, content: str, summary: str, reason: str = "") -> dict[str, Any]: """For simple full-document updates: create pending change, apply, validate, and commit in one call.""" change = propose_doc_change(path=path, content=content, summary=summary, reason=reason) - applied = apply_pending_change(str(change["id"])) - pending_path = f"90-maintenance/pending-changes/{change['id']}.json" - commit = commit_changes(summary=summary, files=[path, pending_path], details=reason) + change_id = str(change["id"]) + pending_path = f"90-maintenance/pending-changes/{change_id}.json" + try: + applied = apply_pending_change(change_id) + commit = commit_changes(summary=summary, files=[path, pending_path], details=reason) + except Exception as exc: + # Full rollback: never leave the pending JSON behind after a failed update_doc. + try: + delete_pending_change(change_id) + except Exception: + pass + raise RuntimeError( + f"update_doc failed; pending change {change_id} was cleaned up (deleted if still pending). " + f"Original error: {type(exc).__name__}: {exc}" + ) from exc return {"change": applied, "commit": commit} @@ -290,6 +307,7 @@ mcp.tool(name="git_status")(git_status) mcp.tool(name="git_diff")(git_diff) mcp.tool(name="list_pending_changes")(list_pending_changes) +mcp.tool(name="delete_pending_change")(delete_pending_change) mcp.tool(name="propose_doc_change")(propose_doc_change) mcp.tool(name="propose_inventory_item_change")(propose_inventory_item_change) mcp.tool(name="apply_pending_change")(apply_pending_change) diff --git a/server/app/pending_changes.py b/server/app/pending_changes.py index 8150398..0f48aa3 100644 --- a/server/app/pending_changes.py +++ b/server/app/pending_changes.py @@ -10,11 +10,11 @@ from pydantic import BaseModel, Field from .config import Settings -from .validation import validate_repository +from .validation import scan_text_for_secrets, validate_repository ChangeKind = Literal["doc", "inventory", "inventory-item"] -ChangeStatus = Literal["pending", "applied", "rejected"] +ChangeStatus = Literal["pending", "applied", "rejected", "corrupt"] class ProposedChangeRequest(BaseModel): @@ -48,14 +48,38 @@ self.directory.mkdir(parents=True, exist_ok=True) def list(self) -> list[dict[str, Any]]: - changes = [self._read_file(path) for path in sorted(self.directory.glob("*.json"))] - return sorted(changes, key=lambda item: item["created_at"], reverse=True) + changes: list[dict[str, Any]] = [] + for path in sorted(self.directory.glob("*.json")): + record = self._read_file(path) + if record is None: + changes.append( + { + "id": path.stem, + "kind": "unknown", + "target": "", + "summary": "Corrupt pending-change file (unreadable JSON)", + "reason": "", + "payload": {}, + "status": "corrupt", + "file": path.name, + "created_at": "", + "updated_at": "", + } + ) + else: + changes.append(record) + return sorted(changes, key=lambda item: item.get("created_at", ""), reverse=True) def get(self, change_id: str) -> dict[str, Any]: path = self._path_for_id(change_id) if not path.exists(): raise PendingChangeError("Pending change not found") - return self._read_file(path) + record = self._read_file(path) + if record is None: + raise PendingChangeError( + f"Pending change {change_id} is corrupt (unreadable JSON); it can be deleted" + ) + return record def create(self, request: ProposedChangeRequest) -> dict[str, Any]: now = datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") @@ -70,6 +94,7 @@ created_at=now, updated_at=now, ) + self._validate_before_write(change) path = self._path_for_id(change.id) path.write_text( json.dumps(change.model_dump(), indent=2, ensure_ascii=False) + "\n", @@ -79,6 +104,10 @@ def apply(self, change_id: str) -> dict[str, Any]: change = self.get(change_id) + if change["status"] == "corrupt": + raise PendingChangeError( + f"Pending change {change_id} is corrupt and cannot be applied; delete it instead" + ) if change["status"] != "pending": raise PendingChangeError("Only pending changes can be applied") @@ -88,6 +117,62 @@ return self._apply_inventory_item_change(change) raise PendingChangeError("Only doc and inventory-item changes can be applied in the current MVP") + def delete(self, change_id: str) -> dict[str, Any]: + path = self._path_for_id(change_id) + if not path.exists(): + raise PendingChangeError("Pending change not found") + record = self._read_file(path) + if record is not None and not isinstance(record, dict): + record = None + if record is not None and record.get("status") not in (None, "pending", "corrupt"): + raise PendingChangeError( + f"Only pending or corrupt changes can be deleted; change {change_id} has status {record.get('status')!r}" + ) + path.unlink() + return {"id": change_id, "status": "deleted"} + + def _validate_before_write(self, change: PendingChange) -> None: + """Reject dangerous payloads BEFORE anything is written to disk. + + This prevents poisoned pending-change files: a rejected request must + leave no trace in 90-maintenance/pending-changes/. + """ + if change.kind == "doc": + self._resolve_doc_target(change.target) + elif change.kind == "inventory-item": + # Format check only: inventory type existence is validated at apply time. + self._parse_inventory_item_target(change.target) + + secret_messages: list[str] = [] + # Scan raw string values first: JSON escaping glues words together + # (newline becomes literal \n), which breaks \b word-boundary matching. + for text in [change.summary, change.reason, *self._iter_strings(change.payload)]: + secret_messages.extend(scan_text_for_secrets(text)) + serialized = json.dumps(change.model_dump(), indent=2, ensure_ascii=False) + secret_messages.extend(scan_text_for_secrets(serialized)) + if secret_messages: + raise PendingChangeError( + "Content rejected before writing pending file: possible secret detected " + f"({len(secret_messages)} pattern match(es)). " + "Do not store raw secrets in the knowledge base; reference the secret storage instead." + ) + + @staticmethod + def _iter_strings(value: Any) -> list[str]: + if isinstance(value, str): + return [value] + if isinstance(value, dict): + strings: list[str] = [] + for item in value.values(): + strings.extend(PendingChangeRepository._iter_strings(item)) + return strings + if isinstance(value, list): + strings = [] + for item in value: + strings.extend(PendingChangeRepository._iter_strings(item)) + return strings + return [] + def _apply_doc_change(self, change: dict[str, Any]) -> dict[str, Any]: target = self._resolve_doc_target(change["target"]) content = self._extract_doc_content(change) @@ -102,6 +187,7 @@ target.unlink(missing_ok=True) else: target.write_text(previous_content, encoding="utf-8") + self._discard_pending_file(str(change["id"])) issues = validation.get("issues", []) details = "\n".join(f" - {i['path']} ({i['code']}): {i['message']}" for i in issues[:10]) raise PendingChangeError(f"Applied change failed validation and was rolled back. Issues:\n{details}") @@ -145,6 +231,7 @@ validation = validate_repository(self.settings) if validation["status"] != "ok": target.write_text(previous_content, encoding="utf-8") + self._discard_pending_file(str(change["id"])) issues = validation.get("issues", []) details = "\n".join(f" - {i['path']} ({i['code']}): {i['message']}" for i in issues[:10]) raise PendingChangeError(f"Applied change failed validation and was rolled back. Issues:\n{details}") @@ -157,6 +244,13 @@ self._write_change(change) return change + def _discard_pending_file(self, change_id: str) -> None: + """Remove this change's own pending JSON so a failed apply leaves no residue.""" + try: + self._path_for_id(change_id).unlink(missing_ok=True) + except OSError: + pass + def _new_id(self) -> str: return datetime.now(UTC).strftime("%Y%m%d%H%M%S") + "-" + uuid4().hex[:8] @@ -229,9 +323,12 @@ return mode @staticmethod - def _read_file(path: Path) -> dict[str, Any]: - return json.loads(path.read_text(encoding="utf-8")) + def _read_file(path: Path) -> dict[str, Any] | None: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return None def _write_change(self, change: dict[str, Any]) -> None: path = self._path_for_id(str(change["id"])) - path.write_text(json.dumps(change, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + path.write_text(json.dumps(change, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") \ No newline at end of file diff --git a/server/app/validation.py b/server/app/validation.py index 2c82985..d40ec80 100644 --- a/server/app/validation.py +++ b/server/app/validation.py @@ -82,6 +82,28 @@ } +def scan_text_for_secrets(text: str) -> list[str]: + """Scan arbitrary text for raw secret patterns. + + Returns human-readable issue messages (without echoing matched values). + Used both by the repository-wide secret scan and by pending-change + pre-write validation. + """ + messages: list[str] = [] + if PRIVATE_KEY_BLOCK_RE.search(text): + messages.append("File contains a private key block marker") + + for line_number, line in enumerate(text.splitlines(), start=1): + match = SECRET_ASSIGNMENT_RE.search(line) + if not match: + continue + value = match.group("value").strip().strip("\"'") + if not value or SAFE_SECRET_REFERENCE_RE.search(value): + continue + messages.append(f"Possible raw secret assignment at line {line_number}") + return messages + + def _validate_schema_json(schema_dir: Path) -> list[ValidationIssue]: issues: list[ValidationIssue] = [] for path in sorted(schema_dir.glob("*.json")): @@ -307,29 +329,13 @@ except UnicodeDecodeError: continue - if PRIVATE_KEY_BLOCK_RE.search(text): + for message in scan_text_for_secrets(text): issues.append( ValidationIssue( path=rel_path, severity="error", code="possible-secret", - message="File contains a private key block marker", + message=message, ) ) - - for line_number, line in enumerate(text.splitlines(), start=1): - match = SECRET_ASSIGNMENT_RE.search(line) - if not match: - continue - value = match.group("value").strip().strip("\"'") - if not value or SAFE_SECRET_REFERENCE_RE.search(value): - continue - issues.append( - ValidationIssue( - path=rel_path, - severity="error", - code="possible-secret", - message=f"Possible raw secret assignment at line {line_number}", - ) - ) - return issues + return issues \ No newline at end of file diff --git a/server/tests/test_pending_changes.py b/server/tests/test_pending_changes.py index 4b78a15..4fc98ed 100644 --- a/server/tests/test_pending_changes.py +++ b/server/tests/test_pending_changes.py @@ -101,7 +101,13 @@ raise AssertionError("Expected PendingChangeError") assert not (tmp_path / "10-systems" / "bad.md").exists() - assert repo.get(created["id"])["status"] == "pending" + # Full rollback: the pending JSON must not survive a failed apply. + try: + repo.get(created["id"]) + except PendingChangeError: + pass + else: + raise AssertionError("Pending change file should have been removed after failed apply") def test_rejects_non_doc_apply(tmp_path) -> None: diff --git a/server/tests/test_pending_changes_fix.py b/server/tests/test_pending_changes_fix.py new file mode 100644 index 0000000..c5a43e6 --- /dev/null +++ b/server/tests/test_pending_changes_fix.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import json + +import pytest + +from app.config import Settings +from app.pending_changes import PendingChangeError, PendingChangeRepository, ProposedChangeRequest +from app.validation import scan_text_for_secrets + + +def _copy_schema_files(tmp_path) -> None: + (tmp_path / "schemas").mkdir() + source_schema = Settings().repo_root / "schemas" + for schema in source_schema.glob("*.json"): + (tmp_path / "schemas" / schema.name).write_text(schema.read_text(), encoding="utf-8") + + +def _create_empty_inventory(tmp_path) -> None: + (tmp_path / "40-inventory").mkdir() + for name in [ + "backups", + "databases", + "domains", + "endpoints", + "hardware", + "hosts", + "integrations", + "networks", + "projects", + "services", + "traffic-routes", + "virtual-machines", + ]: + (tmp_path / "40-inventory" / f"{name}.yml").write_text("---\n[]\n", encoding="utf-8") + + +def test_scan_text_for_secrets_detects_assignment() -> None: + messages = scan_text_for_secrets("password: hunter2\n") + assert len(messages) == 1 + assert "line 1" in messages[0] + assert "hunter2" not in messages[0] + + +def test_scan_text_for_secrets_allows_safe_references() -> None: + assert scan_text_for_secrets("password: secret reference\n") == [] + assert scan_text_for_secrets("token: none\n") == [] + + +def test_create_rejects_secret_payload_without_writing_file(tmp_path) -> None: + repo = PendingChangeRepository(Settings(tmp_path)) + request = ProposedChangeRequest( + kind="doc", + target="10-systems/example.md", + summary="Poison the queue", + payload={"content": "---\nowner: gmikcon\nstatus: active\nlast_reviewed: 2026-05-09\nreview_interval: 90d\nconfidence: medium\nsource_of_truth: test\n---\n\npassword: hunter2\n"}, + ) + + with pytest.raises(PendingChangeError, match="possible secret"): + repo.create(request) + + assert list((tmp_path / "90-maintenance" / "pending-changes").glob("*.json")) == [] + + +def test_create_rejects_doc_target_escape(tmp_path) -> None: + repo = PendingChangeRepository(Settings(tmp_path)) + request = ProposedChangeRequest( + kind="doc", + target="../outside.md", + summary="Escape attempt", + payload={"content": "# Outside\n"}, + ) + + with pytest.raises(PendingChangeError, match="escapes repository root"): + repo.create(request) + + assert list((tmp_path / "90-maintenance" / "pending-changes").glob("*.json")) == [] + + +def test_failed_apply_removes_pending_file(tmp_path, monkeypatch) -> None: + _copy_schema_files(tmp_path) + _create_empty_inventory(tmp_path) + from app import pending_changes as pc + + monkeypatch.setattr(pc, "validate_repository", lambda settings: {"status": "issues", "issue_count": 1, "issues": [{"path": "x", "code": "test", "message": "forced failure"}]}) + repo = PendingChangeRepository(Settings(tmp_path)) + created = repo.create( + ProposedChangeRequest( + kind="doc", + target="10-systems/example.md", + summary="Add example doc", + payload={"content": "---\nowner: gmikcon\nstatus: active\nlast_reviewed: 2026-05-09\nreview_interval: 90d\nconfidence: medium\nsource_of_truth: test\n---\n\n# Example\n"}, + ) + ) + pending_file = tmp_path / "90-maintenance" / "pending-changes" / f"{created['id']}.json" + assert pending_file.exists() + + with pytest.raises(PendingChangeError, match="failed validation"): + repo.apply(created["id"]) + + assert not pending_file.exists() + + +def test_delete_pending_change(tmp_path) -> None: + repo = PendingChangeRepository(Settings(tmp_path)) + created = repo.create( + ProposedChangeRequest( + kind="inventory-item", + target="virtual-machines/gnauth", + summary="Update gnauth", + payload={"runs_services": ["gnexus-auth"]}, + ) + ) + + result = repo.delete(created["id"]) + assert result == {"id": created["id"], "status": "deleted"} + assert not (tmp_path / "90-maintenance" / "pending-changes" / f"{created['id']}.json").exists() + + with pytest.raises(PendingChangeError, match="not found"): + repo.delete(created["id"]) + + +def test_delete_refuses_applied_change(tmp_path) -> None: + _copy_schema_files(tmp_path) + _create_empty_inventory(tmp_path) + repo = PendingChangeRepository(Settings(tmp_path)) + created = repo.create( + ProposedChangeRequest( + kind="doc", + target="10-systems/example.md", + summary="Add example doc", + payload={"content": "---\nowner: gmikcon\nstatus: active\nlast_reviewed: 2026-05-09\nreview_interval: 90d\nconfidence: medium\nsource_of_truth: test\n---\n\n# Example\n"}, + ) + ) + repo.apply(created["id"]) + + with pytest.raises(PendingChangeError, match="Only pending or corrupt"): + repo.delete(created["id"]) + + +def test_list_tolerates_corrupt_json(tmp_path) -> None: + repo = PendingChangeRepository(Settings(tmp_path)) + created = repo.create( + ProposedChangeRequest( + kind="inventory-item", + target="virtual-machines/gnauth", + summary="Update gnauth", + payload={"runs_services": ["gnexus-auth"]}, + ) + ) + corrupt = tmp_path / "90-maintenance" / "pending-changes" / "20260101000000-deadbeef.json" + corrupt.write_text("{not valid json", encoding="utf-8") + + changes = repo.list() + statuses = {change["id"]: change["status"] for change in changes} + assert statuses[created["id"]] == "pending" + assert statuses["20260101000000-deadbeef"] == "corrupt" + + with pytest.raises(PendingChangeError, match="corrupt"): + repo.get("20260101000000-deadbeef") + + with pytest.raises(PendingChangeError, match="corrupt"): + repo.apply("20260101000000-deadbeef") + + result = repo.delete("20260101000000-deadbeef") + assert result == {"id": "20260101000000-deadbeef", "status": "deleted"} + + +def test_happy_path_create_apply_still_works(tmp_path) -> None: + _copy_schema_files(tmp_path) + _create_empty_inventory(tmp_path) + repo = PendingChangeRepository(Settings(tmp_path)) + created = repo.create( + ProposedChangeRequest( + kind="doc", + target="10-systems/example.md", + summary="Add example doc", + payload={"content": "---\nowner: gmikcon\nstatus: active\nlast_reviewed: 2026-05-09\nreview_interval: 90d\nconfidence: medium\nsource_of_truth: test\n---\n\n# Example\n"}, + ) + ) + + applied = repo.apply(created["id"]) + assert applied["status"] == "applied" + assert (tmp_path / "10-systems" / "example.md").exists() \ No newline at end of file