diff --git a/10-systems/servers/service-host-navi-rei.md b/10-systems/servers/service-host-navi-rei.md new file mode 100644 index 0000000..7718004 --- /dev/null +++ b/10-systems/servers/service-host-navi-rei.md @@ -0,0 +1,31 @@ +--- +title: service host — navi rei (192.168.1.168) +type: host +ip: 192.168.1.168 +purpose: Navi rei instance and local Ollama embeddings server for the hive +owner: gmikcon +status: active +confidence: 95 +last_reviewed: 2026-10-12 +review_interval: 90d +source_of_truth: local +--- + +# service host — navi rei instance (192.168.1.168) + +Local Ubuntu 22.04 host (8 vCPU, 8 GiB RAM, no GPU). Runs the **rei** Navi instance (navi-1 backend on port 8099) and the **local Ollama embeddings server** used by the whole hive. + +## Local Ollama embeddings service + +- Endpoint: `http://192.168.1.168:11434` — referenced as `EMBEDDING_OLLAMA_HOST` by other hive instances (zoe, gntasks, etc.). +- Model: `nomic-embed-text:latest` (137M, F16), CPU inference only (no GPU on this host). +- **Two systemd units exist — do not confuse them:** + - `ollama-embed.service` — the working unit, holds port 11434, `WantedBy=default.target`, enabled. This is the one that must stay running. + - `ollama.service` — stock Ollama unit, same `ExecStart` and same port. At boot it loses the port race to `ollama-embed` and crash-loops forever (`bind: address already in use`, ~140k restarts observed by 2026-10-12). + +## 2026-10-12: crash-loop fix + +- Found during cpu_recovered event analysis (panel event #822): `ollama.service` had been crash-looping since boot (5+ days), spamming the journal (~160k lines/24h, journal 2.9 GB). +- Fix applied: `sudo systemctl stop ollama.service && sudo systemctl disable ollama.service` (symlink removed from `default.target.wants`). `ollama-embed.service` untouched, embeddings verified working (`/api/ps`, `/api/embed` OK). +- Rollback if ever needed: `sudo systemctl enable --now ollama.service` — but the port conflict will return; the proper long-term fix is removing the stock unit or pointing it at another port. +- CPU spikes on this host (up to 100%, load 3–7) are normally embedding batch jobs from hive instances — self-recovering, no notify needed unless they persist >30 min or RAM/swap grows. diff --git a/90-maintenance/pending-changes/20261012182348-6745cf93.json b/90-maintenance/pending-changes/20261012182348-6745cf93.json new file mode 100644 index 0000000..5e1b139 --- /dev/null +++ b/90-maintenance/pending-changes/20261012182348-6745cf93.json @@ -0,0 +1,19 @@ +{ + "id": "20261012182348-6745cf93", + "kind": "doc", + "target": "10-systems/servers/service-host-navi-rei.md", + "summary": "Add doc for navi host (192.168.1.168): local ollama embeddings service, two-unit layout, crash-loop fix 2026-10-12", + "reason": "Fix applied on 2026-10-12: stock ollama.service crash-looped since boot (port conflict with ollama-embed.service); stopped and disabled. Documented so hive instances referencing 192.168.1.168:11434 know the unit layout.", + "payload": { + "content": "---\ntitle: service host — navi rei (192.168.1.168)\ntype: host\nip: 192.168.1.168\npurpose: Navi rei instance and local Ollama embeddings server for the hive\nowner: gmikcon\nstatus: active\nconfidence: 95\nlast_reviewed: 2026-10-12\nreview_interval: 90d\nsource_of_truth: local\n---\n\n# service host — navi rei instance (192.168.1.168)\n\nLocal Ubuntu 22.04 host (8 vCPU, 8 GiB RAM, no GPU). Runs the **rei** Navi instance (navi-1 backend on port 8099) and the **local Ollama embeddings server** used by the whole hive.\n\n## Local Ollama embeddings service\n\n- Endpoint: `http://192.168.1.168:11434` — referenced as `EMBEDDING_OLLAMA_HOST` by other hive instances (zoe, gntasks, etc.).\n- Model: `nomic-embed-text:latest` (137M, F16), CPU inference only (no GPU on this host).\n- **Two systemd units exist — do not confuse them:**\n - `ollama-embed.service` — the working unit, holds port 11434, `WantedBy=default.target`, enabled. This is the one that must stay running.\n - `ollama.service` — stock Ollama unit, same `ExecStart` and same port. At boot it loses the port race to `ollama-embed` and crash-loops forever (`bind: address already in use`, ~140k restarts observed by 2026-10-12).\n\n## 2026-10-12: crash-loop fix\n\n- Found during cpu_recovered event analysis (panel event #822): `ollama.service` had been crash-looping since boot (5+ days), spamming the journal (~160k lines/24h, journal 2.9 GB).\n- Fix applied: `sudo systemctl stop ollama.service && sudo systemctl disable ollama.service` (symlink removed from `default.target.wants`). `ollama-embed.service` untouched, embeddings verified working (`/api/ps`, `/api/embed` OK).\n- Rollback if ever needed: `sudo systemctl enable --now ollama.service` — but the port conflict will return; the proper long-term fix is removing the stock unit or pointing it at another port.\n- CPU spikes on this host (up to 100%, load 3–7) are normally embedding batch jobs from hive instances — self-recovering, no notify needed unless they persist >30 min or RAM/swap grows." + }, + "status": "applied", + "created_at": "2026-10-12T18:23:48Z", + "updated_at": "2026-10-12T18:23:50Z", + "applied_at": "2026-10-12T18:23:50Z", + "validation": { + "status": "ok", + "issue_count": 0, + "issues": [] + } +}