diff --git a/scripts/backup-db.sh b/scripts/backup-db.sh index 3db04f3..7323928 100755 --- a/scripts/backup-db.sh +++ b/scripts/backup-db.sh @@ -1,6 +1,22 @@ #!/bin/bash set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +on_exit() { + status=$? + trap - EXIT + if [ "$status" -ne 0 ] && [ -n "${SYNAPSE_URL:-}" ] && [ -n "${SYNAPSE_API_KEY:-}" ]; then + "$SCRIPT_DIR/notify-synapse.sh" \ + "Gnexus Book backup failed" \ + "backup.failed" \ + "backup-db.sh exited with $status on $(hostname)" \ + "high" >/dev/null 2>&1 || true + fi + exit "$status" +} +trap on_exit EXIT + DB_URL="${GNEXUS_BOOK_DATABASE_URL:-}" if [ -z "$DB_URL" ]; then echo "GNEXUS_BOOK_DATABASE_URL is required" >&2 diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh new file mode 100755 index 0000000..ba419ae --- /dev/null +++ b/scripts/healthcheck.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +URL="${GNEXUS_BOOK_HEALTH_URL:-http://127.0.0.1:8000/health}" + +notify() { + if [ -n "${SYNAPSE_URL:-}" ] && [ -n "${SYNAPSE_API_KEY:-}" ]; then + "$SCRIPT_DIR/notify-synapse.sh" \ + "Gnexus Book health check failed" \ + "health.failed" \ + "$1" \ + "high" >/dev/null 2>&1 || true + fi +} + +if ! RESPONSE="$(curl -fsS --max-time 5 "$URL")"; then + MESSAGE="Health request failed: $URL on $(hostname)" + echo "$MESSAGE" >&2 + notify "$MESSAGE" + exit 1 +fi + +if ! python3 -c ' +import json +import sys + +ok = {"ok", "healthy", "pass", "passing", "up"} +data = json.load(sys.stdin) +status = str(data.get("status", "")).lower() +if status not in ok: + sys.exit(1) +db = (data.get("checks") or {}).get("db") +if db and str(db.get("status", "")).lower() not in ok: + sys.exit(1) +' <<<"$RESPONSE"; then + MESSAGE="Health response is not ok: $RESPONSE" + echo "$MESSAGE" >&2 + notify "$MESSAGE" + exit 1 +fi + +echo "ok" diff --git a/scripts/notify-synapse.sh b/scripts/notify-synapse.sh new file mode 100755 index 0000000..de1b79a --- /dev/null +++ b/scripts/notify-synapse.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -euo pipefail + +: "${SYNAPSE_URL:?SYNAPSE_URL is required}" +: "${SYNAPSE_API_KEY:?SYNAPSE_API_KEY is required}" + +SOURCE="${SYNAPSE_DEFAULT_SOURCE:-gnexus-book}" +SUBJECT="${1:-Gnexus Book alert}" +ACTION="${2:-alert}" +TEXT="${3:-}" +PRIORITY="${4:-high}" +ENDPOINT="${SYNAPSE_URL%/}/api/v1/events" + +BODY="$(python3 - "$SOURCE" "$SUBJECT" "$ACTION" "$TEXT" "$PRIORITY" <<'PY' +import json +import sys + +source, subject, action, text, priority = sys.argv[1:6] +print( + json.dumps( + { + "source": source, + "subject": subject, + "action": action, + "priority": priority, + "payload": {"service": "gnexus-book", "text": text}, + "dedup_key": f"gnexus-book:{action}:{subject}", + "ttl_seconds": 3600, + }, + ensure_ascii=False, + ) +) +PY +)" + +curl -fsS --max-time 10 \ + -X POST "$ENDPOINT" \ + -H "Authorization: Bearer $SYNAPSE_API_KEY" \ + -H "Content-Type: application/json" \ + --data-binary "$BODY" >/dev/null diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh new file mode 100755 index 0000000..30b9f2d --- /dev/null +++ b/scripts/restore-drill.sh @@ -0,0 +1,93 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +DB_URL="${GNEXUS_BOOK_DATABASE_URL:-}" +if [ -z "$DB_URL" ]; then + echo "GNEXUS_BOOK_DATABASE_URL is required" >&2 + exit 2 +fi + +DEST="${GNEXUS_BOOK_BACKUP_DIR:-$HOME/NetTera8/service/backups/gnexus-book/db}" +DUMP="${1:-}" +if [ -z "$DUMP" ]; then + DUMP="$(ls -1t "$DEST"/gnexus_book_*.dump 2>/dev/null | head -1 || true)" +fi +if [ -z "$DUMP" ] || [ ! -f "$DUMP" ]; then + echo "Dump not found; pass a dump path or set GNEXUS_BOOK_BACKUP_DIR" >&2 + exit 2 +fi + +read -r MAINT_PG_URL TEMP_PG_URL TEMP_APP_URL TEMP_DB <<<"$( + DB_URL="$DB_URL" "$REPO_ROOT/server/.venv/bin/python" - <<'PY' +import os +from datetime import datetime +from sqlalchemy.engine.url import make_url + +url = make_url(os.environ["DB_URL"]) +name = "gnexus_book_restore_" + datetime.now().strftime("%Y%m%d_%H%M%S") + "_" + str(os.getpid()) +maintenance = url.set(database="postgres") +temp = url.set(database=name) + + +def pg(value): + return value.render_as_string(hide_password=False).replace( + "postgresql+psycopg", "postgresql" + ) + + +print(pg(maintenance), pg(temp), temp.render_as_string(hide_password=False), name) +PY +)" + +CREATED=0 +cleanup() { + status=$? + trap - EXIT + if [ "$CREATED" = "1" ]; then + psql "$MAINT_PG_URL" -v ON_ERROR_STOP=1 \ + -c "DROP DATABASE IF EXISTS \"$TEMP_DB\" WITH (FORCE)" >/dev/null 2>&1 || true + fi + if [ "$status" -ne 0 ] && [ -n "${SYNAPSE_URL:-}" ] && [ -n "${SYNAPSE_API_KEY:-}" ]; then + "$SCRIPT_DIR/notify-synapse.sh" \ + "Gnexus Book restore drill failed" \ + "restore.failed" \ + "restore-drill.sh exited with $status for $(basename "$DUMP") on $(hostname)" \ + "high" >/dev/null 2>&1 || true + fi + exit "$status" +} +trap cleanup EXIT + +pg_restore --list "$DUMP" >/dev/null +psql "$MAINT_PG_URL" -v ON_ERROR_STOP=1 -c "CREATE DATABASE \"$TEMP_DB\"" >/dev/null +CREATED=1 +pg_restore --no-owner --no-acl --dbname "$TEMP_PG_URL" "$DUMP" + +cd "$REPO_ROOT/server" +GNEXUS_BOOK_STORAGE_BACKEND=db \ +GNEXUS_BOOK_DATABASE_URL="$TEMP_APP_URL" \ +.venv/bin/python - <<'PY' +import os +import sys + +from app.storage.db import Database +from app.storage.importer import database_counts +from app.storage.validation import validate_database + +database = Database(os.environ["GNEXUS_BOOK_DATABASE_URL"]) +counts = database_counts(database) +validation = validate_database(database) +print( + "documents={documents} inventory_items={inventory_items} validation={validation}".format( + documents=counts["documents"], + inventory_items=counts["inventory_items"], + validation=validation["status"], + ) +) +if counts["documents"] == 0 or validation["status"] != "ok": + sys.exit(1) +PY + +echo "restore drill ok: $(basename "$DUMP")"