diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh index ba419ae..ce3a4cc 100755 --- a/scripts/healthcheck.sh +++ b/scripts/healthcheck.sh @@ -14,14 +14,58 @@ fi } +notify_disk() { + if [ -n "${SYNAPSE_URL:-}" ] && [ -n "${SYNAPSE_API_KEY:-}" ]; then + "$SCRIPT_DIR/notify-synapse.sh" \ + "Gnexus Book backup disk space" \ + "$1" \ + "$2" \ + "$3" >/dev/null 2>&1 || true + fi +} + +STATUS=0 + +check_disk() { + local path="${GNEXUS_BOOK_DISK_PATH:-}" + local used warn max message + [ -n "$path" ] || return 0 + if [ ! -d "$path" ]; then + message="Backup path unavailable: $path on $(hostname)" + echo "$message" >&2 + notify "$message" + return 1 + fi + used="$(df -P "$path" | awk 'NR==2 {gsub(/%/, "", $5); print $5}')" + warn="${GNEXUS_BOOK_DISK_WARN_PERCENT:-90}" + max="${GNEXUS_BOOK_DISK_MAX_PERCENT:-98}" + if [ -n "$used" ] && [ "$used" -ge "$max" ]; then + message="Backup disk critical: $path ${used}% used on $(hostname)" + echo "$message" >&2 + notify_disk "disk.critical" "$message" "high" + return 1 + fi + if [ -n "$used" ] && [ "$used" -ge "$warn" ]; then + message="Backup disk warning: $path ${used}% used on $(hostname)" + echo "$message" >&2 + notify_disk "disk.warning" "$message" "normal" + echo "disk_warning=${used}%" + return 0 + fi + echo "disk_ok=${used}%" + return 0 +} + +if ! check_disk; then + STATUS=1 +fi + if ! RESPONSE="$(curl -fsS --max-time 5 "$URL")"; then MESSAGE="Health request failed: $URL on $(hostname)" echo "$MESSAGE" >&2 notify "$MESSAGE" - exit 1 -fi - -if ! python3 -c ' + STATUS=1 +elif ! python3 -c ' import json import sys @@ -37,7 +81,11 @@ MESSAGE="Health response is not ok: $RESPONSE" echo "$MESSAGE" >&2 notify "$MESSAGE" - exit 1 + STATUS=1 fi -echo "ok" +if [ "$STATUS" -eq 0 ]; then + echo "ok" +else + exit 1 +fi diff --git a/systemd/gnexus-book-healthcheck.service b/systemd/gnexus-book-healthcheck.service new file mode 100644 index 0000000..6f61c75 --- /dev/null +++ b/systemd/gnexus-book-healthcheck.service @@ -0,0 +1,10 @@ +[Unit] +Description=Gnexus Book health check +After=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/gnexus-book +EnvironmentFile=-/home/ubuntu/gnexus-book/server/.env +ExecStart=/home/ubuntu/gnexus-book/scripts/healthcheck.sh +Nice=10 diff --git a/systemd/gnexus-book-healthcheck.timer b/systemd/gnexus-book-healthcheck.timer new file mode 100644 index 0000000..577a8da --- /dev/null +++ b/systemd/gnexus-book-healthcheck.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Scheduled Gnexus Book health check + +[Timer] +OnBootSec=2min +OnUnitActiveSec=5min + +[Install] +WantedBy=timers.target