diff --git a/9router-watchdog.sh b/9router-watchdog.sh new file mode 100644 index 0000000..852339c --- /dev/null +++ b/9router-watchdog.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# 9router-watchdog.sh — local health monitor + auto-restart. +# +# Runs from cron (e.g. every minute). Checks the 9router API endpoint; +# if it does NOT respond with HTTP 200, the service is (re)started via +# start-9router.sh. +# +# Uses absolute paths because cron has a minimal PATH. +# Run as root (so it can drop privileges via runuser inside start-9router.sh). + +CURL=/usr/bin/curl +WRAPPER=/usr/local/bin/start-9router.sh +URL=http://127.0.0.1:20128/v1/models +LOG=/home/ardhi/9router.log + +# If the wrapper is missing, nothing we can do. +[ -x "$WRAPPER" ] || exit 0 + +# Probe the API. -f makes curl exit non-zero on HTTP >= 400. +if "$CURL" -fs -m 5 -o /dev/null "$URL" 2>/dev/null; then + # Healthy — nothing to do. + exit 0 +fi + +# Unhealthy (no response / non-200) → restart. +echo "[$(date '+%Y-%m-%d %H:%M:%S')] watchdog: 9router not healthy, restarting" >> "$LOG" 2>/dev/null +"$WRAPPER" +exit 0 diff --git a/README.md b/README.md index 75012e4..ec6502a 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,38 @@ This project solves all three with a tiny **wrapper script** + an **adaptive cro --- +## Optional: local watchdog (auto-restart if it dies) + +`@reboot` starts the service at boot but does **not** restart it if the parent +`-t` process later dies. Add a local watchdog that probes the API every minute +and restarts 9router if it isn't healthy. + +`9router-watchdog.sh` checks `http://127.0.0.1:20128/v1/models` — if it does **not** +return HTTP 200, it calls `start-9router.sh` again. Run it from root's crontab: + +```bash +# root crontab — every minute +* * * * * /usr/local/bin/9router-watchdog.sh +``` + +Or just run `sudo ./install.sh` (it registers both the `@reboot` job and the +per-minute watchdog for you). + +Notes: +- The check is a real HTTP 200 probe (not just "is the port open"), so a hung + Next.js process that still holds the port will also be detected and restarted. +- Max recovery delay is ~1 minute (the cron interval). +- The watchdog runs **locally** on the same machine — no external monitoring infra needed. + +--- + ## Files | File | Purpose | |------|---------| | `start-9router.sh` | Detached launcher. Drops to `ardhi` if run as root, runs directly if already `ardhi`. Uses absolute paths (cron `@reboot` has a minimal `PATH`). | -| `install.sh` | One-shot setup: installs the wrapper, fixes log ownership, and registers the `@reboot` cron as **root**. | +| `9router-watchdog.sh` | Local health monitor. Probes `/v1/models`; restarts via the wrapper if not HTTP 200. | +| `install.sh` | One-shot setup: installs both scripts, fixes log ownership, and registers the `@reboot` cron + per-minute watchdog as **root**. | | `README.md` | This file. | --- diff --git a/install.sh b/install.sh index 5ca4e2e..c425cd4 100755 --- a/install.sh +++ b/install.sh @@ -5,20 +5,27 @@ set -e WRAPPER_SRC="./start-9router.sh" WRAPPER_DST="/usr/local/bin/start-9router.sh" +WATCHDOG_SRC="./9router-watchdog.sh" +WATCHDOG_DST="/usr/local/bin/9router-watchdog.sh" LOG="/home/ardhi/9router.log" USER="ardhi" echo "==> Installing wrapper to $WRAPPER_DST" install -m 0755 "$WRAPPER_SRC" "$WRAPPER_DST" +echo "==> Installing watchdog to $WATCHDOG_DST" +install -m 0755 "$WATCHDOG_SRC" "$WATCHDOG_DST" + echo "==> Preparing log file ($LOG) owned by $USER" touch "$LOG" chown "$USER:$USER" "$LOG" chmod 644 "$LOG" -echo "==> Registering @reboot cron as ROOT" -# Must be root's crontab (wrapper uses runuser to drop privileges). -( crontab -l 2>/dev/null | grep -v 'start-9router.sh'; echo "@reboot $WRAPPER_DST" ) | crontab - +echo "==> Registering @reboot cron + watchdog as ROOT" +# Must be root's crontab (wrapper/watchdog use runuser to drop privileges). +( crontab -l 2>/dev/null | grep -v 'start-9router.sh'; grep -v '9router-watchdog.sh'; \ + echo "@reboot $WRAPPER_DST"; \ + echo "* * * * * $WATCHDOG_DST" ) | crontab - echo "==> Ensuring cron daemon is active" systemctl enable cron >/dev/null 2>&1 || true @@ -30,3 +37,4 @@ systemctl disable 9router >/dev/null 2>&1 || true echo "==> Done. Reboot to verify:" echo " ss -ltnp | grep 20128" echo " curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:20128/v1/models" +echo "==> Watchdog runs every minute (cron) and restarts if /v1/models != 200."