From 24fbc365183ee62cea7beaac8efb5258cb4d0eee Mon Sep 17 00:00:00 2001 From: librelad Date: Fri, 17 Jul 2026 22:53:54 +0100 Subject: [PATCH] fix(health): robust crash-loop detection (RestartCount delta) + task display MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crash-loop detection keyed on `docker ps --filter status=restarting`, but a backed-off loop sits "exited" between restarts once docker's backoff grows to tens of seconds — so a slowed loop is missed. Detect via RestartCount CLIMBING between scans (what a crash loop actually is), unioned with the instantaneous restarting signal for fast loops. Baseline counts persist in .health_restart_counts, written only by the throttled check so the heal's re-scans don't disturb the delta. Also give the system_health_heal / system_network_heal tasks proper display (they fell through to the raw command + generic ⚙️): friendly titles in formatCommandForUser, type icons (🩺 / 🌐) in getTaskTypeIcon, and action labels in formatActionTitle — so they read as "LibrePortal - Repair Control Plane" etc. with an icon, like install tasks. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: librelad --- .../components/tasks/js/tasks-format.js | 11 +++- .../components/tasks/js/tasks-list-render.js | 2 + scripts/docker/health/docker_health_scan.sh | 55 +++++++++++++------ .../generators/system/webui_system_health.sh | 9 +++ 4 files changed, 58 insertions(+), 19 deletions(-) diff --git a/containers/libreportal/frontend/components/tasks/js/tasks-format.js b/containers/libreportal/frontend/components/tasks/js/tasks-format.js index 1fdb727..8f40a60 100644 --- a/containers/libreportal/frontend/components/tasks/js/tasks-format.js +++ b/containers/libreportal/frontend/components/tasks/js/tasks-format.js @@ -41,9 +41,13 @@ Object.assign(TasksManager.prototype, { { match: /^libreportal regen\b/, title: 'LibrePortal - Regenerate WebUI Data' }, // -- System maintenance ------------------------------------------------ - { match: /^libreportal system reclaim\b/, title: 'LibrePortal - Reclaim Space' }, - { match: /^libreportal system image rm\b/, title: 'LibrePortal - Remove Images' }, - { match: /^libreportal verify\b/, title: 'LibrePortal - Verify System' }, + { match: /^libreportal system reclaim\b/, title: 'LibrePortal - Reclaim Space' }, + { match: /^libreportal system image rm\b/, title: 'LibrePortal - Remove Images' }, + { match: /^libreportal verify\b/, title: 'LibrePortal - Verify System' }, + { match: /^libreportal system health heal\b/, title: 'LibrePortal - Repair Control Plane' }, + { match: /^libreportal system health check\b/, title: 'LibrePortal - Check Control Plane' }, + { match: /^libreportal system network heal\b/, title: 'LibrePortal - Heal Network' }, + { match: /^libreportal system network check\b/, title: 'LibrePortal - Check Network' }, // -- Backup: per-app (these capture the app slug) ---------------------- { match: /^libreportal backup app create (\w+)/, title: (m) => `${displayName(m[1])} - Create Backup` }, @@ -138,6 +142,7 @@ Object.assign(TasksManager.prototype, { 'config_update': 'Update Config', 'update_config': 'Update Config', 'system_update': 'Update System', 'system_reclaim': 'Reclaim Space', 'system_image_rm': 'Remove Images', 'verify': 'Verify System', + 'system_health_heal': 'Repair Control Plane', 'system_network_heal': 'Heal Network', 'updater_check': 'Check for Updates', 'updater_apply': 'Update', 'updater_apply_all': 'Update All', 'updater_rollback': 'Roll Back', 'artifact_apply': 'Apply Hotfix', 'artifact_revert': 'Revert Hotfix', diff --git a/containers/libreportal/frontend/components/tasks/js/tasks-list-render.js b/containers/libreportal/frontend/components/tasks/js/tasks-list-render.js index 3194195..57c655c 100644 --- a/containers/libreportal/frontend/components/tasks/js/tasks-list-render.js +++ b/containers/libreportal/frontend/components/tasks/js/tasks-list-render.js @@ -306,6 +306,8 @@ Object.assign(TasksManager.prototype, { 'delete': { icon: '🗑️', class: 'delete' }, 'delete_all': { icon: '🗑️', class: 'delete' }, 'system_image_rm': { icon: '🗑️', class: 'delete' }, + 'system_health_heal': { icon: '🩺', class: 'verify' }, + 'system_network_heal': { icon: '🌐', class: 'restart' }, 'verify': { icon: '🛡️', class: 'verify' }, 'setup-config': { icon: '🛠️', class: 'setup' }, 'setup-finalize': { icon: '🎉', class: 'setup' }, diff --git a/scripts/docker/health/docker_health_scan.sh b/scripts/docker/health/docker_health_scan.sh index 2247cbe..a32c073 100644 --- a/scripts/docker/health/docker_health_scan.sh +++ b/scripts/docker/health/docker_health_scan.sh @@ -13,9 +13,12 @@ # HEALTH_WEBUI_REACHABLE "true"/"false"/"unknown" — that host port accepts a TCP connect # HEALTH_SCAN_ERROR — human note when the daemon is off (else "") # HEALTH_CRASHLOOPS (array) — "app|container|restartcount" per container -# stuck restarting (State=restarting and -# RestartCount >= CFG_HEALTH_CRASHLOOP_LIMIT): -# a crash-loop churning the shared network. +# crash-looping: RestartCount >= CFG_HEALTH_ +# CRASHLOOP_LIMIT AND (caught mid-restart OR +# its count CLIMBED since the previous scan). +# HEALTH_RESTART_SNAPSHOT — "container|restartcount" lines for every +# container, for the throttled caller to +# persist as the next scan's delta baseline. # # The incident this guards: an app crash-loop (trivy, offline, FATAL on every # boot) churned the rootless port-forwarder until the WebUI's published port was @@ -54,21 +57,41 @@ dockerHealthScan() { fi HEALTH_DAEMON_OK="true" - # Crash-loopers: containers docker reports as "restarting" whose RestartCount - # has already climbed past the limit — i.e. actively churning, not a one-off - # restart. Container name is enough to name the offender in the badge; derive - # a friendly app label by trimming the conventional "-service" suffix. - local names name rc app - names=$(dockerCommandRun "docker ps -a --filter status=restarting --format '{{.Names}}'" 2>/dev/null) - while IFS= read -r name; do + # Crash-loop detection. A backed-off loop sits "exited" between restarts + # (docker's restart backoff grows to tens of seconds), so an instantaneous + # `status=restarting` check misses it once it slows. We flag a container when + # RestartCount >= limit AND either it's caught mid-restart OR its RestartCount + # has CLIMBED since the previous scan — that delta over time is what "crash + # loop" actually means (and it won't false-positive a container that restarted + # a few times historically and is now stable). Baseline counts persist in + # $state_file, written ONLY by the throttled caller (webuiSystemHealthCheck) + # via HEALTH_RESTART_SNAPSHOT, so the heal's rapid re-scans don't disturb the + # baseline. One inspect over all containers (name|count|status|restarting). + local state_file="${containers_dir}/libreportal/frontend/data/system/.health_restart_counts" + HEALTH_RESTART_SNAPSHOT="" + + declare -A _prev_rc=() + if [[ -f "$state_file" ]]; then + local pn pc + while IFS='|' read -r pn pc; do + [[ -n "$pn" ]] && _prev_rc["$pn"]="$pc" + done < "$state_file" + fi + + local snap name rc status restarting prev + snap=$(dockerCommandRun "docker ps -aq | xargs -r docker inspect --format '{{.Name}}|{{.RestartCount}}|{{.State.Status}}|{{.State.Restarting}}'" 2>/dev/null) + while IFS='|' read -r name rc status restarting; do + name="${name#/}" # docker .Name carries a leading '/' [[ -n "$name" ]] || continue - rc=$(dockerCommandRun "docker inspect --format '{{.RestartCount}}' '$name'" 2>/dev/null | tr -dc '0-9') - [[ -n "$rc" ]] || rc=0 - if (( rc >= limit )); then - app="${name%-service}" - HEALTH_CRASHLOOPS+=("${app}|${name}|${rc}") + [[ "$rc" =~ ^[0-9]+$ ]] || rc=0 + HEALTH_RESTART_SNAPSHOT+="${name}|${rc}"$'\n' + (( rc >= limit )) || continue + prev="${_prev_rc[$name]-}" + if [[ "$restarting" == "true" || "$status" == "restarting" ]] \ + || { [[ "$prev" =~ ^[0-9]+$ ]] && (( rc > prev )); }; then + HEALTH_CRASHLOOPS+=("${name%-service}|${name}|${rc}") fi - done <<< "$names" + done <<< "$snap" # Core WebUI container: present? running? host port reachable? local webui iport wstate diff --git a/scripts/webui/data/generators/system/webui_system_health.sh b/scripts/webui/data/generators/system/webui_system_health.sh index d7a58cc..f032551 100644 --- a/scripts/webui/data/generators/system/webui_system_health.sh +++ b/scripts/webui/data/generators/system/webui_system_health.sh @@ -112,6 +112,15 @@ webuiSystemHealthCheck() { EOF runFileWrite "$final_file" < "$temp_file"; rm -f "$temp_file" + # Persist the restart-count baseline for the next scan's crash-loop delta. + # ONLY the throttled check writes it (the heal re-scans between actions and + # must not disturb the baseline), so the delta measures a full interval. + if [[ -n "$HEALTH_RESTART_SNAPSHOT" ]]; then + local rc_tmp; rc_tmp="$(mktemp)" + printf '%s' "$HEALTH_RESTART_SNAPSHOT" > "$rc_tmp" + runFileWrite "${system_dir}/.health_restart_counts" < "$rc_tmp"; rm -f "$rc_tmp" + fi + # Self-dispatch a heal on an auto-healable problem, throttled by a cooldown # stamp. The WebUI can't be clicked when its own port is down, so the poll # must drive the fix — this is the backend counterpart of the "Heal now"