#!/bin/bash # Read-only rootless-docker / control-plane health scan — the shared detection # used by both the WebUI status generator (webuiSystemHealthCheck) and the heal # verb (system health heal), so the two never diverge. # # dockerHealthScan sets these globals (call it DIRECTLY, never in $(...) — a # subshell would drop them): # HEALTH_DAEMON_OK "true"/"false" — rootless docker daemon reachable # HEALTH_WEBUI_PRESENT "true"/"false" — the core WebUI container exists # HEALTH_WEBUI_RUNNING "true"/"false" — ...and is running (not exited/restarting) # HEALTH_WEBUI_PORT — its published host port (docker port), "" if none # HEALTH_WEBUI_REACHABLE "true"/"false"/"unknown" — that host port accepts a TCP connect # HEALTH_SCAN_ERROR — human note when the daemon is off (else "") # HEALTH_CRASHLOOPS (array) — "app|container|restartcount" per container # crash-looping: RestartCount >= CFG_HEALTH_ # CRASHLOOP_LIMIT AND (caught mid-restart OR # its count CLIMBED since the previous scan). # HEALTH_RESTART_SNAPSHOT — "container|restartcount" lines for every # container, for the throttled caller to # persist as the next scan's delta baseline. # # The incident this guards: an app crash-loop (trivy, offline, FATAL on every # boot) churned the rootless port-forwarder until the WebUI's published port was # torn down — the container stayed healthy INSIDE, but nothing on the host could # reach it. So we probe the host-visible port forward, not just container state. # # Nothing here mutates state. # The core WebUI container + its internal port (the control plane we must keep # reachable). Kept as tiny functions so a rename only touches one place. _healthWebuiContainer() { echo "libreportal-service"; } _healthWebuiInternalPort() { echo "1111"; } # TCP-connect probe from the host (manager context). Success => the published # port forward is live. Uses bash /dev/tcp (always present) with a timeout so a # black-holed forward can't hang the poll. _healthTcpReachable() { local host="$1" port="$2" [[ -n "$port" ]] || return 2 timeout 4 bash -c "exec 3<>/dev/tcp/${host}/${port}" 2>/dev/null } dockerHealthScan() { HEALTH_DAEMON_OK="false"; HEALTH_WEBUI_PRESENT="false"; HEALTH_WEBUI_RUNNING="false" HEALTH_WEBUI_PORT=""; HEALTH_WEBUI_REACHABLE="unknown"; HEALTH_SCAN_ERROR="" HEALTH_CRASHLOOPS=() local limit="${CFG_HEALTH_CRASHLOOP_LIMIT:-3}" [[ "$limit" =~ ^[0-9]+$ ]] || limit=3 # Daemon reachable? Never alarm on what we can't verify — a daemon blip (or a # mid-recycle window) is transient, not a conflict. if ! dockerCommandRun "docker info" >/dev/null 2>&1; then HEALTH_SCAN_ERROR="docker daemon unreachable" return 0 fi HEALTH_DAEMON_OK="true" # Crash-loop detection. A backed-off loop sits "exited" between restarts # (docker's restart backoff grows to tens of seconds), so an instantaneous # `status=restarting` check misses it once it slows. We flag a container when # RestartCount >= limit AND either it's caught mid-restart OR its RestartCount # has CLIMBED since the previous scan — that delta over time is what "crash # loop" actually means (and it won't false-positive a container that restarted # a few times historically and is now stable). Baseline counts persist in # $state_file, written ONLY by the throttled caller (webuiSystemHealthCheck) # via HEALTH_RESTART_SNAPSHOT, so the heal's rapid re-scans don't disturb the # baseline. One inspect over all containers (name|count|status|restarting). local state_file="$(webuiDir)/frontend/data/system/.health_restart_counts" HEALTH_RESTART_SNAPSHOT="" declare -A _prev_rc=() if [[ -f "$state_file" ]]; then local pn pc while IFS='|' read -r pn pc; do [[ -n "$pn" ]] && _prev_rc["$pn"]="$pc" done < "$state_file" fi local snap name rc status restarting prev snap=$(dockerCommandRun "docker ps -aq | xargs -r docker inspect --format '{{.Name}}|{{.RestartCount}}|{{.State.Status}}|{{.State.Restarting}}'" 2>/dev/null) while IFS='|' read -r name rc status restarting; do name="${name#/}" # docker .Name carries a leading '/' [[ -n "$name" ]] || continue [[ "$rc" =~ ^[0-9]+$ ]] || rc=0 HEALTH_RESTART_SNAPSHOT+="${name}|${rc}"$'\n' (( rc >= limit )) || continue prev="${_prev_rc[$name]-}" if [[ "$restarting" == "true" || "$status" == "restarting" ]] \ || { [[ "$prev" =~ ^[0-9]+$ ]] && (( rc > prev )); }; then HEALTH_CRASHLOOPS+=("${name%-service}|${name}|${rc}") fi done <<< "$snap" # Core WebUI container: present? running? host port reachable? local webui iport wstate webui="$(_healthWebuiContainer)" iport="$(_healthWebuiInternalPort)" wstate=$(dockerCommandRun "docker inspect --format '{{.State.Status}}' '$webui'" 2>/dev/null | tr -d '[:space:]') if [[ -z "$wstate" ]]; then return 0 # container not found — nothing more to probe fi HEALTH_WEBUI_PRESENT="true" [[ "$wstate" == "running" ]] && HEALTH_WEBUI_RUNNING="true" # docker's view of the published host port (e.g. "1111/tcp -> 0.0.0.0:9781"). # Note: this reports the INTENDED mapping even when the rootless forward has # been torn down — which is exactly why we then TCP-probe it for real. HEALTH_WEBUI_PORT=$(dockerCommandRun "docker port '$webui' '$iport'" 2>/dev/null | head -1 | sed -n 's/.*:\([0-9][0-9]*\)$/\1/p') if [[ "$HEALTH_WEBUI_RUNNING" == "true" && -n "$HEALTH_WEBUI_PORT" ]]; then if _healthTcpReachable "127.0.0.1" "$HEALTH_WEBUI_PORT"; then HEALTH_WEBUI_REACHABLE="true" else HEALTH_WEBUI_REACHABLE="false" fi fi }