#!/bin/bash # Read-only rootless-docker / control-plane health scan — the shared detection # used by both the WebUI status generator (webuiSystemHealthCheck) and the heal # verb (system health heal), so the two never diverge. # # dockerHealthScan sets these globals (call it DIRECTLY, never in $(...) — a # subshell would drop them): # HEALTH_DAEMON_OK "true"/"false" — rootless docker daemon reachable # HEALTH_WEBUI_PRESENT "true"/"false" — the core WebUI container exists # HEALTH_WEBUI_RUNNING "true"/"false" — ...and is running (not exited/restarting) # HEALTH_WEBUI_PORT — its published host port (docker port), "" if none # HEALTH_WEBUI_REACHABLE "true"/"false"/"unknown" — that host port accepts a TCP connect # HEALTH_SCAN_ERROR — human note when the daemon is off (else "") # HEALTH_CRASHLOOPS (array) — "app|container|restartcount" per container # stuck restarting (State=restarting and # RestartCount >= CFG_HEALTH_CRASHLOOP_LIMIT): # a crash-loop churning the shared network. # # The incident this guards: an app crash-loop (trivy, offline, FATAL on every # boot) churned the rootless port-forwarder until the WebUI's published port was # torn down — the container stayed healthy INSIDE, but nothing on the host could # reach it. So we probe the host-visible port forward, not just container state. # # Nothing here mutates state. # The core WebUI container + its internal port (the control plane we must keep # reachable). Kept as tiny functions so a rename only touches one place. _healthWebuiContainer() { echo "libreportal-service"; } _healthWebuiInternalPort() { echo "1111"; } # TCP-connect probe from the host (manager context). Success => the published # port forward is live. Uses bash /dev/tcp (always present) with a timeout so a # black-holed forward can't hang the poll. _healthTcpReachable() { local host="$1" port="$2" [[ -n "$port" ]] || return 2 timeout 4 bash -c "exec 3<>/dev/tcp/${host}/${port}" 2>/dev/null } dockerHealthScan() { HEALTH_DAEMON_OK="false"; HEALTH_WEBUI_PRESENT="false"; HEALTH_WEBUI_RUNNING="false" HEALTH_WEBUI_PORT=""; HEALTH_WEBUI_REACHABLE="unknown"; HEALTH_SCAN_ERROR="" HEALTH_CRASHLOOPS=() local limit="${CFG_HEALTH_CRASHLOOP_LIMIT:-3}" [[ "$limit" =~ ^[0-9]+$ ]] || limit=3 # Daemon reachable? Never alarm on what we can't verify — a daemon blip (or a # mid-recycle window) is transient, not a conflict. if ! dockerCommandRun "docker info" >/dev/null 2>&1; then HEALTH_SCAN_ERROR="docker daemon unreachable" return 0 fi HEALTH_DAEMON_OK="true" # Crash-loopers: containers docker reports as "restarting" whose RestartCount # has already climbed past the limit — i.e. actively churning, not a one-off # restart. Container name is enough to name the offender in the badge; derive # a friendly app label by trimming the conventional "-service" suffix. local names name rc app names=$(dockerCommandRun "docker ps -a --filter status=restarting --format '{{.Names}}'" 2>/dev/null) while IFS= read -r name; do [[ -n "$name" ]] || continue rc=$(dockerCommandRun "docker inspect --format '{{.RestartCount}}' '$name'" 2>/dev/null | tr -dc '0-9') [[ -n "$rc" ]] || rc=0 if (( rc >= limit )); then app="${name%-service}" HEALTH_CRASHLOOPS+=("${app}|${name}|${rc}") fi done <<< "$names" # Core WebUI container: present? running? host port reachable? local webui iport wstate webui="$(_healthWebuiContainer)" iport="$(_healthWebuiInternalPort)" wstate=$(dockerCommandRun "docker inspect --format '{{.State.Status}}' '$webui'" 2>/dev/null | tr -d '[:space:]') if [[ -z "$wstate" ]]; then return 0 # container not found — nothing more to probe fi HEALTH_WEBUI_PRESENT="true" [[ "$wstate" == "running" ]] && HEALTH_WEBUI_RUNNING="true" # docker's view of the published host port (e.g. "1111/tcp -> 0.0.0.0:9781"). # Note: this reports the INTENDED mapping even when the rootless forward has # been torn down — which is exactly why we then TCP-probe it for real. HEALTH_WEBUI_PORT=$(dockerCommandRun "docker port '$webui' '$iport'" 2>/dev/null | head -1 | sed -n 's/.*:\([0-9][0-9]*\)$/\1/p') if [[ "$HEALTH_WEBUI_RUNNING" == "true" && -n "$HEALTH_WEBUI_PORT" ]]; then if _healthTcpReachable "127.0.0.1" "$HEALTH_WEBUI_PORT"; then HEALTH_WEBUI_REACHABLE="true" else HEALTH_WEBUI_REACHABLE="false" fi fi }