LibrePortal/scripts/docker/health/docker_health_scan.sh

#!/bin/bash

# Read-only rootless-docker / control-plane health scan — the shared detection
# used by both the WebUI status generator (webuiSystemHealthCheck) and the heal
# verb (system health heal), so the two never diverge.
#
# dockerHealthScan sets these globals (call it DIRECTLY, never in $(...) — a
# subshell would drop them):
#   HEALTH_DAEMON_OK       "true"/"false" — rootless docker daemon reachable
#   HEALTH_WEBUI_PRESENT   "true"/"false" — the core WebUI container exists
#   HEALTH_WEBUI_RUNNING   "true"/"false" — ...and is running (not exited/restarting)
#   HEALTH_WEBUI_PORT                     — its published host port (docker port), "" if none
#   HEALTH_WEBUI_REACHABLE "true"/"false"/"unknown" — that host port accepts a TCP connect
#   HEALTH_SCAN_ERROR                     — human note when the daemon is off (else "")
#   HEALTH_CRASHLOOPS  (array)            — "app|container|restartcount" per container
#                                           stuck restarting (State=restarting and
#                                           RestartCount >= CFG_HEALTH_CRASHLOOP_LIMIT):
#                                           a crash-loop churning the shared network.
#
# The incident this guards: an app crash-loop (trivy, offline, FATAL on every
# boot) churned the rootless port-forwarder until the WebUI's published port was
# torn down — the container stayed healthy INSIDE, but nothing on the host could
# reach it. So we probe the host-visible port forward, not just container state.
#
# Nothing here mutates state.

# The core WebUI container + its internal port (the control plane we must keep
# reachable). Kept as tiny functions so a rename only touches one place.
_healthWebuiContainer()     { echo "libreportal-service"; }
_healthWebuiInternalPort()  { echo "1111"; }

# TCP-connect probe from the host (manager context). Success => the published
# port forward is live. Uses bash /dev/tcp (always present) with a timeout so a
# black-holed forward can't hang the poll.
_healthTcpReachable() {
    local host="$1" port="$2"
    [[ -n "$port" ]] || return 2
    timeout 4 bash -c "exec 3<>/dev/tcp/${host}/${port}" 2>/dev/null
}

dockerHealthScan() {
    HEALTH_DAEMON_OK="false"; HEALTH_WEBUI_PRESENT="false"; HEALTH_WEBUI_RUNNING="false"
    HEALTH_WEBUI_PORT=""; HEALTH_WEBUI_REACHABLE="unknown"; HEALTH_SCAN_ERROR=""
    HEALTH_CRASHLOOPS=()

    local limit="${CFG_HEALTH_CRASHLOOP_LIMIT:-3}"
    [[ "$limit" =~ ^[0-9]+$ ]] || limit=3

    # Daemon reachable? Never alarm on what we can't verify — a daemon blip (or a
    # mid-recycle window) is transient, not a conflict.
    if ! dockerCommandRun "docker info" >/dev/null 2>&1; then
        HEALTH_SCAN_ERROR="docker daemon unreachable"
        return 0
    fi
    HEALTH_DAEMON_OK="true"

    # Crash-loopers: containers docker reports as "restarting" whose RestartCount
    # has already climbed past the limit — i.e. actively churning, not a one-off
    # restart. Container name is enough to name the offender in the badge; derive
    # a friendly app label by trimming the conventional "-service" suffix.
    local names name rc app
    names=$(dockerCommandRun "docker ps -a --filter status=restarting --format '{{.Names}}'" 2>/dev/null)
    while IFS= read -r name; do
        [[ -n "$name" ]] || continue
        rc=$(dockerCommandRun "docker inspect --format '{{.RestartCount}}' '$name'" 2>/dev/null | tr -dc '0-9')
        [[ -n "$rc" ]] || rc=0
        if (( rc >= limit )); then
            app="${name%-service}"
            HEALTH_CRASHLOOPS+=("${app}|${name}|${rc}")
        fi
    done <<< "$names"

    # Core WebUI container: present? running? host port reachable?
    local webui iport wstate
    webui="$(_healthWebuiContainer)"
    iport="$(_healthWebuiInternalPort)"

    wstate=$(dockerCommandRun "docker inspect --format '{{.State.Status}}' '$webui'" 2>/dev/null | tr -d '[:space:]')
    if [[ -z "$wstate" ]]; then
        return 0   # container not found — nothing more to probe
    fi
    HEALTH_WEBUI_PRESENT="true"
    [[ "$wstate" == "running" ]] && HEALTH_WEBUI_RUNNING="true"

    # docker's view of the published host port (e.g. "1111/tcp -> 0.0.0.0:9781").
    # Note: this reports the INTENDED mapping even when the rootless forward has
    # been torn down — which is exactly why we then TCP-probe it for real.
    HEALTH_WEBUI_PORT=$(dockerCommandRun "docker port '$webui' '$iport'" 2>/dev/null | head -1 | sed -n 's/.*:\([0-9][0-9]*\)$/\1/p')

    if [[ "$HEALTH_WEBUI_RUNNING" == "true" && -n "$HEALTH_WEBUI_PORT" ]]; then
        if _healthTcpReachable "127.0.0.1" "$HEALTH_WEBUI_PORT"; then
            HEALTH_WEBUI_REACHABLE="true"
        else
            HEALTH_WEBUI_REACHABLE="false"
        fi
    fi
}