LibrePortal/scripts/docker/health/docker_health_scan.sh

#!/bin/bash

# Read-only rootless-docker / control-plane health scan — the shared detection
# used by both the WebUI status generator (webuiSystemHealthCheck) and the heal
# verb (system health heal), so the two never diverge.
#
# dockerHealthScan sets these globals (call it DIRECTLY, never in $(...) — a
# subshell would drop them):
#   HEALTH_DAEMON_OK       "true"/"false" — rootless docker daemon reachable
#   HEALTH_WEBUI_PRESENT   "true"/"false" — the core WebUI container exists
#   HEALTH_WEBUI_RUNNING   "true"/"false" — ...and is running (not exited/restarting)
#   HEALTH_WEBUI_PORT                     — its published host port (docker port), "" if none
#   HEALTH_WEBUI_REACHABLE "true"/"false"/"unknown" — that host port accepts a TCP connect
#   HEALTH_SCAN_ERROR                     — human note when the daemon is off (else "")
#   HEALTH_CRASHLOOPS  (array)            — "app|container|restartcount" per container
#                                           crash-looping: RestartCount >= CFG_HEALTH_
#                                           CRASHLOOP_LIMIT AND (caught mid-restart OR
#                                           its count CLIMBED since the previous scan).
#   HEALTH_RESTART_SNAPSHOT               — "container|restartcount" lines for every
#                                           container, for the throttled caller to
#                                           persist as the next scan's delta baseline.
#
# The incident this guards: an app crash-loop (trivy, offline, FATAL on every
# boot) churned the rootless port-forwarder until the WebUI's published port was
# torn down — the container stayed healthy INSIDE, but nothing on the host could
# reach it. So we probe the host-visible port forward, not just container state.
#
# Nothing here mutates state.

# The core WebUI container + its internal port (the control plane we must keep
# reachable). Kept as tiny functions so a rename only touches one place.
_healthWebuiContainer()     { echo "libreportal-service"; }
_healthWebuiInternalPort()  { echo "1111"; }

# TCP-connect probe from the host (manager context). Success => the published
# port forward is live. Uses bash /dev/tcp (always present) with a timeout so a
# black-holed forward can't hang the poll.
_healthTcpReachable() {
    local host="$1" port="$2"
    [[ -n "$port" ]] || return 2
    timeout 4 bash -c "exec 3<>/dev/tcp/${host}/${port}" 2>/dev/null
}

dockerHealthScan() {
    HEALTH_DAEMON_OK="false"; HEALTH_WEBUI_PRESENT="false"; HEALTH_WEBUI_RUNNING="false"
    HEALTH_WEBUI_PORT=""; HEALTH_WEBUI_REACHABLE="unknown"; HEALTH_SCAN_ERROR=""
    HEALTH_CRASHLOOPS=()

    local limit="${CFG_HEALTH_CRASHLOOP_LIMIT:-3}"
    [[ "$limit" =~ ^[0-9]+$ ]] || limit=3

    # Daemon reachable? Never alarm on what we can't verify — a daemon blip (or a
    # mid-recycle window) is transient, not a conflict.
    if ! dockerCommandRun "docker info" >/dev/null 2>&1; then
        HEALTH_SCAN_ERROR="docker daemon unreachable"
        return 0
    fi
    HEALTH_DAEMON_OK="true"

    # Crash-loop detection. A backed-off loop sits "exited" between restarts
    # (docker's restart backoff grows to tens of seconds), so an instantaneous
    # `status=restarting` check misses it once it slows. We flag a container when
    # RestartCount >= limit AND either it's caught mid-restart OR its RestartCount
    # has CLIMBED since the previous scan — that delta over time is what "crash
    # loop" actually means (and it won't false-positive a container that restarted
    # a few times historically and is now stable). Baseline counts persist in
    # $state_file, written ONLY by the throttled caller (webuiSystemHealthCheck)
    # via HEALTH_RESTART_SNAPSHOT, so the heal's rapid re-scans don't disturb the
    # baseline. One inspect over all containers (name|count|status|restarting).
    local state_file="${containers_dir}/libreportal/frontend/data/system/.health_restart_counts"
    HEALTH_RESTART_SNAPSHOT=""

    declare -A _prev_rc=()
    if [[ -f "$state_file" ]]; then
        local pn pc
        while IFS='|' read -r pn pc; do
            [[ -n "$pn" ]] && _prev_rc["$pn"]="$pc"
        done < "$state_file"
    fi

    local snap name rc status restarting prev
    snap=$(dockerCommandRun "docker ps -aq | xargs -r docker inspect --format '{{.Name}}|{{.RestartCount}}|{{.State.Status}}|{{.State.Restarting}}'" 2>/dev/null)
    while IFS='|' read -r name rc status restarting; do
        name="${name#/}"                       # docker .Name carries a leading '/'
        [[ -n "$name" ]] || continue
        [[ "$rc" =~ ^[0-9]+$ ]] || rc=0
        HEALTH_RESTART_SNAPSHOT+="${name}|${rc}"$'\n'
        (( rc >= limit )) || continue
        prev="${_prev_rc[$name]-}"
        if [[ "$restarting" == "true" || "$status" == "restarting" ]] \
           || { [[ "$prev" =~ ^[0-9]+$ ]] && (( rc > prev )); }; then
            HEALTH_CRASHLOOPS+=("${name%-service}|${name}|${rc}")
        fi
    done <<< "$snap"

    # Core WebUI container: present? running? host port reachable?
    local webui iport wstate
    webui="$(_healthWebuiContainer)"
    iport="$(_healthWebuiInternalPort)"

    wstate=$(dockerCommandRun "docker inspect --format '{{.State.Status}}' '$webui'" 2>/dev/null | tr -d '[:space:]')
    if [[ -z "$wstate" ]]; then
        return 0   # container not found — nothing more to probe
    fi
    HEALTH_WEBUI_PRESENT="true"
    [[ "$wstate" == "running" ]] && HEALTH_WEBUI_RUNNING="true"

    # docker's view of the published host port (e.g. "1111/tcp -> 0.0.0.0:9781").
    # Note: this reports the INTENDED mapping even when the rootless forward has
    # been torn down — which is exactly why we then TCP-probe it for real.
    HEALTH_WEBUI_PORT=$(dockerCommandRun "docker port '$webui' '$iport'" 2>/dev/null | head -1 | sed -n 's/.*:\([0-9][0-9]*\)$/\1/p')

    if [[ "$HEALTH_WEBUI_RUNNING" == "true" && -n "$HEALTH_WEBUI_PORT" ]]; then
        if _healthTcpReachable "127.0.0.1" "$HEALTH_WEBUI_PORT"; then
            HEALTH_WEBUI_REACHABLE="true"
        else
            HEALTH_WEBUI_REACHABLE="false"
        fi
    fi
}