LibrePortal/scripts/docker/health/docker_health_heal.sh

#!/bin/bash

# Rootless-docker / control-plane health heal — the mutating half of the
# detector. Runs ONLY through the task system (see cli_system_commands.sh
# `health heal`, which enqueues unless LIBREPORTAL_TASK_EXEC=1), never a direct
# API. It re-scans before each heavier step so it does the least it can.
#
# Escalation ladder:
#   1) Stop crash-loopers — an app FATAL-ing on every boot (e.g. trivy offline)
#      churns the rootless port-forwarder; stopping it removes the churn. This is
#      the failure cap: a broken opt-in app can't take the control plane down.
#      (unless-stopped honours a manual stop across a daemon recycle, so a
#      stopped crash-looper stays down and can't resume the churn.)
#   2) Restart the WebUI container — re-publishes a mildly-lost port forward.
#   3) Recycle the rootless docker daemon — rebuilds the netns/port-forward state
#      when churn has already corrupted it (a container restart alone won't fix
#      it), then explicitly starts the core container (it may carry no restart
#      policy, so a daemon recycle would otherwise leave it down).
#
# Re-runs the read-only check at the end to rewrite health_status.json (the badge
# clears, or stays if anything is still unhealed).
#
#   dockerHealthHeal
dockerHealthHeal() {
    isHeader "Healing docker / control-plane health"

    dockerHealthScan   # populate HEALTH_* (call direct, NOT in $(...))

    if [[ "$HEALTH_DAEMON_OK" != "true" ]]; then
        isError "Docker daemon unreachable — cannot heal from here."
        declare -f webuiSystemHealthCheck >/dev/null 2>&1 && webuiSystemHealthCheck "force" >/dev/null 2>&1
        return 1
    fi

    local rootless="true"
    [[ "${CFG_DOCKER_INSTALL_TYPE:-rootless}" == "rootless" ]] || rootless="false"

    local acted=0

    # 1) Stop crash-loopers (the failure cap).
    local row cname
    for row in "${HEALTH_CRASHLOOPS[@]}"; do
        cname="${row#*|}"; cname="${cname%%|*}"   # app|container|rc -> container
        [[ "$cname" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]] || continue
        isNotice "Stopping crash-looping container '$cname' (restart storm churning the network)…"
        dockerCommandRun "docker stop '$cname'" >/dev/null 2>&1
        ((acted++))
    done

    # 2/3) Restore the WebUI host port if the container is up but unreachable.
    if [[ "$HEALTH_WEBUI_RUNNING" == "true" && "$HEALTH_WEBUI_REACHABLE" == "false" ]]; then
        local webui; webui="$(_healthWebuiContainer)"
        isNotice "WebUI up but host port ${HEALTH_WEBUI_PORT:-?} unreachable — restarting '$webui' to re-publish…"
        dockerCommandRun "docker restart '$webui'" >/dev/null 2>&1
        ((acted++))
        sleep 4
        dockerHealthScan

        if [[ "$HEALTH_WEBUI_REACHABLE" != "true" && "$rootless" == "true" ]]; then
            isNotice "Still unreachable — recycling the rootless docker daemon to rebuild port-forward state…"
            dockerCommandRun "systemctl --user restart docker.service" >/dev/null 2>&1
            sleep 8
            # The core container may have no restart policy — bring it back explicitly.
            dockerCommandRun "docker start '$webui'" >/dev/null 2>&1
            sleep 4
            ((acted++))
        fi
    fi

    # Rewrite the status file + report what (if anything) remains.
    declare -f webuiSystemHealthCheck >/dev/null 2>&1 && webuiSystemHealthCheck "force" >/dev/null 2>&1
    dockerHealthScan

    if [[ "$HEALTH_WEBUI_RUNNING" == "true" && "$HEALTH_WEBUI_REACHABLE" == "false" ]]; then
        isError "Health heal ran ${acted} action(s); WebUI still unreachable on port ${HEALTH_WEBUI_PORT:-?} — manual inspection needed."
        return 1
    fi
    if (( acted == 0 )); then
        isSuccessful "Nothing to heal — control plane healthy."
    else
        isSuccessful "Health heal complete — ${acted} action(s); control plane reachable."
    fi
}