#!/bin/bash # Rootless-docker / control-plane health heal — the mutating half of the # detector. Runs ONLY through the task system (see cli_system_commands.sh # `health heal`, which enqueues unless LIBREPORTAL_TASK_EXEC=1), never a direct # API. It re-scans before each heavier step so it does the least it can. # # Escalation ladder: # 1) Stop crash-loopers — an app FATAL-ing on every boot (e.g. trivy offline) # churns the rootless port-forwarder; stopping it removes the churn. This is # the failure cap: a broken opt-in app can't take the control plane down. # (unless-stopped honours a manual stop across a daemon recycle, so a # stopped crash-looper stays down and can't resume the churn.) # 2) Restart the WebUI container — re-publishes a mildly-lost port forward. # 3) Recycle the rootless docker daemon — rebuilds the netns/port-forward state # when churn has already corrupted it (a container restart alone won't fix # it), then explicitly starts the core container (it may carry no restart # policy, so a daemon recycle would otherwise leave it down). # # Re-runs the read-only check at the end to rewrite health_status.json (the badge # clears, or stays if anything is still unhealed). # # dockerHealthHeal dockerHealthHeal() { isHeader "Fixing system issues" dockerHealthScan # populate HEALTH_* (call direct, NOT in $(...)) if [[ "$HEALTH_DAEMON_OK" != "true" ]]; then isError "Docker engine unreachable — cannot fix from here." declare -f webuiSystemHealthCheck >/dev/null 2>&1 && webuiSystemHealthCheck "force" >/dev/null 2>&1 return 1 fi local rootless="true" [[ "${CFG_DOCKER_INSTALL_TYPE:-rootless}" == "rootless" ]] || rootless="false" local acted=0 # 1) Stop crash-loopers (the failure cap). local row cname for row in "${HEALTH_CRASHLOOPS[@]}"; do cname="${row#*|}"; cname="${cname%%|*}" # app|container|rc -> container [[ "$cname" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]] || continue isNotice "Stopping '$cname' — stuck in a restart loop (churning the network)…" dockerCommandRun "docker stop '$cname'" >/dev/null 2>&1 ((acted++)) done # 2/3) Restore the WebUI host port if the container is up but unreachable. if [[ "$HEALTH_WEBUI_RUNNING" == "true" && "$HEALTH_WEBUI_REACHABLE" == "false" ]]; then local webui; webui="$(_healthWebuiContainer)" isNotice "WebUI unreachable on port ${HEALTH_WEBUI_PORT:-?} — restarting '$webui' to restore access…" dockerCommandRun "docker restart '$webui'" >/dev/null 2>&1 ((acted++)) sleep 4 dockerHealthScan if [[ "$HEALTH_WEBUI_REACHABLE" != "true" && "$rootless" == "true" ]]; then isNotice "Still unreachable — restarting Docker to restore access…" dockerCommandRun "systemctl --user restart docker.service" >/dev/null 2>&1 sleep 8 # The core container may have no restart policy — bring it back explicitly. dockerCommandRun "docker start '$webui'" >/dev/null 2>&1 sleep 4 ((acted++)) fi fi # Rewrite the status file + report what (if anything) remains. declare -f webuiSystemHealthCheck >/dev/null 2>&1 && webuiSystemHealthCheck "force" >/dev/null 2>&1 dockerHealthScan if [[ "$HEALTH_WEBUI_RUNNING" == "true" && "$HEALTH_WEBUI_REACHABLE" == "false" ]]; then isError "Ran ${acted} fix action(s); WebUI still unreachable on port ${HEALTH_WEBUI_PORT:-?} — manual check needed." return 1 fi if (( acted == 0 )); then isSuccessful "Nothing to fix — system healthy." else isSuccessful "Fixed — ${acted} action(s); WebUI reachable." fi }