LibrePortal/scripts/cli/commands/system/cli_system_commands.sh
librelad 2d1e4aa98f feat(health): self-healing control-plane watchdog + crash-loop failure cap
An offline trivy install crash-looped (server FATALs when it can't fetch the
vuln DB), and on rootless docker the restart storm churned the shared network's
port-forwarder until the WebUI's own published host port was torn down — the
WebUI stayed healthy INSIDE its container but was unreachable from the host, with
nothing detecting or healing it.

Three fixes, in the house self-healing style (mirrors the network-drift trio):

1. Control-plane health checker wired into the existing task-processor idle poll
   (maybeRegenPoll), no new daemon. dockerHealthScan (read-only) detects daemon
   down, a WebUI running-but-host-port-unreachable (the port-forward corruption),
   and crash-looping containers. webuiSystemHealthCheck writes
   frontend/data/system/health_status.json + self-dispatches a heal — the user
   can't click a button on a dead WebUI, so the poll drives the fix. Frontend
   health-notifier surfaces a topbar badge + dashboard banner + details panel.

2. Failure cap, enforced centrally by dockerHealthHeal (task-gated): stops
   crash-loopers (removing the churn), restarts the WebUI to re-publish a lost
   port forward, and — only if that fails — recycles the rootless daemon and
   restarts the core container. Caps every app immediately, no template churn.

3. Trivy no longer crash-loops offline: the server runs in a shell retry-loop so
   the container stays Up and quietly retries on a backoff instead of exiting
   FATAL. Verified: container stays Up across repeated DB-download failures.
   Core WebUI compose gains restart: unless-stopped so it self-recovers after a
   reboot / daemon recycle instead of staying down.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: librelad <librelad@digitalangels.vip>
2026-07-17 22:03:02 +01:00

162 lines
5.6 KiB
Bash
Executable File

#!/bin/bash
# System Commands Handler
# Handles all system subcommands by calling core functions
# Safe disk reclaim: clear the whole build cache (-a; it's pure cache, always
# safe to drop) and remove dangling images. Never touches volumes or in-use
# images. runFileOp hits the right daemon (rootless: as the install user).
reclaimDockerSpace()
{
isHeader "Reclaiming Space"
runFileOp docker builder prune -af >/dev/null 2>&1
checkSuccess "Cleared build cache"
runFileOp docker image prune -f >/dev/null 2>&1
checkSuccess "Removed dangling images"
isSuccessful "Done"
}
# Remove specific Docker images by id/ref. Args:
# $1 force flag — "-f" to force-remove (e.g. in-use images), else empty
# $2 comma-separated list of image refs (the WebUI sends full sha256:… ids)
# The WebUI Storage page calls this through the task system (see the
# system_image_rm action) — never a direct API. Each ref is validated before it
# reaches docker; removal continues past per-image failures and reports a tally.
removeDockerImages()
{
local force_flag="$1" ids_csv="$2"
isHeader "Removing Images"
if [[ -z "$ids_csv" ]]; then
isError "No images specified."
return 1
fi
local removed=0 failed=0 id
local IFS=','
local -a ids=($ids_csv)
unset IFS
for id in "${ids[@]}"; do
id="${id//[[:space:]]/}"
[[ -n "$id" ]] || continue
# Accept only a sha256 digest or a conservative repo[:tag] ref — no shell
# metacharacters can reach docker even though this arrives via the task
# command string.
if [[ ! "$id" =~ ^sha256:[a-f0-9]{12,64}$ && ! "$id" =~ ^[A-Za-z0-9][A-Za-z0-9._/:@-]*$ ]]; then
isError "Skipping invalid image ref: $id"
failed=$((failed + 1)); continue
fi
# $force_flag is intentionally unquoted: it's either empty or "-f".
if runFileOp docker image rm $force_flag "$id" >/dev/null 2>&1; then
isSuccessful "Removed $id"
removed=$((removed + 1))
else
isError "Could not remove $id (in use, or has dependent child images)"
failed=$((failed + 1))
fi
done
isNotice "Done — removed ${removed}, failed/skipped ${failed}."
[[ "$failed" -eq 0 ]]
}
cliHandleSystemCommands()
{
local action="$initial_command2"
case "$action" in
"status")
tagsValidateShowSystemStatus
;;
"update")
checkUpdates
;;
"reset")
runReinstall
;;
"reclaim")
reclaimDockerSpace
;;
"network")
# libreportal system network check [force] (read-only, rewrites
# network_status.json — used by the task-processor poll + WebUI)
# libreportal system network heal [<app>] (mutating — re-IPs
# stranded apps from the corrected subnet, ports preserved; routes
# through the task system like update apply)
case "$initial_command3" in
"check")
webuiSystemNetworkCheck "${initial_command4:-force}"
;;
"heal")
if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then
networkHealConflicts "$initial_command4"
else
cliTaskRun "libreportal system network heal${initial_command4:+ $initial_command4}" "system_network_heal" "" ""
fi
;;
*)
isNotice "Invalid network command: $initial_command3"
cliShowSystemHelp
;;
esac
;;
"health")
# libreportal system health check [force] (read-only, rewrites
# health_status.json — used by the task-processor poll + WebUI;
# self-dispatches a heal when the control plane is unreachable)
# libreportal system health heal (mutating — stops
# crash-loopers, repairs the WebUI port forward, recycles the
# rootless daemon; routes through the task system like network heal)
case "$initial_command3" in
"check")
webuiSystemHealthCheck "${initial_command4:-force}"
;;
"heal")
if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then
dockerHealthHeal
else
cliTaskRun "libreportal system health heal" "system_health_heal" "" ""
fi
;;
*)
isNotice "Invalid health command: $initial_command3"
cliShowSystemHelp
;;
esac
;;
"image")
# libreportal system image rm [--force] <comma-separated ids>
case "$initial_command3" in
"rm"|"remove")
local img_force="" img_ids=""
if [[ "$initial_command4" == "--force" || "$initial_command4" == "-f" ]]; then
img_force="-f"; img_ids="$initial_command5"
else
img_ids="$initial_command4"
fi
removeDockerImages "$img_force" "$img_ids"
;;
*)
isNotice "Invalid image command: $initial_command3"
cliShowSystemHelp
;;
esac
;;
*)
isNotice "Invalid system command: $action"
cliShowSystemHelp
;;
esac
}