Merge claude/1
This commit is contained in:
librelad
commit
e66a5759da
@ -41,9 +41,13 @@ Object.assign(TasksManager.prototype, {
|
||||
{ match: /^libreportal regen\b/, title: 'LibrePortal - Regenerate WebUI Data' },
|
||||
|
||||
// -- System maintenance ------------------------------------------------
|
||||
{ match: /^libreportal system reclaim\b/, title: 'LibrePortal - Reclaim Space' },
|
||||
{ match: /^libreportal system image rm\b/, title: 'LibrePortal - Remove Images' },
|
||||
{ match: /^libreportal verify\b/, title: 'LibrePortal - Verify System' },
|
||||
{ match: /^libreportal system reclaim\b/, title: 'LibrePortal - Reclaim Space' },
|
||||
{ match: /^libreportal system image rm\b/, title: 'LibrePortal - Remove Images' },
|
||||
{ match: /^libreportal verify\b/, title: 'LibrePortal - Verify System' },
|
||||
{ match: /^libreportal system health heal\b/, title: 'LibrePortal - Repair Control Plane' },
|
||||
{ match: /^libreportal system health check\b/, title: 'LibrePortal - Check Control Plane' },
|
||||
{ match: /^libreportal system network heal\b/, title: 'LibrePortal - Heal Network' },
|
||||
{ match: /^libreportal system network check\b/, title: 'LibrePortal - Check Network' },
|
||||
|
||||
// -- Backup: per-app (these capture the app slug) ----------------------
|
||||
{ match: /^libreportal backup app create (\w+)/, title: (m) => `${displayName(m[1])} - Create Backup` },
|
||||
@ -138,6 +142,7 @@ Object.assign(TasksManager.prototype, {
|
||||
'config_update': 'Update Config', 'update_config': 'Update Config',
|
||||
'system_update': 'Update System', 'system_reclaim': 'Reclaim Space',
|
||||
'system_image_rm': 'Remove Images', 'verify': 'Verify System',
|
||||
'system_health_heal': 'Repair Control Plane', 'system_network_heal': 'Heal Network',
|
||||
'updater_check': 'Check for Updates', 'updater_apply': 'Update',
|
||||
'updater_apply_all': 'Update All', 'updater_rollback': 'Roll Back',
|
||||
'artifact_apply': 'Apply Hotfix', 'artifact_revert': 'Revert Hotfix',
|
||||
|
||||
@ -306,6 +306,8 @@ Object.assign(TasksManager.prototype, {
|
||||
'delete': { icon: '🗑️', class: 'delete' },
|
||||
'delete_all': { icon: '🗑️', class: 'delete' },
|
||||
'system_image_rm': { icon: '🗑️', class: 'delete' },
|
||||
'system_health_heal': { icon: '🩺', class: 'verify' },
|
||||
'system_network_heal': { icon: '🌐', class: 'restart' },
|
||||
'verify': { icon: '🛡️', class: 'verify' },
|
||||
'setup-config': { icon: '🛠️', class: 'setup' },
|
||||
'setup-finalize': { icon: '🎉', class: 'setup' },
|
||||
|
||||
@ -13,9 +13,12 @@
|
||||
# HEALTH_WEBUI_REACHABLE "true"/"false"/"unknown" — that host port accepts a TCP connect
|
||||
# HEALTH_SCAN_ERROR — human note when the daemon is off (else "")
|
||||
# HEALTH_CRASHLOOPS (array) — "app|container|restartcount" per container
|
||||
# stuck restarting (State=restarting and
|
||||
# RestartCount >= CFG_HEALTH_CRASHLOOP_LIMIT):
|
||||
# a crash-loop churning the shared network.
|
||||
# crash-looping: RestartCount >= CFG_HEALTH_
|
||||
# CRASHLOOP_LIMIT AND (caught mid-restart OR
|
||||
# its count CLIMBED since the previous scan).
|
||||
# HEALTH_RESTART_SNAPSHOT — "container|restartcount" lines for every
|
||||
# container, for the throttled caller to
|
||||
# persist as the next scan's delta baseline.
|
||||
#
|
||||
# The incident this guards: an app crash-loop (trivy, offline, FATAL on every
|
||||
# boot) churned the rootless port-forwarder until the WebUI's published port was
|
||||
@ -54,21 +57,41 @@ dockerHealthScan() {
|
||||
fi
|
||||
HEALTH_DAEMON_OK="true"
|
||||
|
||||
# Crash-loopers: containers docker reports as "restarting" whose RestartCount
|
||||
# has already climbed past the limit — i.e. actively churning, not a one-off
|
||||
# restart. Container name is enough to name the offender in the badge; derive
|
||||
# a friendly app label by trimming the conventional "-service" suffix.
|
||||
local names name rc app
|
||||
names=$(dockerCommandRun "docker ps -a --filter status=restarting --format '{{.Names}}'" 2>/dev/null)
|
||||
while IFS= read -r name; do
|
||||
# Crash-loop detection. A backed-off loop sits "exited" between restarts
|
||||
# (docker's restart backoff grows to tens of seconds), so an instantaneous
|
||||
# `status=restarting` check misses it once it slows. We flag a container when
|
||||
# RestartCount >= limit AND either it's caught mid-restart OR its RestartCount
|
||||
# has CLIMBED since the previous scan — that delta over time is what "crash
|
||||
# loop" actually means (and it won't false-positive a container that restarted
|
||||
# a few times historically and is now stable). Baseline counts persist in
|
||||
# $state_file, written ONLY by the throttled caller (webuiSystemHealthCheck)
|
||||
# via HEALTH_RESTART_SNAPSHOT, so the heal's rapid re-scans don't disturb the
|
||||
# baseline. One inspect over all containers (name|count|status|restarting).
|
||||
local state_file="${containers_dir}/libreportal/frontend/data/system/.health_restart_counts"
|
||||
HEALTH_RESTART_SNAPSHOT=""
|
||||
|
||||
declare -A _prev_rc=()
|
||||
if [[ -f "$state_file" ]]; then
|
||||
local pn pc
|
||||
while IFS='|' read -r pn pc; do
|
||||
[[ -n "$pn" ]] && _prev_rc["$pn"]="$pc"
|
||||
done < "$state_file"
|
||||
fi
|
||||
|
||||
local snap name rc status restarting prev
|
||||
snap=$(dockerCommandRun "docker ps -aq | xargs -r docker inspect --format '{{.Name}}|{{.RestartCount}}|{{.State.Status}}|{{.State.Restarting}}'" 2>/dev/null)
|
||||
while IFS='|' read -r name rc status restarting; do
|
||||
name="${name#/}" # docker .Name carries a leading '/'
|
||||
[[ -n "$name" ]] || continue
|
||||
rc=$(dockerCommandRun "docker inspect --format '{{.RestartCount}}' '$name'" 2>/dev/null | tr -dc '0-9')
|
||||
[[ -n "$rc" ]] || rc=0
|
||||
if (( rc >= limit )); then
|
||||
app="${name%-service}"
|
||||
HEALTH_CRASHLOOPS+=("${app}|${name}|${rc}")
|
||||
[[ "$rc" =~ ^[0-9]+$ ]] || rc=0
|
||||
HEALTH_RESTART_SNAPSHOT+="${name}|${rc}"$'\n'
|
||||
(( rc >= limit )) || continue
|
||||
prev="${_prev_rc[$name]-}"
|
||||
if [[ "$restarting" == "true" || "$status" == "restarting" ]] \
|
||||
|| { [[ "$prev" =~ ^[0-9]+$ ]] && (( rc > prev )); }; then
|
||||
HEALTH_CRASHLOOPS+=("${name%-service}|${name}|${rc}")
|
||||
fi
|
||||
done <<< "$names"
|
||||
done <<< "$snap"
|
||||
|
||||
# Core WebUI container: present? running? host port reachable?
|
||||
local webui iport wstate
|
||||
|
||||
@ -112,6 +112,15 @@ webuiSystemHealthCheck() {
|
||||
EOF
|
||||
runFileWrite "$final_file" < "$temp_file"; rm -f "$temp_file"
|
||||
|
||||
# Persist the restart-count baseline for the next scan's crash-loop delta.
|
||||
# ONLY the throttled check writes it (the heal re-scans between actions and
|
||||
# must not disturb the baseline), so the delta measures a full interval.
|
||||
if [[ -n "$HEALTH_RESTART_SNAPSHOT" ]]; then
|
||||
local rc_tmp; rc_tmp="$(mktemp)"
|
||||
printf '%s' "$HEALTH_RESTART_SNAPSHOT" > "$rc_tmp"
|
||||
runFileWrite "${system_dir}/.health_restart_counts" < "$rc_tmp"; rm -f "$rc_tmp"
|
||||
fi
|
||||
|
||||
# Self-dispatch a heal on an auto-healable problem, throttled by a cooldown
|
||||
# stamp. The WebUI can't be clicked when its own port is down, so the poll
|
||||
# must drive the fix — this is the backend counterpart of the "Heal now"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user